PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $confdesc = {
 113     onboot => {
 114         optional => 1,
 115         type => 'boolean',
 116         description => "Specifies whether a VM will be started during system bootup.",
 117         default => 0,
 118     },
 119     autostart => {
 120         optional => 1,
 121         type => 'boolean',
 122         description => "Automatic restart after crash (currently ignored).",
 123         default => 0,
 124     },
 125     hotplug => {
 126         optional => 1,
 127         type => 'string', format => 'pve-hotplug-features',
 128         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 129         default => 'network,disk,usb',
 130     },
 131     reboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 135         default => 1,
 136     },
 137     lock => {
 138         optional => 1,
 139         type => 'string',
 140         description => "Lock/unlock the VM.",
 141         enum => [qw(migrate backup snapshot rollback)],
 142     },
 143     cpulimit => {
 144         optional => 1,
 145         type => 'number',
 146         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 147         minimum => 0,
 148         maximum => 128,
 149         default => 0,
 150     },
 151     cpuunits => {
 152         optional => 1,
 153         type => 'integer',
 154         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 155         minimum => 0,
 156         maximum => 500000,
 157         default => 1000,
 158     },
 159     memory => {
 160         optional => 1,
 161         type => 'integer',
 162         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 163         minimum => 16,
 164         default => 512,
 165     },
 166     balloon => {
 167         optional => 1,
 168         type => 'integer',
 169         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 170         minimum => 0,
 171     },
 172     shares => {
 173         optional => 1,
 174         type => 'integer',
 175         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 176         minimum => 0,
 177         maximum => 50000,
 178         default => 1000,
 179     },
 180     keyboard => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 184         enum => PVE::Tools::kvmkeymaplist(),
 185         default => 'en-us',
 186     },
 187     name => {
 188         optional => 1,
 189         type => 'string', format => 'dns-name',
 190         description => "Set a name for the VM. Only used on the configuration web interface.",
 191     },
 192     scsihw => {
 193         optional => 1,
 194         type => 'string',
 195         description => "scsi controller model",
 196         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 197         default => 'lsi',
 198     },
 199     description => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 203     },
 204     ostype => {
 205         optional => 1,
 206         type => 'string',
 207         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 208         description => <<EODESC,
 209 Used to enable special optimization/features for specific
 210 operating systems:
 211
 212 other  => unspecified OS
 213 wxp    => Microsoft Windows XP
 214 w2k    => Microsoft Windows 2000
 215 w2k3   => Microsoft Windows 2003
 216 w2k8   => Microsoft Windows 2008
 217 wvista => Microsoft Windows Vista
 218 win7   => Microsoft Windows 7
 219 win8   => Microsoft Windows 8/2012
 220 l24    => Linux 2.4 Kernel
 221 l26    => Linux 2.6/3.X Kernel
 222 solaris => solaris/opensolaris/openindiania kernel
 223
 224 other|l24|l26|solaris                       ... no special behaviour
 225 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 226 EODESC
 227     },
 228     boot => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 232         pattern => '[acdn]{1,4}',
 233         default => 'cdn',
 234     },
 235     bootdisk => {
 236         optional => 1,
 237         type => 'string', format => 'pve-qm-bootdisk',
 238         description => "Enable booting from specified disk.",
 239         pattern => '(ide|sata|scsi|virtio)\d+',
 240     },
 241     smp => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "The number of CPUs. Please use option -sockets instead.",
 245         minimum => 1,
 246         default => 1,
 247     },
 248     sockets => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "The number of CPU sockets.",
 252         minimum => 1,
 253         default => 1,
 254     },
 255     cores => {
 256         optional => 1,
 257         type => 'integer',
 258         description => "The number of cores per socket.",
 259         minimum => 1,
 260         default => 1,
 261     },
 262     numa => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable NUMA.",
 266         default => 0,
 267     },
 268     vcpus => {
 269         optional => 1,
 270         type => 'integer',
 271         description => "Number of hotplugged vcpus.",
 272         minimum => 1,
 273         default => 0,
 274     },
 275     acpi => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Enable/disable ACPI.",
 279         default => 1,
 280     },
 281     agent => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Enable/disable Qemu GuestAgent.",
 285         default => 0,
 286     },
 287     kvm => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Enable/disable KVM hardware virtualization.",
 291         default => 1,
 292     },
 293     tdf => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable time drift fix.",
 297         default => 0,
 298     },
 299     localtime => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 303     },
 304     freeze => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 308     },
 309     vga => {
 310         optional => 1,
 311         type => 'string',
 312         description => "Select the VGA type. If you want to use high resolution" .
 313             " modes (>= 1280x1024x16) then you should use the options " .
 314             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 315             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 316             "display sever. For win* OS you can select how many independent " .
 317             "displays you want, Linux guests can add displays them self. " .
 318             "You can also run without any graphic card, using a serial device" .
 319             " as terminal.",
 320         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 321     },
 322     watchdog => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-watchdog',
 325         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 326         description => "Create a virtual hardware watchdog device. Once enabled" .
 327             " (by a guest action), the watchdog must be periodically polled " .
 328             "by an agent inside the guest or else the watchdog will reset " .
 329             "the guest (or execute the respective action specified)",
 330     },
 331     startdate => {
 332         optional => 1,
 333         type => 'string',
 334         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 335         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 336         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 337         default => 'now',
 338     },
 339     startup =>  get_standard_option('pve-startup-order'),
 340     template => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Enable/disable Template.",
 344         default => 0,
 345     },
 346     args => {
 347         optional => 1,
 348         type => 'string',
 349         description => <<EODESCR,
 350 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 351
 352 args: -no-reboot -no-hpet
 353 EODESCR
 354     },
 355     tablet => {
 356         optional => 1,
 357         type => 'boolean',
 358         default => 1,
 359         description => "Enable/disable the USB tablet device. This device is " .
 360             "usually needed to allow absolute mouse positioning with VNC. " .
 361             "Else the mouse runs out of sync with normal VNC clients. " .
 362             "If you're running lots of console-only guests on one host, " .
 363             "you may consider disabling this to save some context switches. " .
 364             "This is turned off by default if you use spice (-vga=qxl).",
 365     },
 366     migrate_speed => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 370         minimum => 0,
 371         default => 0,
 372     },
 373     migrate_downtime => {
 374         optional => 1,
 375         type => 'number',
 376         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 377         minimum => 0,
 378         default => 0.1,
 379     },
 380     cdrom => {
 381         optional => 1,
 382         type => 'string', format => 'pve-qm-drive',
 383         typetext => 'volume',
 384         description => "This is an alias for option -ide2",
 385     },
 386     cpu => {
 387         optional => 1,
 388         description => "Emulated CPU type.",
 389         type => 'string',
 390         format => $cpu_fmt,
 391     },
 392     parent => get_standard_option('pve-snapshot-name', {
 393         optional => 1,
 394         description => "Parent snapshot name. This is used internally, and should not be modified.",
 395     }),
 396     snaptime => {
 397         optional => 1,
 398         description => "Timestamp for snapshots.",
 399         type => 'integer',
 400         minimum => 0,
 401     },
 402     vmstate => {
 403         optional => 1,
 404         type => 'string', format => 'pve-volume-id',
 405         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 406     },
 407     machine => {
 408         description => "Specific the Qemu machine type.",
 409         type => 'string',
 410         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 411         maxLength => 40,
 412         optional => 1,
 413     },
 414     smbios1 => {
 415         description => "Specify SMBIOS type 1 fields.",
 416         type => 'string', format => 'pve-qm-smbios1',
 417         maxLength => 256,
 418         optional => 1,
 419     },
 420     protection => {
 421         optional => 1,
 422         type => 'boolean',
 423         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 424         default => 0,
 425     },
 426     bios => {
 427         optional => 1,
 428         type => 'string',
 429         enum => [ qw(seabios ovmf) ],
 430         description => "Select BIOS implementation.",
 431         default => 'seabios',
 432     },
 433 };
 434
 435 # what about other qemu settings ?
 436 #cpu => 'string',
 437 #machine => 'string',
 438 #fda => 'file',
 439 #fdb => 'file',
 440 #mtdblock => 'file',
 441 #sd => 'file',
 442 #pflash => 'file',
 443 #snapshot => 'bool',
 444 #bootp => 'file',
 445 ##tftp => 'dir',
 446 ##smb => 'dir',
 447 #kernel => 'file',
 448 #append => 'string',
 449 #initrd => 'file',
 450 ##soundhw => 'string',
 451
 452 while (my ($k, $v) = each %$confdesc) {
 453     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 454 }
 455
 456 my $MAX_IDE_DISKS = 4;
 457 my $MAX_SCSI_DISKS = 14;
 458 my $MAX_VIRTIO_DISKS = 16;
 459 my $MAX_SATA_DISKS = 6;
 460 my $MAX_USB_DEVICES = 5;
 461 my $MAX_NETS = 32;
 462 my $MAX_UNUSED_DISKS = 8;
 463 my $MAX_HOSTPCI_DEVICES = 4;
 464 my $MAX_SERIAL_PORTS = 4;
 465 my $MAX_PARALLEL_PORTS = 3;
 466 my $MAX_NUMA = 8;
 467 my $MAX_MEM = 4194304;
 468 my $STATICMEM = 1024;
 469
 470 my $numa_fmt = {
 471     cpus => {
 472         type => "string",
 473         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 474         description => "CPUs accessing this numa node.",
 475         format_description => "id[-id];...",
 476     },
 477     memory => {
 478         type => "number",
 479         description => "Amount of memory this numa node provides.",
 480         format_description => "mb",
 481         optional => 1,
 482     },
 483     hostnodes => {
 484         type => "string",
 485         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 486         description => "host numa nodes to use",
 487         format_description => "id[-id];...",
 488         optional => 1,
 489     },
 490     policy => {
 491         type => 'string',
 492         enum => [qw(preferred bind interleave)],
 493         format_description => 'preferred|bind|interleave',
 494         description => "numa allocation policy.",
 495         optional => 1,
 496     },
 497 };
 498 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 499 my $numadesc = {
 500     optional => 1,
 501     type => 'string', format => $numa_fmt,
 502     description => "numa topology",
 503 };
 504 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 505
 506 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 507     $confdesc->{"numa$i"} = $numadesc;
 508 }
 509
 510 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 511                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 512                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 513 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 514
 515 my $netdesc = {
 516     optional => 1,
 517     type => 'string', format => 'pve-qm-net',
 518     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 519     description => <<EODESCR,
 520 Specify network devices.
 521
 522 MODEL is one of: $nic_model_list_txt
 523
 524 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 525 automatically generated if not specified.
 526
 527 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 528
 529 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 530
 531 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 532
 533 10.0.2.2   Gateway
 534 10.0.2.3   DNS Server
 535 10.0.2.4   SMB Server
 536
 537 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 538
 539 EODESCR
 540 };
 541 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 542
 543 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 544     $confdesc->{"net$i"} = $netdesc;
 545 }
 546
 547 PVE::JSONSchema::register_format('pve-volume-id-or-none', \&verify_volume_id_or_none);
 548 sub verify_volume_id_or_none {
 549     my ($volid, $noerr) = @_;
 550
 551     return $volid if $volid eq 'none';
 552     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 553     if ($@) {
 554         return undef if $noerr;
 555         die $@;
 556     }
 557     return $volid;
 558 }
 559
 560 my $drivename_hash;
 561
 562 my %drivedesc_base = (
 563     volume => { alias => 'file' },
 564     file => {
 565         type => 'string',
 566         format => 'pve-volume-id-or-none',
 567         default_key => 1,
 568         format_description => 'volume',
 569         description => "The drive's backing volume.",
 570     },
 571     media => {
 572         type => 'string',
 573         format_description => 'cdrom|disk',
 574         enum => [qw(cdrom disk)],
 575         description => "The drive's media type.",
 576         default => 'disk',
 577         optional => 1
 578     },
 579     cyls => {
 580         type => 'integer',
 581         format_description => 'count',
 582         description => "Force the drive's physical geometry to have a specific cylinder count.",
 583         optional => 1
 584     },
 585     heads => {
 586         type => 'integer',
 587         format_description => 'count',
 588         description => "Force the drive's physical geometry to have a specific head count.",
 589         optional => 1
 590     },
 591     secs => {
 592         type => 'integer',
 593         format_description => 'count',
 594         description => "Force the drive's physical geometry to have a specific sector count.",
 595         optional => 1
 596     },
 597     trans => {
 598         type => 'string',
 599         format_description => 'none|lba|auto',
 600         enum => [qw(none lba auto)],
 601         description => "Force disk geometry bios translation mode.",
 602         optional => 1,
 603     },
 604     snapshot => {
 605         type => 'boolean',
 606         format_description => 'on|off',
 607         description => "Whether the drive should be included when making snapshots.",
 608         optional => 1,
 609     },
 610     cache => {
 611         type => 'string',
 612         format_description => 'none|writethrough|writeback|unsafe|directsync',
 613         enum => [qw(none writethrough writeback unsafe directsync)],
 614         description => "The drive's cache mode",
 615         optional => 1,
 616     },
 617     format => {
 618         type => 'string',
 619         format_description => 'drive format',
 620         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 621         description => "The drive's backing file's data format.",
 622         optional => 1,
 623     },
 624     size => {
 625         type => 'string',
 626         format => 'disk-size',
 627         description => "Disk size. This is purely informational and has no effect.",
 628         optional => 1,
 629     },
 630     backup => {
 631         type => 'boolean',
 632         format_description => 'on|off',
 633         description => "Whether the drive should be included when making backups.",
 634         optional => 1,
 635     },
 636     werror => {
 637         type => 'string',
 638         format_description => 'enospc|ignore|report|stop',
 639         enum => [qw(enospc ignore report stop)],
 640         description => 'Write error action.',
 641         optional => 1,
 642     },
 643     aio => {
 644         type => 'string',
 645         format_description => 'native|threads',
 646         enum => [qw(native threads)],
 647         description => 'AIO type to use.',
 648         optional => 1,
 649     },
 650     discard => {
 651         type => 'string',
 652         format_description => 'ignore|on',
 653         enum => [qw(ignore on)],
 654         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 655         optional => 1,
 656     },
 657     detect_zeroes => {
 658         type => 'boolean',
 659         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 660         optional => 1,
 661     },
 662     serial => {
 663         type => 'string',
 664         format => 'urlencoded',
 665         format_description => 'serial',
 666         maxLength => 20*3, # *3 since it's %xx url enoded
 667         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 668         optional => 1,
 669     }
 670 );
 671
 672 my %rerror_fmt = (
 673     rerror => {
 674         type => 'string',
 675         format_description => 'ignore|report|stop',
 676         enum => [qw(ignore report stop)],
 677         description => 'Read error action.',
 678         optional => 1,
 679     },
 680 );
 681
 682 my %iothread_fmt = ( iothread => {
 683         type => 'boolean',
 684         format_description => 'off|on',
 685         description => "Whether to use iothreads for this drive",
 686         optional => 1,
 687 });
 688
 689 my %model_fmt = (
 690     model => {
 691         type => 'string',
 692         format => 'urlencoded',
 693         format_description => 'model',
 694         maxLength => 40*3, # *3 since it's %xx url enoded
 695         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 696         optional => 1,
 697     },
 698 );
 699
 700 my %queues_fmt = (
 701     queues => {
 702         type => 'integer',
 703         format_description => 'nbqueues',
 704         description => "Number of queues.",
 705         minimum => 2,
 706         optional => 1
 707     }
 708 );
 709
 710 my $add_throttle_desc = sub {
 711     my ($key, $type, $what, $size, $longsize) = @_;
 712     $drivedesc_base{$key} = {
 713         type => $type,
 714         format_description => $size,
 715         description => "Maximum $what speed in $longsize per second.",
 716         optional => 1,
 717     };
 718 };
 719 # throughput: (leaky bucket)
 720 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 721 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 722 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 723 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 724 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 725 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 726 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 727 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 728 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 729
 730 # pools: (pool of IO before throttling starts taking effect)
 731 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 732 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 733 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 734 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 735 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 736 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 737
 738 my $ide_fmt = {
 739     %drivedesc_base,
 740     %rerror_fmt,
 741     %model_fmt,
 742 };
 743
 744 my $idedesc = {
 745     optional => 1,
 746     type => 'string', format => $ide_fmt,
 747     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 748 };
 749 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 750
 751 my $scsi_fmt = {
 752     %drivedesc_base,
 753     %iothread_fmt,
 754     %queues_fmt,
 755 };
 756 my $scsidesc = {
 757     optional => 1,
 758     type => 'string', format => $scsi_fmt,
 759     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 760 };
 761 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 762
 763 my $sata_fmt = {
 764     %drivedesc_base,
 765     %rerror_fmt,
 766 };
 767 my $satadesc = {
 768     optional => 1,
 769     type => 'string', format => $sata_fmt,
 770     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 771 };
 772 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 773
 774 my $virtio_fmt = {
 775     %drivedesc_base,
 776     %iothread_fmt,
 777     %rerror_fmt,
 778 };
 779 my $virtiodesc = {
 780     optional => 1,
 781     type => 'string', format => $virtio_fmt,
 782     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 783 };
 784 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 785
 786 my $alldrive_fmt = {
 787     %drivedesc_base,
 788     %rerror_fmt,
 789     %iothread_fmt,
 790     %model_fmt,
 791     %queues_fmt,
 792 };
 793
 794 my $usb_fmt = {
 795     host => {
 796         default_key => 1,
 797         type => 'string', format => 'pve-qm-usb-device',
 798         format_description => 'HOSTUSBDEVICE|spice',
 799         description => 'The Host USB device or port or the value spice',
 800     },
 801     usb3 => {
 802         optional => 1,
 803         type => 'boolean',
 804         format_description => 'yes|no',
 805         description => 'Specifies whether if given host option is a USB3 device or port',
 806     },
 807 };
 808
 809 my $usbdesc = {
 810     optional => 1,
 811     type => 'string', format => $usb_fmt,
 812     description => <<EODESCR,
 813 Configure an USB device (n is 0 to 4). This can be used to
 814 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 815
 816 'bus-port(.port)*' (decimal numbers) or
 817 'vendor_id:product_id' (hexadeciaml numbers) or
 818 'spice'
 819
 820 You can use the 'lsusb -t' command to list existing usb devices.
 821
 822 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 823
 824 The value 'spice' can be used to add a usb redirection devices for spice.
 825
 826 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 827
 828 EODESCR
 829 };
 830 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 831
 832 my $hostpcidesc = {
 833         optional => 1,
 834         type => 'string', format => 'pve-qm-hostpci',
 835         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 836         description => <<EODESCR,
 837 Map host pci devices. HOSTPCIDEVICE syntax is:
 838
 839 'bus:dev.func' (hexadecimal numbers)
 840
 841 You can us the 'lspci' command to list existing pci devices.
 842
 843 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 844
 845 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 846
 847 Experimental: user reported problems with this option.
 848 EODESCR
 849 };
 850 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 851
 852 my $serialdesc = {
 853         optional => 1,
 854         type => 'string',
 855         pattern => '(/dev/.+|socket)',
 856         description =>  <<EODESCR,
 857 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 858
 859 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 860
 861 Experimental: user reported problems with this option.
 862 EODESCR
 863 };
 864
 865 my $paralleldesc= {
 866         optional => 1,
 867         type => 'string',
 868         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 869         description =>  <<EODESCR,
 870 Map host parallel devices (n is 0 to 2).
 871
 872 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 873
 874 Experimental: user reported problems with this option.
 875 EODESCR
 876 };
 877
 878 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 879     $confdesc->{"parallel$i"} = $paralleldesc;
 880 }
 881
 882 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 883     $confdesc->{"serial$i"} = $serialdesc;
 884 }
 885
 886 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 887     $confdesc->{"hostpci$i"} = $hostpcidesc;
 888 }
 889
 890 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 891     $drivename_hash->{"ide$i"} = 1;
 892     $confdesc->{"ide$i"} = $idedesc;
 893 }
 894
 895 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 896     $drivename_hash->{"sata$i"} = 1;
 897     $confdesc->{"sata$i"} = $satadesc;
 898 }
 899
 900 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 901     $drivename_hash->{"scsi$i"} = 1;
 902     $confdesc->{"scsi$i"} = $scsidesc ;
 903 }
 904
 905 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 906     $drivename_hash->{"virtio$i"} = 1;
 907     $confdesc->{"virtio$i"} = $virtiodesc;
 908 }
 909
 910 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 911     $confdesc->{"usb$i"} = $usbdesc;
 912 }
 913
 914 my $unuseddesc = {
 915     optional => 1,
 916     type => 'string', format => 'pve-volume-id',
 917     description => "Reference to unused volumes.",
 918 };
 919
 920 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 921     $confdesc->{"unused$i"} = $unuseddesc;
 922 }
 923
 924 my $kvm_api_version = 0;
 925
 926 sub kvm_version {
 927
 928     return $kvm_api_version if $kvm_api_version;
 929
 930     my $fh = IO::File->new("</dev/kvm") ||
 931         return 0;
 932
 933     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 934         $kvm_api_version = $v;
 935     }
 936
 937     $fh->close();
 938
 939     return  $kvm_api_version;
 940 }
 941
 942 my $kvm_user_version;
 943
 944 sub kvm_user_version {
 945
 946     return $kvm_user_version if $kvm_user_version;
 947
 948     $kvm_user_version = 'unknown';
 949
 950     my $code = sub {
 951         my $line = shift;
 952         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 953             $kvm_user_version = $2;
 954         }
 955     };
 956
 957     eval { run_command("kvm -version", outfunc => $code); };
 958     warn $@ if $@;
 959
 960     return $kvm_user_version;
 961
 962 }
 963
 964 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 965
 966 sub valid_drive_names {
 967     # order is important - used to autoselect boot disk
 968     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 969             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 970             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 971             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 972 }
 973
 974 sub is_valid_drivename {
 975     my $dev = shift;
 976
 977     return defined($drivename_hash->{$dev});
 978 }
 979
 980 sub option_exists {
 981     my $key = shift;
 982     return defined($confdesc->{$key});
 983 }
 984
 985 sub nic_models {
 986     return $nic_model_list;
 987 }
 988
 989 sub os_list_description {
 990
 991     return {
 992         other => 'Other',
 993         wxp => 'Windows XP',
 994         w2k => 'Windows 2000',
 995         w2k3 =>, 'Windows 2003',
 996         w2k8 => 'Windows 2008',
 997         wvista => 'Windows Vista',
 998         win7 => 'Windows 7',
 999         win8 => 'Windows 8/2012',
1000         l24 => 'Linux 2.4',
1001         l26 => 'Linux 2.6',
1002     };
1003 }
1004
1005 my $cdrom_path;
1006
1007 sub get_cdrom_path {
1008
1009     return  $cdrom_path if $cdrom_path;
1010
1011     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1012     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1013     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1014 }
1015
1016 sub get_iso_path {
1017     my ($storecfg, $vmid, $cdrom) = @_;
1018
1019     if ($cdrom eq 'cdrom') {
1020         return get_cdrom_path();
1021     } elsif ($cdrom eq 'none') {
1022         return '';
1023     } elsif ($cdrom =~ m|^/|) {
1024         return $cdrom;
1025     } else {
1026         return PVE::Storage::path($storecfg, $cdrom);
1027     }
1028 }
1029
1030 # try to convert old style file names to volume IDs
1031 sub filename_to_volume_id {
1032     my ($vmid, $file, $media) = @_;
1033
1034     if (!($file eq 'none' || $file eq 'cdrom' ||
1035           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1036
1037         return undef if $file =~ m|/|;
1038
1039         if ($media && $media eq 'cdrom') {
1040             $file = "local:iso/$file";
1041         } else {
1042             $file = "local:$vmid/$file";
1043         }
1044     }
1045
1046     return $file;
1047 }
1048
1049 sub verify_media_type {
1050     my ($opt, $vtype, $media) = @_;
1051
1052     return if !$media;
1053
1054     my $etype;
1055     if ($media eq 'disk') {
1056         $etype = 'images';
1057     } elsif ($media eq 'cdrom') {
1058         $etype = 'iso';
1059     } else {
1060         die "internal error";
1061     }
1062
1063     return if ($vtype eq $etype);
1064
1065     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1066 }
1067
1068 sub cleanup_drive_path {
1069     my ($opt, $storecfg, $drive) = @_;
1070
1071     # try to convert filesystem paths to volume IDs
1072
1073     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1074         ($drive->{file} !~ m|^/dev/.+|) &&
1075         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1076         ($drive->{file} !~ m/^\d+$/)) {
1077         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1078         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1079         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1080         verify_media_type($opt, $vtype, $drive->{media});
1081         $drive->{file} = $volid;
1082     }
1083
1084     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1085 }
1086
1087 sub parse_hotplug_features {
1088     my ($data) = @_;
1089
1090     my $res = {};
1091
1092     return $res if $data eq '0';
1093
1094     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1095
1096     foreach my $feature (PVE::Tools::split_list($data)) {
1097         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1098             $res->{$1} = 1;
1099         } else {
1100             warn "ignoring unknown hotplug feature '$feature'\n";
1101         }
1102     }
1103     return $res;
1104 }
1105
1106 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1107 sub pve_verify_hotplug_features {
1108     my ($value, $noerr) = @_;
1109
1110     return $value if parse_hotplug_features($value);
1111
1112     return undef if $noerr;
1113
1114     die "unable to parse hotplug option\n";
1115 }
1116
1117 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1118 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1119 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1120 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1121 #        [,iothread=on][,serial=serial][,model=model]
1122
1123 sub parse_drive {
1124     my ($key, $data) = @_;
1125
1126     my ($interface, $index);
1127
1128     if ($key =~ m/^([^\d]+)(\d+)$/) {
1129         $interface = $1;
1130         $index = $2;
1131     } else {
1132         return undef;
1133     }
1134
1135     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1136                                      : $confdesc->{$key}->{format};
1137     if (!$desc) {
1138         warn "invalid drive key: $key\n";
1139         return undef;
1140     }
1141     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1142     return undef if !$res;
1143     $res->{interface} = $interface;
1144     $res->{index} = $index;
1145
1146     my $error = 0;
1147     foreach my $opt (qw(bps bps_rd bps_wr)) {
1148         if (my $bps = defined(delete $res->{$opt})) {
1149             if (defined($res->{"m$opt"})) {
1150                 warn "both $opt and m$opt specified\n";
1151                 ++$error;
1152                 next;
1153             }
1154             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1155         }
1156     }
1157     return undef if $error;
1158
1159     return undef if $res->{mbps_rd} && $res->{mbps};
1160     return undef if $res->{mbps_wr} && $res->{mbps};
1161     return undef if $res->{iops_rd} && $res->{iops};
1162     return undef if $res->{iops_wr} && $res->{iops};
1163
1164     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1165         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1166         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1167         return undef if $res->{interface} eq 'virtio';
1168     }
1169
1170     if (my $size = $res->{size}) {
1171         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1172     }
1173
1174     return $res;
1175 }
1176
1177 sub print_drive {
1178     my ($vmid, $drive) = @_;
1179     my $data = { %$drive };
1180     delete $data->{$_} for qw(index interface);
1181     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1182 }
1183
1184 sub scsi_inquiry {
1185     my($fh, $noerr) = @_;
1186
1187     my $SG_IO = 0x2285;
1188     my $SG_GET_VERSION_NUM = 0x2282;
1189
1190     my $versionbuf = "\x00" x 8;
1191     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1192     if (!$ret) {
1193         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1194         return undef;
1195     }
1196     my $version = unpack("I", $versionbuf);
1197     if ($version < 30000) {
1198         die "scsi generic interface too old\n"  if !$noerr;
1199         return undef;
1200     }
1201
1202     my $buf = "\x00" x 36;
1203     my $sensebuf = "\x00" x 8;
1204     my $cmd = pack("C x3 C x1", 0x12, 36);
1205
1206     # see /usr/include/scsi/sg.h
1207     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1208
1209     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1210                       length($sensebuf), 0, length($buf), $buf,
1211                       $cmd, $sensebuf, 6000);
1212
1213     $ret = ioctl($fh, $SG_IO, $packet);
1214     if (!$ret) {
1215         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1216         return undef;
1217     }
1218
1219     my @res = unpack($sg_io_hdr_t, $packet);
1220     if ($res[17] || $res[18]) {
1221         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1222         return undef;
1223     }
1224
1225     my $res = {};
1226     (my $byte0, my $byte1, $res->{vendor},
1227      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1228
1229     $res->{removable} = $byte1 & 128 ? 1 : 0;
1230     $res->{type} = $byte0 & 31;
1231
1232     return $res;
1233 }
1234
1235 sub path_is_scsi {
1236     my ($path) = @_;
1237
1238     my $fh = IO::File->new("+<$path") || return undef;
1239     my $res = scsi_inquiry($fh, 1);
1240     close($fh);
1241
1242     return $res;
1243 }
1244
1245 sub machine_type_is_q35 {
1246     my ($conf) = @_;
1247
1248     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1249 }
1250
1251 sub print_tabletdevice_full {
1252     my ($conf) = @_;
1253
1254     my $q35 = machine_type_is_q35($conf);
1255
1256     # we use uhci for old VMs because tablet driver was buggy in older qemu
1257     my $usbbus = $q35 ? "ehci" : "uhci";
1258
1259     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1260 }
1261
1262 sub print_drivedevice_full {
1263     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1264
1265     my $device = '';
1266     my $maxdev = 0;
1267
1268     if ($drive->{interface} eq 'virtio') {
1269         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1270         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1271         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1272     } elsif ($drive->{interface} eq 'scsi') {
1273
1274         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1275         my $unit = $drive->{index} % $maxdev;
1276         my $devicetype = 'hd';
1277         my $path = '';
1278         if (drive_is_cdrom($drive)) {
1279             $devicetype = 'cd';
1280         } else {
1281             if ($drive->{file} =~ m|^/|) {
1282                 $path = $drive->{file};
1283                 if (my $info = path_is_scsi($path)) {
1284                     if ($info->{type} == 0) {
1285                         $devicetype = 'block';
1286                     } elsif ($info->{type} == 1) { # tape
1287                         $devicetype = 'generic';
1288                     }
1289                 }
1290             } else {
1291                  $path = PVE::Storage::path($storecfg, $drive->{file});
1292             }
1293
1294             if($path =~ m/^iscsi\:\/\//){
1295                 $devicetype = 'generic';
1296             }
1297         }
1298
1299         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1300             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1301         } else {
1302             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1303         }
1304
1305     } elsif ($drive->{interface} eq 'ide'){
1306         $maxdev = 2;
1307         my $controller = int($drive->{index} / $maxdev);
1308         my $unit = $drive->{index} % $maxdev;
1309         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1310
1311         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1312         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1313             $model = URI::Escape::uri_unescape($model);
1314             $device .= ",model=$model";
1315         }
1316     } elsif ($drive->{interface} eq 'sata'){
1317         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1318         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1319         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1320     } elsif ($drive->{interface} eq 'usb') {
1321         die "implement me";
1322         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1323     } else {
1324         die "unsupported interface type";
1325     }
1326
1327     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1328
1329     return $device;
1330 }
1331
1332 sub get_initiator_name {
1333     my $initiator;
1334
1335     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1336     while (defined(my $line = <$fh>)) {
1337         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1338         $initiator = $1;
1339         last;
1340     }
1341     $fh->close();
1342
1343     return $initiator;
1344 }
1345
1346 sub print_drive_full {
1347     my ($storecfg, $vmid, $drive) = @_;
1348
1349     my $path;
1350     my $volid = $drive->{file};
1351     my $format;
1352
1353     if (drive_is_cdrom($drive)) {
1354         $path = get_iso_path($storecfg, $vmid, $volid);
1355     } else {
1356         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1357         if ($storeid) {
1358             $path = PVE::Storage::path($storecfg, $volid);
1359             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1360             $format = qemu_img_format($scfg, $volname);
1361         } else {
1362             $path = $volid;
1363             $format = "raw";
1364         }
1365    }
1366
1367     my $opts = '';
1368     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1369     foreach my $o (@qemu_drive_options) {
1370         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1371     }
1372     if (my $serial = $drive->{serial}) {
1373         $serial = URI::Escape::uri_unescape($serial);
1374         $opts .= ",serial=$serial";
1375     }
1376
1377     $opts .= ",format=$format" if $format && !$drive->{format};
1378
1379     foreach my $o (qw(bps bps_rd bps_wr)) {
1380         my $v = $drive->{"m$o"};
1381         $opts .= ",$o=" . int($v*1024*1024) if $v;
1382     }
1383
1384     my $cache_direct = 0;
1385
1386     if (my $cache = $drive->{cache}) {
1387         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1388     } elsif (!drive_is_cdrom($drive)) {
1389         $opts .= ",cache=none";
1390         $cache_direct = 1;
1391     }
1392
1393     # aio native works only with O_DIRECT
1394     if (!$drive->{aio}) {
1395         if($cache_direct) {
1396             $opts .= ",aio=native";
1397         } else {
1398             $opts .= ",aio=threads";
1399         }
1400     }
1401
1402     if (!drive_is_cdrom($drive)) {
1403         my $detectzeroes;
1404         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1405             $detectzeroes = 'off';
1406         } elsif ($drive->{discard}) {
1407             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1408         } else {
1409             # This used to be our default with discard not being specified:
1410             $detectzeroes = 'on';
1411         }
1412         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1413     }
1414
1415     my $pathinfo = $path ? "file=$path," : '';
1416
1417     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1418 }
1419
1420 sub print_netdevice_full {
1421     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1422
1423     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1424
1425     my $device = $net->{model};
1426     if ($net->{model} eq 'virtio') {
1427          $device = 'virtio-net-pci';
1428      };
1429
1430     my $pciaddr = print_pci_addr("$netid", $bridges);
1431     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1432     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1433         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1434         my $vectors = $net->{queues} * 2 + 2;
1435         $tmpstr .= ",vectors=$vectors,mq=on";
1436     }
1437     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1438
1439     if ($use_old_bios_files) {
1440         my $romfile;
1441         if ($device eq 'virtio-net-pci') {
1442             $romfile = 'pxe-virtio.rom';
1443         } elsif ($device eq 'e1000') {
1444             $romfile = 'pxe-e1000.rom';
1445         } elsif ($device eq 'ne2k') {
1446             $romfile = 'pxe-ne2k_pci.rom';
1447         } elsif ($device eq 'pcnet') {
1448             $romfile = 'pxe-pcnet.rom';
1449         } elsif ($device eq 'rtl8139') {
1450             $romfile = 'pxe-rtl8139.rom';
1451         }
1452         $tmpstr .= ",romfile=$romfile" if $romfile;
1453     }
1454
1455     return $tmpstr;
1456 }
1457
1458 sub print_netdev_full {
1459     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1460
1461     my $i = '';
1462     if ($netid =~ m/^net(\d+)$/) {
1463         $i = int($1);
1464     }
1465
1466     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1467
1468     my $ifname = "tap${vmid}i$i";
1469
1470     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1471     die "interface name '$ifname' is too long (max 15 character)\n"
1472         if length($ifname) >= 16;
1473
1474     my $vhostparam = '';
1475     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1476
1477     my $vmname = $conf->{name} || "vm$vmid";
1478
1479     my $netdev = "";
1480     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1481
1482     if ($net->{bridge}) {
1483         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1484     } else {
1485         $netdev = "type=user,id=$netid,hostname=$vmname";
1486     }
1487
1488     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1489
1490     return $netdev;
1491 }
1492
1493 sub drive_is_cdrom {
1494     my ($drive) = @_;
1495
1496     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1497
1498 }
1499
1500 sub parse_number_sets {
1501     my ($set) = @_;
1502     my $res = [];
1503     foreach my $part (split(/;/, $set)) {
1504         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1505             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1506             push @$res, [ $1, $2 ];
1507         } else {
1508             die "invalid range: $part\n";
1509         }
1510     }
1511     return $res;
1512 }
1513
1514 sub parse_numa {
1515     my ($data) = @_;
1516
1517     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1518     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1519     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1520     return $res;
1521 }
1522
1523 sub parse_hostpci {
1524     my ($value) = @_;
1525
1526     return undef if !$value;
1527
1528
1529     my @list = split(/,/, $value);
1530     my $found;
1531
1532     my $res = {};
1533     foreach my $kv (@list) {
1534
1535         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1536             $found = 1;
1537             if(defined($4)){
1538                 push @{$res->{pciid}}, { id => $2 , function => $4};
1539
1540             }else{
1541                 my $pcidevices = lspci($2);
1542                 $res->{pciid} = $pcidevices->{$2};
1543             }
1544         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1545             $res->{rombar} = $1;
1546         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1547             $res->{'x-vga'} = $1;
1548         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1549             $res->{pcie} = 1 if $1 == 1;
1550         } else {
1551             warn "unknown hostpci setting '$kv'\n";
1552         }
1553     }
1554
1555     return undef if !$found;
1556
1557     return $res;
1558 }
1559
1560 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1561 sub parse_net {
1562     my ($data) = @_;
1563
1564     my $res = {};
1565
1566     foreach my $kvp (split(/,/, $data)) {
1567
1568         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1569             my $model = lc($1);
1570             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1571             $res->{model} = $model;
1572             $res->{macaddr} = $mac;
1573         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1574             $res->{bridge} = $1;
1575         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1576             $res->{queues} = $1;
1577         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1578             $res->{rate} = $1;
1579         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1580             $res->{tag} = $1;
1581         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1582             $res->{trunks} = $1;
1583         } elsif ($kvp =~ m/^firewall=([01])$/) {
1584             $res->{firewall} = $1;
1585         } elsif ($kvp =~ m/^link_down=([01])$/) {
1586             $res->{link_down} = $1;
1587         } else {
1588             return undef;
1589         }
1590
1591     }
1592
1593     return undef if !$res->{model};
1594
1595     return $res;
1596 }
1597
1598 sub print_net {
1599     my $net = shift;
1600
1601     my $res = "$net->{model}";
1602     $res .= "=$net->{macaddr}" if $net->{macaddr};
1603     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1604     $res .= ",rate=$net->{rate}" if $net->{rate};
1605     $res .= ",tag=$net->{tag}" if $net->{tag};
1606     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1607     $res .= ",firewall=1" if $net->{firewall};
1608     $res .= ",link_down=1" if $net->{link_down};
1609     $res .= ",queues=$net->{queues}" if $net->{queues};
1610
1611     return $res;
1612 }
1613
1614 sub add_random_macs {
1615     my ($settings) = @_;
1616
1617     foreach my $opt (keys %$settings) {
1618         next if $opt !~ m/^net(\d+)$/;
1619         my $net = parse_net($settings->{$opt});
1620         next if !$net;
1621         $settings->{$opt} = print_net($net);
1622     }
1623 }
1624
1625 sub vm_is_volid_owner {
1626     my ($storecfg, $vmid, $volid) = @_;
1627
1628     if ($volid !~  m|^/|) {
1629         my ($path, $owner);
1630         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1631         if ($owner && ($owner == $vmid)) {
1632             return 1;
1633         }
1634     }
1635
1636     return undef;
1637 }
1638
1639 sub split_flagged_list {
1640     my $text = shift || '';
1641     $text =~ s/[,;]/ /g;
1642     $text =~ s/^\s+//;
1643     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1644 }
1645
1646 sub join_flagged_list {
1647     my ($how, $lst) = @_;
1648     join $how, map { $lst->{$_} . $_ } keys %$lst;
1649 }
1650
1651 sub vmconfig_delete_pending_option {
1652     my ($conf, $key, $force) = @_;
1653
1654     delete $conf->{pending}->{$key};
1655     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1656     $pending_delete_hash->{$key} = $force ? '!' : '';
1657     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1658 }
1659
1660 sub vmconfig_undelete_pending_option {
1661     my ($conf, $key) = @_;
1662
1663     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1664     delete $pending_delete_hash->{$key};
1665
1666     if (%$pending_delete_hash) {
1667         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1668     } else {
1669         delete $conf->{pending}->{delete};
1670     }
1671 }
1672
1673 sub vmconfig_register_unused_drive {
1674     my ($storecfg, $vmid, $conf, $drive) = @_;
1675
1676     if (!drive_is_cdrom($drive)) {
1677         my $volid = $drive->{file};
1678         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1679             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1680         }
1681     }
1682 }
1683
1684 sub vmconfig_cleanup_pending {
1685     my ($conf) = @_;
1686
1687     # remove pending changes when nothing changed
1688     my $changes;
1689     foreach my $opt (keys %{$conf->{pending}}) {
1690         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1691             $changes = 1;
1692             delete $conf->{pending}->{$opt};
1693         }
1694     }
1695
1696     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1697     my $pending_delete_hash = {};
1698     while (my ($opt, $force) = each %$current_delete_hash) {
1699         if (defined($conf->{$opt})) {
1700             $pending_delete_hash->{$opt} = $force;
1701         } else {
1702             $changes = 1;
1703         }
1704     }
1705
1706     if (%$pending_delete_hash) {
1707         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1708     } else {
1709         delete $conf->{pending}->{delete};
1710     }
1711
1712     return $changes;
1713 }
1714
1715 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1716 my $smbios1_fmt = {
1717     uuid => {
1718         type => 'string',
1719         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1720         format_description => 'UUID',
1721         optional => 1,
1722     },
1723     version => {
1724         type => 'string',
1725         pattern => '\S+',
1726         format_description => 'str',
1727         optional => 1,
1728     },
1729     serial => {
1730         type => 'string',
1731         pattern => '\S+',
1732         format_description => 'str',
1733         optional => 1,
1734     },
1735     manufacturer => {
1736         type => 'string',
1737         pattern => '\S+',
1738         format_description => 'name',
1739         optional => 1,
1740     },
1741     product => {
1742         type => 'string',
1743         pattern => '\S+',
1744         format_description => 'name',
1745         optional => 1,
1746     },
1747     sku => {
1748         type => 'string',
1749         pattern => '\S+',
1750         format_description => 'str',
1751         optional => 1,
1752     },
1753     family => {
1754         type => 'string',
1755         pattern => '\S+',
1756         format_description => 'str',
1757         optional => 1,
1758     },
1759 };
1760
1761 sub parse_smbios1 {
1762     my ($data) = @_;
1763
1764     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1765     warn $@ if $@;
1766     return $res;
1767 }
1768
1769 sub print_smbios1 {
1770     my ($smbios1) = @_;
1771     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1772 }
1773
1774 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1775
1776 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1777 sub verify_bootdisk {
1778     my ($value, $noerr) = @_;
1779
1780     return $value if is_valid_drivename($value);
1781
1782     return undef if $noerr;
1783
1784     die "invalid boot disk '$value'\n";
1785 }
1786
1787 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1788 sub verify_net {
1789     my ($value, $noerr) = @_;
1790
1791     return $value if parse_net($value);
1792
1793     return undef if $noerr;
1794
1795     die "unable to parse network options\n";
1796 }
1797
1798 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1799 sub verify_hostpci {
1800     my ($value, $noerr) = @_;
1801
1802     return $value if parse_hostpci($value);
1803
1804     return undef if $noerr;
1805
1806     die "unable to parse pci id\n";
1807 }
1808
1809 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1810 sub verify_watchdog {
1811     my ($value, $noerr) = @_;
1812
1813     return $value if parse_watchdog($value);
1814
1815     return undef if $noerr;
1816
1817     die "unable to parse watchdog options\n";
1818 }
1819
1820 sub parse_watchdog {
1821     my ($value) = @_;
1822
1823     return undef if !$value;
1824
1825     my $res = {};
1826
1827     foreach my $p (split(/,/, $value)) {
1828         next if $p =~ m/^\s*$/;
1829
1830         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1831             $res->{model} = $2;
1832         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1833             $res->{action} = $2;
1834         } else {
1835             return undef;
1836         }
1837     }
1838
1839     return $res;
1840 }
1841
1842 sub parse_usb_device {
1843     my ($value) = @_;
1844
1845     return undef if !$value;
1846
1847     my $res = {};
1848     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1849         $res->{vendorid} = $2;
1850         $res->{productid} = $4;
1851     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1852         $res->{hostbus} = $1;
1853         $res->{hostport} = $2;
1854     } elsif ($value =~ m/^spice$/i) {
1855         $res->{spice} = 1;
1856     } else {
1857         return undef;
1858     }
1859
1860     return $res;
1861 }
1862
1863 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1864 sub verify_usb_device {
1865     my ($value, $noerr) = @_;
1866
1867     return $value if parse_usb_device($value);
1868
1869     return undef if $noerr;
1870
1871     die "unable to parse usb device\n";
1872 }
1873
1874 # add JSON properties for create and set function
1875 sub json_config_properties {
1876     my $prop = shift;
1877
1878     foreach my $opt (keys %$confdesc) {
1879         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1880         $prop->{$opt} = $confdesc->{$opt};
1881     }
1882
1883     return $prop;
1884 }
1885
1886 sub check_type {
1887     my ($key, $value) = @_;
1888
1889     die "unknown setting '$key'\n" if !$confdesc->{$key};
1890
1891     my $type = $confdesc->{$key}->{type};
1892
1893     if (!defined($value)) {
1894         die "got undefined value\n";
1895     }
1896
1897     if ($value =~ m/[\n\r]/) {
1898         die "property contains a line feed\n";
1899     }
1900
1901     if ($type eq 'boolean') {
1902         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1903         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1904         die "type check ('boolean') failed - got '$value'\n";
1905     } elsif ($type eq 'integer') {
1906         return int($1) if $value =~ m/^(\d+)$/;
1907         die "type check ('integer') failed - got '$value'\n";
1908     } elsif ($type eq 'number') {
1909         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1910         die "type check ('number') failed - got '$value'\n";
1911     } elsif ($type eq 'string') {
1912         if (my $fmt = $confdesc->{$key}->{format}) {
1913             if ($fmt eq 'pve-qm-drive') {
1914                 # special case - we need to pass $key to parse_drive()
1915                 my $drive = parse_drive($key, $value);
1916                 return $value if $drive;
1917                 die "unable to parse drive options\n";
1918             }
1919             PVE::JSONSchema::check_format($fmt, $value);
1920             return $value;
1921         }
1922         $value =~ s/^\"(.*)\"$/$1/;
1923         return $value;
1924     } else {
1925         die "internal error"
1926     }
1927 }
1928
1929 sub check_iommu_support{
1930     #fixme : need to check IOMMU support
1931     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1932
1933     my $iommu=1;
1934     return $iommu;
1935
1936 }
1937
1938 sub touch_config {
1939     my ($vmid) = @_;
1940
1941     my $conf = PVE::QemuConfig->config_file($vmid);
1942     utime undef, undef, $conf;
1943 }
1944
1945 sub destroy_vm {
1946     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1947
1948     my $conffile = PVE::QemuConfig->config_file($vmid);
1949
1950     my $conf = PVE::QemuConfig->load_config($vmid);
1951
1952     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1953
1954     # only remove disks owned by this VM
1955     foreach_drive($conf, sub {
1956         my ($ds, $drive) = @_;
1957
1958         return if drive_is_cdrom($drive);
1959
1960         my $volid = $drive->{file};
1961
1962         return if !$volid || $volid =~ m|^/|;
1963
1964         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1965         return if !$path || !$owner || ($owner != $vmid);
1966
1967         PVE::Storage::vdisk_free($storecfg, $volid);
1968     });
1969
1970     if ($keep_empty_config) {
1971         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1972     } else {
1973         unlink $conffile;
1974     }
1975
1976     # also remove unused disk
1977     eval {
1978         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1979
1980         eval {
1981             PVE::Storage::foreach_volid($dl, sub {
1982                 my ($volid, $sid, $volname, $d) = @_;
1983                 PVE::Storage::vdisk_free($storecfg, $volid);
1984             });
1985         };
1986         warn $@ if $@;
1987
1988     };
1989     warn $@ if $@;
1990 }
1991
1992 sub parse_vm_config {
1993     my ($filename, $raw) = @_;
1994
1995     return undef if !defined($raw);
1996
1997     my $res = {
1998         digest => Digest::SHA::sha1_hex($raw),
1999         snapshots => {},
2000         pending => {},
2001     };
2002
2003     $filename =~ m|/qemu-server/(\d+)\.conf$|
2004         || die "got strange filename '$filename'";
2005
2006     my $vmid = $1;
2007
2008     my $conf = $res;
2009     my $descr;
2010     my $section = '';
2011
2012     my @lines = split(/\n/, $raw);
2013     foreach my $line (@lines) {
2014         next if $line =~ m/^\s*$/;
2015
2016         if ($line =~ m/^\[PENDING\]\s*$/i) {
2017             $section = 'pending';
2018             if (defined($descr)) {
2019                 $descr =~ s/\s+$//;
2020                 $conf->{description} = $descr;
2021             }
2022             $descr = undef;
2023             $conf = $res->{$section} = {};
2024             next;
2025
2026         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2027             $section = $1;
2028             if (defined($descr)) {
2029                 $descr =~ s/\s+$//;
2030                 $conf->{description} = $descr;
2031             }
2032             $descr = undef;
2033             $conf = $res->{snapshots}->{$section} = {};
2034             next;
2035         }
2036
2037         if ($line =~ m/^\#(.*)\s*$/) {
2038             $descr = '' if !defined($descr);
2039             $descr .= PVE::Tools::decode_text($1) . "\n";
2040             next;
2041         }
2042
2043         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2044             $descr = '' if !defined($descr);
2045             $descr .= PVE::Tools::decode_text($2);
2046         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2047             $conf->{snapstate} = $1;
2048         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2049             my $key = $1;
2050             my $value = $2;
2051             $conf->{$key} = $value;
2052         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2053             my $value = $1;
2054             if ($section eq 'pending') {
2055                 $conf->{delete} = $value; # we parse this later
2056             } else {
2057                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2058             }
2059         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2060             my $key = $1;
2061             my $value = $2;
2062             eval { $value = check_type($key, $value); };
2063             if ($@) {
2064                 warn "vm $vmid - unable to parse value of '$key' - $@";
2065             } else {
2066                 my $fmt = $confdesc->{$key}->{format};
2067                 if ($fmt && $fmt eq 'pve-qm-drive') {
2068                     my $v = parse_drive($key, $value);
2069                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2070                         $v->{file} = $volid;
2071                         $value = print_drive($vmid, $v);
2072                     } else {
2073                         warn "vm $vmid - unable to parse value of '$key'\n";
2074                         next;
2075                     }
2076                 }
2077
2078                 if ($key eq 'cdrom') {
2079                     $conf->{ide2} = $value;
2080                 } else {
2081                     $conf->{$key} = $value;
2082                 }
2083             }
2084         }
2085     }
2086
2087     if (defined($descr)) {
2088         $descr =~ s/\s+$//;
2089         $conf->{description} = $descr;
2090     }
2091     delete $res->{snapstate}; # just to be sure
2092
2093     return $res;
2094 }
2095
2096 sub write_vm_config {
2097     my ($filename, $conf) = @_;
2098
2099     delete $conf->{snapstate}; # just to be sure
2100
2101     if ($conf->{cdrom}) {
2102         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2103         $conf->{ide2} = $conf->{cdrom};
2104         delete $conf->{cdrom};
2105     }
2106
2107     # we do not use 'smp' any longer
2108     if ($conf->{sockets}) {
2109         delete $conf->{smp};
2110     } elsif ($conf->{smp}) {
2111         $conf->{sockets} = $conf->{smp};
2112         delete $conf->{cores};
2113         delete $conf->{smp};
2114     }
2115
2116     my $used_volids = {};
2117
2118     my $cleanup_config = sub {
2119         my ($cref, $pending, $snapname) = @_;
2120
2121         foreach my $key (keys %$cref) {
2122             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2123                 $key eq 'snapstate' || $key eq 'pending';
2124             my $value = $cref->{$key};
2125             if ($key eq 'delete') {
2126                 die "propertry 'delete' is only allowed in [PENDING]\n"
2127                     if !$pending;
2128                 # fixme: check syntax?
2129                 next;
2130             }
2131             eval { $value = check_type($key, $value); };
2132             die "unable to parse value of '$key' - $@" if $@;
2133
2134             $cref->{$key} = $value;
2135
2136             if (!$snapname && is_valid_drivename($key)) {
2137                 my $drive = parse_drive($key, $value);
2138                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2139             }
2140         }
2141     };
2142
2143     &$cleanup_config($conf);
2144
2145     &$cleanup_config($conf->{pending}, 1);
2146
2147     foreach my $snapname (keys %{$conf->{snapshots}}) {
2148         die "internal error" if $snapname eq 'pending';
2149         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2150     }
2151
2152     # remove 'unusedX' settings if we re-add a volume
2153     foreach my $key (keys %$conf) {
2154         my $value = $conf->{$key};
2155         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2156             delete $conf->{$key};
2157         }
2158     }
2159
2160     my $generate_raw_config = sub {
2161         my ($conf, $pending) = @_;
2162
2163         my $raw = '';
2164
2165         # add description as comment to top of file
2166         if (defined(my $descr = $conf->{description})) {
2167             if ($descr) {
2168                 foreach my $cl (split(/\n/, $descr)) {
2169                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2170                 }
2171             } else {
2172                 $raw .= "#\n" if $pending;
2173             }
2174         }
2175
2176         foreach my $key (sort keys %$conf) {
2177             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2178             $raw .= "$key: $conf->{$key}\n";
2179         }
2180         return $raw;
2181     };
2182
2183     my $raw = &$generate_raw_config($conf);
2184
2185     if (scalar(keys %{$conf->{pending}})){
2186         $raw .= "\n[PENDING]\n";
2187         $raw .= &$generate_raw_config($conf->{pending}, 1);
2188     }
2189
2190     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2191         $raw .= "\n[$snapname]\n";
2192         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2193     }
2194
2195     return $raw;
2196 }
2197
2198 sub load_defaults {
2199
2200     my $res = {};
2201
2202     # we use static defaults from our JSON schema configuration
2203     foreach my $key (keys %$confdesc) {
2204         if (defined(my $default = $confdesc->{$key}->{default})) {
2205             $res->{$key} = $default;
2206         }
2207     }
2208
2209     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2210     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2211
2212     return $res;
2213 }
2214
2215 sub config_list {
2216     my $vmlist = PVE::Cluster::get_vmlist();
2217     my $res = {};
2218     return $res if !$vmlist || !$vmlist->{ids};
2219     my $ids = $vmlist->{ids};
2220
2221     foreach my $vmid (keys %$ids) {
2222         my $d = $ids->{$vmid};
2223         next if !$d->{node} || $d->{node} ne $nodename;
2224         next if !$d->{type} || $d->{type} ne 'qemu';
2225         $res->{$vmid}->{exists} = 1;
2226     }
2227     return $res;
2228 }
2229
2230 # test if VM uses local resources (to prevent migration)
2231 sub check_local_resources {
2232     my ($conf, $noerr) = @_;
2233
2234     my $loc_res = 0;
2235
2236     $loc_res = 1 if $conf->{hostusb}; # old syntax
2237     $loc_res = 1 if $conf->{hostpci}; # old syntax
2238
2239     foreach my $k (keys %$conf) {
2240         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2241         # sockets are safe: they will recreated be on the target side post-migrate
2242         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2243         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2244     }
2245
2246     die "VM uses local resources\n" if $loc_res && !$noerr;
2247
2248     return $loc_res;
2249 }
2250
2251 # check if used storages are available on all nodes (use by migrate)
2252 sub check_storage_availability {
2253     my ($storecfg, $conf, $node) = @_;
2254
2255     foreach_drive($conf, sub {
2256         my ($ds, $drive) = @_;
2257
2258         my $volid = $drive->{file};
2259         return if !$volid;
2260
2261         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2262         return if !$sid;
2263
2264         # check if storage is available on both nodes
2265         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2266         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2267    });
2268 }
2269
2270 # list nodes where all VM images are available (used by has_feature API)
2271 sub shared_nodes {
2272     my ($conf, $storecfg) = @_;
2273
2274     my $nodelist = PVE::Cluster::get_nodelist();
2275     my $nodehash = { map { $_ => 1 } @$nodelist };
2276     my $nodename = PVE::INotify::nodename();
2277
2278     foreach_drive($conf, sub {
2279         my ($ds, $drive) = @_;
2280
2281         my $volid = $drive->{file};
2282         return if !$volid;
2283
2284         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2285         if ($storeid) {
2286             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2287             if ($scfg->{disable}) {
2288                 $nodehash = {};
2289             } elsif (my $avail = $scfg->{nodes}) {
2290                 foreach my $node (keys %$nodehash) {
2291                     delete $nodehash->{$node} if !$avail->{$node};
2292                 }
2293             } elsif (!$scfg->{shared}) {
2294                 foreach my $node (keys %$nodehash) {
2295                     delete $nodehash->{$node} if $node ne $nodename
2296                 }
2297             }
2298         }
2299     });
2300
2301     return $nodehash
2302 }
2303
2304 sub check_cmdline {
2305     my ($pidfile, $pid) = @_;
2306
2307     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2308     if (defined($fh)) {
2309         my $line = <$fh>;
2310         $fh->close;
2311         return undef if !$line;
2312         my @param = split(/\0/, $line);
2313
2314         my $cmd = $param[0];
2315         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2316
2317         for (my $i = 0; $i < scalar (@param); $i++) {
2318             my $p = $param[$i];
2319             next if !$p;
2320             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2321                 my $p = $param[$i+1];
2322                 return 1 if $p && ($p eq $pidfile);
2323                 return undef;
2324             }
2325         }
2326     }
2327     return undef;
2328 }
2329
2330 sub check_running {
2331     my ($vmid, $nocheck, $node) = @_;
2332
2333     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2334
2335     die "unable to find configuration file for VM $vmid - no such machine\n"
2336         if !$nocheck && ! -f $filename;
2337
2338     my $pidfile = pidfile_name($vmid);
2339
2340     if (my $fd = IO::File->new("<$pidfile")) {
2341         my $st = stat($fd);
2342         my $line = <$fd>;
2343         close($fd);
2344
2345         my $mtime = $st->mtime;
2346         if ($mtime > time()) {
2347             warn "file '$filename' modified in future\n";
2348         }
2349
2350         if ($line =~ m/^(\d+)$/) {
2351             my $pid = $1;
2352             if (check_cmdline($pidfile, $pid)) {
2353                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2354                     return $pid;
2355                 }
2356             }
2357         }
2358     }
2359
2360     return undef;
2361 }
2362
2363 sub vzlist {
2364
2365     my $vzlist = config_list();
2366
2367     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2368
2369     while (defined(my $de = $fd->read)) {
2370         next if $de !~ m/^(\d+)\.pid$/;
2371         my $vmid = $1;
2372         next if !defined($vzlist->{$vmid});
2373         if (my $pid = check_running($vmid)) {
2374             $vzlist->{$vmid}->{pid} = $pid;
2375         }
2376     }
2377
2378     return $vzlist;
2379 }
2380
2381 sub disksize {
2382     my ($storecfg, $conf) = @_;
2383
2384     my $bootdisk = $conf->{bootdisk};
2385     return undef if !$bootdisk;
2386     return undef if !is_valid_drivename($bootdisk);
2387
2388     return undef if !$conf->{$bootdisk};
2389
2390     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2391     return undef if !defined($drive);
2392
2393     return undef if drive_is_cdrom($drive);
2394
2395     my $volid = $drive->{file};
2396     return undef if !$volid;
2397
2398     return $drive->{size};
2399 }
2400
2401 my $last_proc_pid_stat;
2402
2403 # get VM status information
2404 # This must be fast and should not block ($full == false)
2405 # We only query KVM using QMP if $full == true (this can be slow)
2406 sub vmstatus {
2407     my ($opt_vmid, $full) = @_;
2408
2409     my $res = {};
2410
2411     my $storecfg = PVE::Storage::config();
2412
2413     my $list = vzlist();
2414     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2415
2416     my $cpucount = $cpuinfo->{cpus} || 1;
2417
2418     foreach my $vmid (keys %$list) {
2419         next if $opt_vmid && ($vmid ne $opt_vmid);
2420
2421         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2422         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2423
2424         my $d = {};
2425         $d->{pid} = $list->{$vmid}->{pid};
2426
2427         # fixme: better status?
2428         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2429
2430         my $size = disksize($storecfg, $conf);
2431         if (defined($size)) {
2432             $d->{disk} = 0; # no info available
2433             $d->{maxdisk} = $size;
2434         } else {
2435             $d->{disk} = 0;
2436             $d->{maxdisk} = 0;
2437         }
2438
2439         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2440         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2441         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2442
2443         $d->{name} = $conf->{name} || "VM $vmid";
2444         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2445
2446         if ($conf->{balloon}) {
2447             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2448             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2449         }
2450
2451         $d->{uptime} = 0;
2452         $d->{cpu} = 0;
2453         $d->{mem} = 0;
2454
2455         $d->{netout} = 0;
2456         $d->{netin} = 0;
2457
2458         $d->{diskread} = 0;
2459         $d->{diskwrite} = 0;
2460
2461         $d->{template} = PVE::QemuConfig->is_template($conf);
2462
2463         $res->{$vmid} = $d;
2464     }
2465
2466     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2467     foreach my $dev (keys %$netdev) {
2468         next if $dev !~ m/^tap([1-9]\d*)i/;
2469         my $vmid = $1;
2470         my $d = $res->{$vmid};
2471         next if !$d;
2472
2473         $d->{netout} += $netdev->{$dev}->{receive};
2474         $d->{netin} += $netdev->{$dev}->{transmit};
2475
2476         if ($full) {
2477             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2478             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2479         }
2480
2481     }
2482
2483     my $ctime = gettimeofday;
2484
2485     foreach my $vmid (keys %$list) {
2486
2487         my $d = $res->{$vmid};
2488         my $pid = $d->{pid};
2489         next if !$pid;
2490
2491         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2492         next if !$pstat; # not running
2493
2494         my $used = $pstat->{utime} + $pstat->{stime};
2495
2496         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2497
2498         if ($pstat->{vsize}) {
2499             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2500         }
2501
2502         my $old = $last_proc_pid_stat->{$pid};
2503         if (!$old) {
2504             $last_proc_pid_stat->{$pid} = {
2505                 time => $ctime,
2506                 used => $used,
2507                 cpu => 0,
2508             };
2509             next;
2510         }
2511
2512         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2513
2514         if ($dtime > 1000) {
2515             my $dutime = $used -  $old->{used};
2516
2517             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2518             $last_proc_pid_stat->{$pid} = {
2519                 time => $ctime,
2520                 used => $used,
2521                 cpu => $d->{cpu},
2522             };
2523         } else {
2524             $d->{cpu} = $old->{cpu};
2525         }
2526     }
2527
2528     return $res if !$full;
2529
2530     my $qmpclient = PVE::QMPClient->new();
2531
2532     my $ballooncb = sub {
2533         my ($vmid, $resp) = @_;
2534
2535         my $info = $resp->{'return'};
2536         return if !$info->{max_mem};
2537
2538         my $d = $res->{$vmid};
2539
2540         # use memory assigned to VM
2541         $d->{maxmem} = $info->{max_mem};
2542         $d->{balloon} = $info->{actual};
2543
2544         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2545             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2546             $d->{freemem} = $info->{free_mem};
2547         }
2548
2549         $d->{ballooninfo} = $info;
2550     };
2551
2552     my $blockstatscb = sub {
2553         my ($vmid, $resp) = @_;
2554         my $data = $resp->{'return'} || [];
2555         my $totalrdbytes = 0;
2556         my $totalwrbytes = 0;
2557
2558         for my $blockstat (@$data) {
2559             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2560             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2561
2562             $blockstat->{device} =~ s/drive-//;
2563             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2564         }
2565         $res->{$vmid}->{diskread} = $totalrdbytes;
2566         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2567     };
2568
2569     my $statuscb = sub {
2570         my ($vmid, $resp) = @_;
2571
2572         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2573         # this fails if ballon driver is not loaded, so this must be
2574         # the last commnand (following command are aborted if this fails).
2575         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2576
2577         my $status = 'unknown';
2578         if (!defined($status = $resp->{'return'}->{status})) {
2579             warn "unable to get VM status\n";
2580             return;
2581         }
2582
2583         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2584     };
2585
2586     foreach my $vmid (keys %$list) {
2587         next if $opt_vmid && ($vmid ne $opt_vmid);
2588         next if !$res->{$vmid}->{pid}; # not running
2589         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2590     }
2591
2592     $qmpclient->queue_execute(undef, 1);
2593
2594     foreach my $vmid (keys %$list) {
2595         next if $opt_vmid && ($vmid ne $opt_vmid);
2596         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2597     }
2598
2599     return $res;
2600 }
2601
2602 sub foreach_dimm {
2603     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2604
2605     my $dimm_id = 0;
2606     my $current_size = 1024;
2607     my $dimm_size = 512;
2608     return if $current_size == $memory;
2609
2610     for (my $j = 0; $j < 8; $j++) {
2611         for (my $i = 0; $i < 32; $i++) {
2612             my $name = "dimm${dimm_id}";
2613             $dimm_id++;
2614             my $numanode = $i % $sockets;
2615             $current_size += $dimm_size;
2616             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2617             return  $current_size if $current_size >= $memory;
2618         }
2619         $dimm_size *= 2;
2620     }
2621 }
2622
2623 sub foreach_reverse_dimm {
2624     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2625
2626     my $dimm_id = 253;
2627     my $current_size = 4177920;
2628     my $dimm_size = 65536;
2629     return if $current_size == $memory;
2630
2631     for (my $j = 0; $j < 8; $j++) {
2632         for (my $i = 0; $i < 32; $i++) {
2633             my $name = "dimm${dimm_id}";
2634             $dimm_id--;
2635             my $numanode = $i % $sockets;
2636             $current_size -= $dimm_size;
2637             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2638             return  $current_size if $current_size <= $memory;
2639         }
2640         $dimm_size /= 2;
2641     }
2642 }
2643
2644 sub foreach_drive {
2645     my ($conf, $func) = @_;
2646
2647     foreach my $ds (valid_drive_names()) {
2648         next if !defined($conf->{$ds});
2649
2650         my $drive = parse_drive($ds, $conf->{$ds});
2651         next if !$drive;
2652
2653         &$func($ds, $drive);
2654     }
2655 }
2656
2657 sub foreach_volid {
2658     my ($conf, $func) = @_;
2659
2660     my $volhash = {};
2661
2662     my $test_volid = sub {
2663         my ($volid, $is_cdrom) = @_;
2664
2665         return if !$volid;
2666
2667         $volhash->{$volid} = $is_cdrom || 0;
2668     };
2669
2670     foreach_drive($conf, sub {
2671         my ($ds, $drive) = @_;
2672         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2673     });
2674
2675     foreach my $snapname (keys %{$conf->{snapshots}}) {
2676         my $snap = $conf->{snapshots}->{$snapname};
2677         &$test_volid($snap->{vmstate}, 0);
2678         foreach_drive($snap, sub {
2679             my ($ds, $drive) = @_;
2680             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2681         });
2682     }
2683
2684     foreach my $volid (keys %$volhash) {
2685         &$func($volid, $volhash->{$volid});
2686     }
2687 }
2688
2689 sub vga_conf_has_spice {
2690     my ($vga) = @_;
2691
2692     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2693
2694     return $1 || 1;
2695 }
2696
2697 sub config_to_command {
2698     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2699
2700     my $cmd = [];
2701     my $globalFlags = [];
2702     my $machineFlags = [];
2703     my $rtcFlags = [];
2704     my $cpuFlags = [];
2705     my $devices = [];
2706     my $pciaddr = '';
2707     my $bridges = {};
2708     my $kvmver = kvm_user_version();
2709     my $vernum = 0; # unknown
2710     my $ostype = $conf->{ostype};
2711     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2712         $vernum = $1*1000000+$2*1000;
2713     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2714         $vernum = $1*1000000+$2*1000+$3;
2715     }
2716
2717     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2718
2719     my $have_ovz = -f '/proc/vz/vestat';
2720
2721     my $q35 = machine_type_is_q35($conf);
2722     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2723     my $machine_type = $forcemachine || $conf->{machine};
2724     my $use_old_bios_files = undef;
2725     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2726
2727     my $cpuunits = defined($conf->{cpuunits}) ?
2728             $conf->{cpuunits} : $defaults->{cpuunits};
2729
2730     push @$cmd, '/usr/bin/systemd-run';
2731     push @$cmd, '--scope';
2732     push @$cmd, '--slice', "qemu";
2733     push @$cmd, '--unit', $vmid;
2734     # set KillMode=none, so that systemd don't kill those scopes
2735     # at shutdown (pve-manager service should stop the VMs instead)
2736     push @$cmd, '-p', "KillMode=none";
2737     push @$cmd, '-p', "CPUShares=$cpuunits";
2738     if ($conf->{cpulimit}) {
2739         my $cpulimit = int($conf->{cpulimit} * 100);
2740         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2741     }
2742
2743     push @$cmd, '/usr/bin/kvm';
2744
2745     push @$cmd, '-id', $vmid;
2746
2747     my $use_virtio = 0;
2748
2749     my $qmpsocket = qmp_socket($vmid);
2750     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2751     push @$cmd, '-mon', "chardev=qmp,mode=control";
2752
2753
2754     push @$cmd, '-pidfile' , pidfile_name($vmid);
2755
2756     push @$cmd, '-daemonize';
2757
2758     if ($conf->{smbios1}) {
2759         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2760     }
2761
2762     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2763         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2764         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2765         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2766         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2767         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2768         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2769     }
2770
2771     if ($q35) {
2772         # the q35 chipset support native usb2, so we enable usb controller
2773         # by default for this machine type
2774         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2775     } else {
2776         $pciaddr = print_pci_addr("piix3", $bridges);
2777         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2778
2779         my $use_usb2 = 0;
2780         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2781             next if !$conf->{"usb$i"};
2782             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2783             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2784             $use_usb2 = 1;
2785         }
2786         # include usb device config
2787         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2788     }
2789
2790     # add usb3 controller if needed
2791
2792     my $use_usb3 = 0;
2793     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2794         next if !$conf->{"usb$i"};
2795         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2796         next if !$d || !$d->{usb3};
2797         $use_usb3 = 1;
2798     }
2799
2800     $pciaddr = print_pci_addr("xhci", $bridges);
2801     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2802
2803     my $vga = $conf->{vga};
2804
2805     my $qxlnum = vga_conf_has_spice($vga);
2806     $vga = 'qxl' if $qxlnum;
2807
2808     if (!$vga) {
2809         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2810                                 $conf->{ostype} eq 'win7' ||
2811                                 $conf->{ostype} eq 'w2k8')) {
2812             $vga = 'std';
2813         } else {
2814             $vga = 'cirrus';
2815         }
2816     }
2817
2818     # enable absolute mouse coordinates (needed by vnc)
2819     my $tablet;
2820     if (defined($conf->{tablet})) {
2821         $tablet = $conf->{tablet};
2822     } else {
2823         $tablet = $defaults->{tablet};
2824         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2825         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2826     }
2827
2828     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2829
2830     my $kvm_off = 0;
2831     # host pci devices
2832     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2833         my $d = parse_hostpci($conf->{"hostpci$i"});
2834         next if !$d;
2835
2836         my $pcie = $d->{pcie};
2837         if($pcie){
2838             die "q35 machine model is not enabled" if !$q35;
2839             $pciaddr = print_pcie_addr("hostpci$i");
2840         }else{
2841             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2842         }
2843
2844         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2845         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2846         if ($xvga && $xvga ne '') {
2847             $kvm_off = 1;
2848             $vga = 'none';
2849             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2850                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2851             }
2852             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2853                 $xvga = "";
2854             }
2855         }
2856         my $pcidevices = $d->{pciid};
2857         my $multifunction = 1 if @$pcidevices > 1;
2858
2859         my $j=0;
2860         foreach my $pcidevice (@$pcidevices) {
2861
2862             my $id = "hostpci$i";
2863             $id .= ".$j" if $multifunction;
2864             my $addr = $pciaddr;
2865             $addr .= ".$j" if $multifunction;
2866             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2867
2868             if($j == 0){
2869                 $devicestr .= "$rombar$xvga";
2870                 $devicestr .= ",multifunction=on" if $multifunction;
2871             }
2872
2873             push @$devices, '-device', $devicestr;
2874             $j++;
2875         }
2876     }
2877
2878     # usb devices
2879     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2880         next if !$conf->{"usb$i"};
2881         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2882         next if !$d;
2883
2884         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2885         my $usbbus = '';
2886         if (defined($d->{usb3}) && $d->{usb3}) {
2887             $usbbus = ',bus=xhci.0';
2888         }
2889
2890         if (defined($d->{host})) {
2891             $d = parse_usb_device($d->{host});
2892             if (defined($d->{vendorid}) && defined($d->{productid})) {
2893                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2894             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2895                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2896             } elsif (defined($d->{spice}) && $d->{spice}) {
2897                 # usb redir support for spice, currently no usb3
2898                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2899                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2900             }
2901         }
2902     }
2903
2904     # serial devices
2905     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2906         if (my $path = $conf->{"serial$i"}) {
2907             if ($path eq 'socket') {
2908                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2909                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2910                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2911             } else {
2912                 die "no such serial device\n" if ! -c $path;
2913                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2914                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2915             }
2916         }
2917     }
2918
2919     # parallel devices
2920     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2921         if (my $path = $conf->{"parallel$i"}) {
2922             die "no such parallel device\n" if ! -c $path;
2923             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2924             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2925             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2926         }
2927     }
2928
2929     my $vmname = $conf->{name} || "vm$vmid";
2930
2931     push @$cmd, '-name', $vmname;
2932
2933     my $sockets = 1;
2934     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2935     $sockets = $conf->{sockets} if  $conf->{sockets};
2936
2937     my $cores = $conf->{cores} || 1;
2938
2939     my $maxcpus = $sockets * $cores;
2940
2941     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2942
2943     my $allowed_vcpus = $cpuinfo->{cpus};
2944
2945     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2946         if ($allowed_vcpus < $maxcpus);
2947
2948     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2949
2950     push @$cmd, '-nodefaults';
2951
2952     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2953
2954     my $bootindex_hash = {};
2955     my $i = 1;
2956     foreach my $o (split(//, $bootorder)) {
2957         $bootindex_hash->{$o} = $i*100;
2958         $i++;
2959     }
2960
2961     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2962
2963     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2964
2965     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2966
2967     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2968
2969     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2970         my $socket = vnc_socket($vmid);
2971         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2972     } else {
2973         push @$cmd, '-nographic';
2974     }
2975
2976     # time drift fix
2977     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2978
2979     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2980     my $useLocaltime = $conf->{localtime};
2981
2982     if ($ostype) {
2983         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2984
2985         if ($ostype =~ m/^w/) { # windows
2986             $useLocaltime = 1 if !defined($conf->{localtime});
2987
2988             # use time drift fix when acpi is enabled
2989             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2990                 $tdf = 1 if !defined($conf->{tdf});
2991             }
2992         }
2993
2994         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2995             $ostype eq 'wvista') {
2996             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2997             push @$cmd, '-no-hpet';
2998             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2999                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3000                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3001                 push @$cpuFlags , 'hv_time' if !$nokvm;
3002
3003             } else {
3004                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3005             }
3006         }
3007
3008         if ($ostype eq 'win7' || $ostype eq 'win8') {
3009             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3010         }
3011     }
3012
3013     push @$rtcFlags, 'driftfix=slew' if $tdf;
3014
3015     if ($nokvm) {
3016         push @$machineFlags, 'accel=tcg';
3017     } else {
3018         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3019     }
3020
3021     if ($machine_type) {
3022         push @$machineFlags, "type=${machine_type}";
3023     }
3024
3025     if ($conf->{startdate}) {
3026         push @$rtcFlags, "base=$conf->{startdate}";
3027     } elsif ($useLocaltime) {
3028         push @$rtcFlags, 'base=localtime';
3029     }
3030
3031     my $cpu = $nokvm ? "qemu64" : "kvm64";
3032     if (my $cputype = $conf->{cpu}) {
3033         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3034             or die "Cannot parse cpu description: $cputype\n";
3035         $cpu = $cpuconf->{cputype};
3036         $kvm_off = 1 if $cpuconf->{hidden};
3037     }
3038
3039     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3040
3041     push @$cpuFlags , '-x2apic'
3042         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3043
3044     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3045
3046     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3047
3048     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3049
3050         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3051         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3052     }
3053
3054     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3055
3056     push @$cpuFlags, 'kvm=off' if $kvm_off;
3057
3058     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3059
3060     push @$cmd, '-cpu', $cpu;
3061
3062     my $memory = $conf->{memory} || $defaults->{memory};
3063     my $static_memory = 0;
3064     my $dimm_memory = 0;
3065
3066     if ($hotplug_features->{memory}) {
3067         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3068         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3069         $static_memory = $STATICMEM;
3070         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3071         $dimm_memory = $memory - $static_memory;
3072         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3073
3074     } else {
3075
3076         $static_memory = $memory;
3077         push @$cmd, '-m', $static_memory;
3078     }
3079
3080     if ($conf->{numa}) {
3081
3082         my $numa_totalmemory = undef;
3083         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3084             next if !$conf->{"numa$i"};
3085             my $numa = parse_numa($conf->{"numa$i"});
3086             next if !$numa;
3087             # memory
3088             die "missing numa node$i memory value\n" if !$numa->{memory};
3089             my $numa_memory = $numa->{memory};
3090             $numa_totalmemory += $numa_memory;
3091             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3092
3093             # cpus
3094             my $cpulists = $numa->{cpus};
3095             die "missing numa node$i cpus\n" if !defined($cpulists);
3096             my $cpus = join(',', map {
3097                 my ($start, $end) = @$_;
3098                 defined($end) ? "$start-$end" : $start
3099             } @$cpulists);
3100
3101             # hostnodes
3102             my $hostnodelists = $numa->{hostnodes};
3103             if (defined($hostnodelists)) {
3104                 my $hostnodes;
3105                 foreach my $hostnoderange (@$hostnodelists) {
3106                     my ($start, $end) = @$hostnoderange;
3107                     $hostnodes .= ',' if $hostnodes;
3108                     $hostnodes .= $start;
3109                     $hostnodes .= "-$end" if defined($end);
3110                     $end //= $start;
3111                     for (my $i = $start; $i <= $end; ++$i ) {
3112                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3113                     }
3114                 }
3115
3116                 # policy
3117                 my $policy = $numa->{policy};
3118                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3119                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3120             }
3121
3122             push @$cmd, '-object', $numa_object;
3123             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3124         }
3125
3126         die "total memory for NUMA nodes must be equal to vm static memory\n"
3127             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3128
3129         #if no custom tology, we split memory and cores across numa nodes
3130         if(!$numa_totalmemory) {
3131
3132             my $numa_memory = ($static_memory / $sockets) . "M";
3133
3134             for (my $i = 0; $i < $sockets; $i++)  {
3135
3136                 my $cpustart = ($cores * $i);
3137                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3138                 my $cpus = $cpustart;
3139                 $cpus .= "-$cpuend" if $cpuend;
3140
3141                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3142                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3143             }
3144         }
3145     }
3146
3147     if ($hotplug_features->{memory}) {
3148         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3149             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3150             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3151             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3152
3153             #if dimm_memory is not aligned to dimm map
3154             if($current_size > $memory) {
3155                  $conf->{memory} = $current_size;
3156                  PVE::QemuConfig->write_config($vmid, $conf);
3157             }
3158         });
3159     }
3160
3161     push @$cmd, '-S' if $conf->{freeze};
3162
3163     # set keyboard layout
3164     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3165     push @$cmd, '-k', $kb if $kb;
3166
3167     # enable sound
3168     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3169     #push @$cmd, '-soundhw', 'es1370';
3170     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3171
3172     if($conf->{agent}) {
3173         my $qgasocket = qmp_socket($vmid, 1);
3174         my $pciaddr = print_pci_addr("qga0", $bridges);
3175         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3176         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3177         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3178     }
3179
3180     my $spice_port;
3181
3182     if ($qxlnum) {
3183         if ($qxlnum > 1) {
3184             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3185                 for(my $i = 1; $i < $qxlnum; $i++){
3186                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3187                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3188                 }
3189             } else {
3190                 # assume other OS works like Linux
3191                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3192                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3193             }
3194         }
3195
3196         my $pciaddr = print_pci_addr("spice", $bridges);
3197
3198         my $nodename = PVE::INotify::nodename();
3199         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3200         $spice_port = PVE::Tools::next_spice_port($pfamily);
3201
3202         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3203
3204         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3205         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3206         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3207     }
3208
3209     # enable balloon by default, unless explicitly disabled
3210     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3211         $pciaddr = print_pci_addr("balloon0", $bridges);
3212         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3213     }
3214
3215     if ($conf->{watchdog}) {
3216         my $wdopts = parse_watchdog($conf->{watchdog});
3217         $pciaddr = print_pci_addr("watchdog", $bridges);
3218         my $watchdog = $wdopts->{model} || 'i6300esb';
3219         push @$devices, '-device', "$watchdog$pciaddr";
3220         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3221     }
3222
3223     my $vollist = [];
3224     my $scsicontroller = {};
3225     my $ahcicontroller = {};
3226     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3227
3228     # Add iscsi initiator name if available
3229     if (my $initiator = get_initiator_name()) {
3230         push @$devices, '-iscsi', "initiator-name=$initiator";
3231     }
3232
3233     foreach_drive($conf, sub {
3234         my ($ds, $drive) = @_;
3235
3236         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3237             push @$vollist, $drive->{file};
3238         }
3239
3240         $use_virtio = 1 if $ds =~ m/^virtio/;
3241
3242         if (drive_is_cdrom ($drive)) {
3243             if ($bootindex_hash->{d}) {
3244                 $drive->{bootindex} = $bootindex_hash->{d};
3245                 $bootindex_hash->{d} += 1;
3246             }
3247         } else {
3248             if ($bootindex_hash->{c}) {
3249                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3250                 $bootindex_hash->{c} += 1;
3251             }
3252         }
3253
3254         if($drive->{interface} eq 'virtio'){
3255            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3256         }
3257
3258         if ($drive->{interface} eq 'scsi') {
3259
3260             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3261
3262             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3263             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3264
3265             my $iothread = '';
3266             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3267                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3268                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3269             }
3270
3271             my $queues = '';
3272             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3273                 $queues = ",num_queues=$drive->{queues}";
3274             }
3275
3276             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3277             $scsicontroller->{$controller}=1;
3278         }
3279
3280         if ($drive->{interface} eq 'sata') {
3281            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3282            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3283            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3284            $ahcicontroller->{$controller}=1;
3285         }
3286
3287         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3288         push @$devices, '-drive',$drive_cmd;
3289         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3290     });
3291
3292     for (my $i = 0; $i < $MAX_NETS; $i++) {
3293          next if !$conf->{"net$i"};
3294          my $d = parse_net($conf->{"net$i"});
3295          next if !$d;
3296
3297          $use_virtio = 1 if $d->{model} eq 'virtio';
3298
3299          if ($bootindex_hash->{n}) {
3300             $d->{bootindex} = $bootindex_hash->{n};
3301             $bootindex_hash->{n} += 1;
3302          }
3303
3304          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3305          push @$devices, '-netdev', $netdevfull;
3306
3307          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3308          push @$devices, '-device', $netdevicefull;
3309     }
3310
3311     if (!$q35) {
3312         # add pci bridges
3313         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3314            $bridges->{1} = 1;
3315            $bridges->{2} = 1;
3316         }
3317
3318         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3319
3320         while (my ($k, $v) = each %$bridges) {
3321             $pciaddr = print_pci_addr("pci.$k");
3322             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3323         }
3324     }
3325
3326     # add custom args
3327     if ($conf->{args}) {
3328         my $aa = PVE::Tools::split_args($conf->{args});
3329         push @$cmd, @$aa;
3330     }
3331
3332     push @$cmd, @$devices;
3333     push @$cmd, '-rtc', join(',', @$rtcFlags)
3334         if scalar(@$rtcFlags);
3335     push @$cmd, '-machine', join(',', @$machineFlags)
3336         if scalar(@$machineFlags);
3337     push @$cmd, '-global', join(',', @$globalFlags)
3338         if scalar(@$globalFlags);
3339
3340     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3341 }
3342
3343 sub vnc_socket {
3344     my ($vmid) = @_;
3345     return "${var_run_tmpdir}/$vmid.vnc";
3346 }
3347
3348 sub spice_port {
3349     my ($vmid) = @_;
3350
3351     my $res = vm_mon_cmd($vmid, 'query-spice');
3352
3353     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3354 }
3355
3356 sub qmp_socket {
3357     my ($vmid, $qga) = @_;
3358     my $sockettype = $qga ? 'qga' : 'qmp';
3359     return "${var_run_tmpdir}/$vmid.$sockettype";
3360 }
3361
3362 sub pidfile_name {
3363     my ($vmid) = @_;
3364     return "${var_run_tmpdir}/$vmid.pid";
3365 }
3366
3367 sub vm_devices_list {
3368     my ($vmid) = @_;
3369
3370     my $res = vm_mon_cmd($vmid, 'query-pci');
3371     my $devices = {};
3372     foreach my $pcibus (@$res) {
3373         foreach my $device (@{$pcibus->{devices}}) {
3374             next if !$device->{'qdev_id'};
3375             if ($device->{'pci_bridge'}) {
3376                 $devices->{$device->{'qdev_id'}} = 1;
3377                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3378                     next if !$bridge_device->{'qdev_id'};
3379                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3380                     $devices->{$device->{'qdev_id'}}++;
3381                 }
3382             } else {
3383                 $devices->{$device->{'qdev_id'}} = 1;
3384             }
3385         }
3386     }
3387
3388     my $resblock = vm_mon_cmd($vmid, 'query-block');
3389     foreach my $block (@$resblock) {
3390         if($block->{device} =~ m/^drive-(\S+)/){
3391                 $devices->{$1} = 1;
3392         }
3393     }
3394
3395     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3396     foreach my $mice (@$resmice) {
3397         if ($mice->{name} eq 'QEMU HID Tablet') {
3398             $devices->{tablet} = 1;
3399             last;
3400         }
3401     }
3402
3403     return $devices;
3404 }
3405
3406 sub vm_deviceplug {
3407     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3408
3409     my $q35 = machine_type_is_q35($conf);
3410
3411     my $devices_list = vm_devices_list($vmid);
3412     return 1 if defined($devices_list->{$deviceid});
3413
3414     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3415
3416     if ($deviceid eq 'tablet') {
3417
3418         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3419
3420     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3421
3422         qemu_iothread_add($vmid, $deviceid, $device);
3423
3424         qemu_driveadd($storecfg, $vmid, $device);
3425         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3426
3427         qemu_deviceadd($vmid, $devicefull);
3428         eval { qemu_deviceaddverify($vmid, $deviceid); };
3429         if (my $err = $@) {
3430             eval { qemu_drivedel($vmid, $deviceid); };
3431             warn $@ if $@;
3432             die $err;
3433         }
3434
3435     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3436
3437
3438         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3439         my $pciaddr = print_pci_addr($deviceid);
3440         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3441
3442         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3443
3444         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3445             qemu_iothread_add($vmid, $deviceid, $device);
3446             $devicefull .= ",iothread=iothread-$deviceid";
3447         }
3448
3449         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3450             $devicefull .= ",num_queues=$device->{queues}";
3451         }
3452
3453         qemu_deviceadd($vmid, $devicefull);
3454         qemu_deviceaddverify($vmid, $deviceid);
3455
3456     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3457
3458         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3459         qemu_driveadd($storecfg, $vmid, $device);
3460
3461         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3462         eval { qemu_deviceadd($vmid, $devicefull); };
3463         if (my $err = $@) {
3464             eval { qemu_drivedel($vmid, $deviceid); };
3465             warn $@ if $@;
3466             die $err;
3467         }
3468
3469     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3470
3471         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3472
3473         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3474         my $use_old_bios_files = undef;
3475         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3476
3477         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3478         qemu_deviceadd($vmid, $netdevicefull);
3479         eval { qemu_deviceaddverify($vmid, $deviceid); };
3480         if (my $err = $@) {
3481             eval { qemu_netdevdel($vmid, $deviceid); };
3482             warn $@ if $@;
3483             die $err;
3484         }
3485
3486     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3487
3488         my $bridgeid = $2;
3489         my $pciaddr = print_pci_addr($deviceid);
3490         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3491
3492         qemu_deviceadd($vmid, $devicefull);
3493         qemu_deviceaddverify($vmid, $deviceid);
3494
3495     } else {
3496         die "can't hotplug device '$deviceid'\n";
3497     }
3498
3499     return 1;
3500 }
3501
3502 # fixme: this should raise exceptions on error!
3503 sub vm_deviceunplug {
3504     my ($vmid, $conf, $deviceid) = @_;
3505
3506     my $devices_list = vm_devices_list($vmid);
3507     return 1 if !defined($devices_list->{$deviceid});
3508
3509     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3510
3511     if ($deviceid eq 'tablet') {
3512
3513         qemu_devicedel($vmid, $deviceid);
3514
3515     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3516
3517         qemu_devicedel($vmid, $deviceid);
3518         qemu_devicedelverify($vmid, $deviceid);
3519         qemu_drivedel($vmid, $deviceid);
3520         qemu_iothread_del($conf, $vmid, $deviceid);
3521
3522     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3523
3524         qemu_devicedel($vmid, $deviceid);
3525         qemu_devicedelverify($vmid, $deviceid);
3526         qemu_iothread_del($conf, $vmid, $deviceid);
3527
3528     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3529
3530         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3531         my $device = parse_drive($deviceid, $conf->{$deviceid});
3532         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3533
3534         qemu_devicedel($vmid, $deviceid);
3535         qemu_drivedel($vmid, $deviceid);
3536         qemu_deletescsihw($conf, $vmid, $deviceid);
3537
3538     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3539
3540         qemu_devicedel($vmid, $deviceid);
3541         qemu_devicedelverify($vmid, $deviceid);
3542         qemu_netdevdel($vmid, $deviceid);
3543
3544     } else {
3545         die "can't unplug device '$deviceid'\n";
3546     }
3547
3548     return 1;
3549 }
3550
3551 sub qemu_deviceadd {
3552     my ($vmid, $devicefull) = @_;
3553
3554     $devicefull = "driver=".$devicefull;
3555     my %options =  split(/[=,]/, $devicefull);
3556
3557     vm_mon_cmd($vmid, "device_add" , %options);
3558 }
3559
3560 sub qemu_devicedel {
3561     my ($vmid, $deviceid) = @_;
3562
3563     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3564 }
3565
3566 sub qemu_iothread_add {
3567     my($vmid, $deviceid, $device) = @_;
3568
3569     if ($device->{iothread}) {
3570         my $iothreads = vm_iothreads_list($vmid);
3571         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3572     }
3573 }
3574
3575 sub qemu_iothread_del {
3576     my($conf, $vmid, $deviceid) = @_;
3577
3578     my $device = parse_drive($deviceid, $conf->{$deviceid});
3579     if ($device->{iothread}) {
3580         my $iothreads = vm_iothreads_list($vmid);
3581         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3582     }
3583 }
3584
3585 sub qemu_objectadd {
3586     my($vmid, $objectid, $qomtype) = @_;
3587
3588     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3589
3590     return 1;
3591 }
3592
3593 sub qemu_objectdel {
3594     my($vmid, $objectid) = @_;
3595
3596     vm_mon_cmd($vmid, "object-del", id => $objectid);
3597
3598     return 1;
3599 }
3600
3601 sub qemu_driveadd {
3602     my ($storecfg, $vmid, $device) = @_;
3603
3604     my $drive = print_drive_full($storecfg, $vmid, $device);
3605     $drive =~ s/\\/\\\\/g;
3606     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3607
3608     # If the command succeeds qemu prints: "OK"
3609     return 1 if $ret =~ m/OK/s;
3610
3611     die "adding drive failed: $ret\n";
3612 }
3613
3614 sub qemu_drivedel {
3615     my($vmid, $deviceid) = @_;
3616
3617     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3618     $ret =~ s/^\s+//;
3619
3620     return 1 if $ret eq "";
3621
3622     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3623     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3624
3625     die "deleting drive $deviceid failed : $ret\n";
3626 }
3627
3628 sub qemu_deviceaddverify {
3629     my ($vmid, $deviceid) = @_;
3630
3631     for (my $i = 0; $i <= 5; $i++) {
3632          my $devices_list = vm_devices_list($vmid);
3633          return 1 if defined($devices_list->{$deviceid});
3634          sleep 1;
3635     }
3636
3637     die "error on hotplug device '$deviceid'\n";
3638 }
3639
3640
3641 sub qemu_devicedelverify {
3642     my ($vmid, $deviceid) = @_;
3643
3644     # need to verify that the device is correctly removed as device_del
3645     # is async and empty return is not reliable
3646
3647     for (my $i = 0; $i <= 5; $i++) {
3648          my $devices_list = vm_devices_list($vmid);
3649          return 1 if !defined($devices_list->{$deviceid});
3650          sleep 1;
3651     }
3652
3653     die "error on hot-unplugging device '$deviceid'\n";
3654 }
3655
3656 sub qemu_findorcreatescsihw {
3657     my ($storecfg, $conf, $vmid, $device) = @_;
3658
3659     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3660
3661     my $scsihwid="$controller_prefix$controller";
3662     my $devices_list = vm_devices_list($vmid);
3663
3664     if(!defined($devices_list->{$scsihwid})) {
3665         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3666     }
3667
3668     return 1;
3669 }
3670
3671 sub qemu_deletescsihw {
3672     my ($conf, $vmid, $opt) = @_;
3673
3674     my $device = parse_drive($opt, $conf->{$opt});
3675
3676     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3677         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3678         return 1;
3679     }
3680
3681     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3682
3683     my $devices_list = vm_devices_list($vmid);
3684     foreach my $opt (keys %{$devices_list}) {
3685         if (PVE::QemuServer::is_valid_drivename($opt)) {
3686             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3687             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3688                 return 1;
3689             }
3690         }
3691     }
3692
3693     my $scsihwid="scsihw$controller";
3694
3695     vm_deviceunplug($vmid, $conf, $scsihwid);
3696
3697     return 1;
3698 }
3699
3700 sub qemu_add_pci_bridge {
3701     my ($storecfg, $conf, $vmid, $device) = @_;
3702
3703     my $bridges = {};
3704
3705     my $bridgeid;
3706
3707     print_pci_addr($device, $bridges);
3708
3709     while (my ($k, $v) = each %$bridges) {
3710         $bridgeid = $k;
3711     }
3712     return 1 if !defined($bridgeid) || $bridgeid < 1;
3713
3714     my $bridge = "pci.$bridgeid";
3715     my $devices_list = vm_devices_list($vmid);
3716
3717     if (!defined($devices_list->{$bridge})) {
3718         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3719     }
3720
3721     return 1;
3722 }
3723
3724 sub qemu_set_link_status {
3725     my ($vmid, $device, $up) = @_;
3726
3727     vm_mon_cmd($vmid, "set_link", name => $device,
3728                up => $up ? JSON::true : JSON::false);
3729 }
3730
3731 sub qemu_netdevadd {
3732     my ($vmid, $conf, $device, $deviceid) = @_;
3733
3734     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3735     my %options =  split(/[=,]/, $netdev);
3736
3737     vm_mon_cmd($vmid, "netdev_add",  %options);
3738     return 1;
3739 }
3740
3741 sub qemu_netdevdel {
3742     my ($vmid, $deviceid) = @_;
3743
3744     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3745 }
3746
3747 sub qemu_cpu_hotplug {
3748     my ($vmid, $conf, $vcpus) = @_;
3749
3750     my $sockets = 1;
3751     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3752     $sockets = $conf->{sockets} if  $conf->{sockets};
3753     my $cores = $conf->{cores} || 1;
3754     my $maxcpus = $sockets * $cores;
3755
3756     $vcpus = $maxcpus if !$vcpus;
3757
3758     die "you can't add more vcpus than maxcpus\n"
3759         if $vcpus > $maxcpus;
3760
3761     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3762     die "online cpu unplug is not yet possible\n"
3763         if $vcpus < $currentvcpus;
3764
3765     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3766     die "vcpus in running vm is different than configuration\n"
3767         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3768
3769     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3770         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3771     }
3772 }
3773
3774 sub qemu_memory_hotplug {
3775     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3776
3777     return $value if !check_running($vmid);
3778
3779     my $memory = $conf->{memory} || $defaults->{memory};
3780     $value = $defaults->{memory} if !$value;
3781     return $value if $value == $memory;
3782
3783     my $static_memory = $STATICMEM;
3784     my $dimm_memory = $memory - $static_memory;
3785
3786     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3787     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3788
3789
3790     my $sockets = 1;
3791     $sockets = $conf->{sockets} if $conf->{sockets};
3792
3793     if($value > $memory) {
3794
3795         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3796             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3797
3798                 return if $current_size <= $conf->{memory};
3799
3800                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3801                 if (my $err = $@) {
3802                     eval { qemu_objectdel($vmid, "mem-$name"); };
3803                     die $err;
3804                 }
3805
3806                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3807                 if (my $err = $@) {
3808                     eval { qemu_objectdel($vmid, "mem-$name"); };
3809                     die $err;
3810                 }
3811                 #update conf after each succesful module hotplug
3812                 $conf->{memory} = $current_size;
3813                 PVE::QemuConfig->write_config($vmid, $conf);
3814         });
3815
3816     } else {
3817
3818         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3819             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3820
3821                 return if $current_size >= $conf->{memory};
3822                 print "try to unplug memory dimm $name\n";
3823
3824                 my $retry = 0;
3825                 while (1) {
3826                     eval { qemu_devicedel($vmid, $name) };
3827                     sleep 3;
3828                     my $dimm_list = qemu_dimm_list($vmid);
3829                     last if !$dimm_list->{$name};
3830                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3831                     $retry++;
3832                 }
3833
3834                 #update conf after each succesful module unplug
3835                 $conf->{memory} = $current_size;
3836
3837                 eval { qemu_objectdel($vmid, "mem-$name"); };
3838                 PVE::QemuConfig->write_config($vmid, $conf);
3839         });
3840     }
3841 }
3842
3843 sub qemu_dimm_list {
3844     my ($vmid) = @_;
3845
3846     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3847     my $dimms = {};
3848
3849     foreach my $dimm (@$dimmarray) {
3850
3851         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3852         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3853         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3854         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3855         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3856     }
3857     return $dimms;
3858 }
3859
3860 sub qemu_block_set_io_throttle {
3861     my ($vmid, $deviceid,
3862         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3863         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3864
3865     return if !check_running($vmid) ;
3866
3867     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3868         bps => int($bps),
3869         bps_rd => int($bps_rd),
3870         bps_wr => int($bps_wr),
3871         iops => int($iops),
3872         iops_rd => int($iops_rd),
3873         iops_wr => int($iops_wr),
3874         bps_max => int($bps_max),
3875         bps_rd_max => int($bps_rd_max),
3876         bps_wr_max => int($bps_wr_max),
3877         iops_max => int($iops_max),
3878         iops_rd_max => int($iops_rd_max),
3879         iops_wr_max => int($iops_wr_max)
3880     );
3881
3882 }
3883
3884 # old code, only used to shutdown old VM after update
3885 sub __read_avail {
3886     my ($fh, $timeout) = @_;
3887
3888     my $sel = new IO::Select;
3889     $sel->add($fh);
3890
3891     my $res = '';
3892     my $buf;
3893
3894     my @ready;
3895     while (scalar (@ready = $sel->can_read($timeout))) {
3896         my $count;
3897         if ($count = $fh->sysread($buf, 8192)) {
3898             if ($buf =~ /^(.*)\(qemu\) $/s) {
3899                 $res .= $1;
3900                 last;
3901             } else {
3902                 $res .= $buf;
3903             }
3904         } else {
3905             if (!defined($count)) {
3906                 die "$!\n";
3907             }
3908             last;
3909         }
3910     }
3911
3912     die "monitor read timeout\n" if !scalar(@ready);
3913
3914     return $res;
3915 }
3916
3917 # old code, only used to shutdown old VM after update
3918 sub vm_monitor_command {
3919     my ($vmid, $cmdstr, $nocheck) = @_;
3920
3921     my $res;
3922
3923     eval {
3924         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3925
3926         my $sname = "${var_run_tmpdir}/$vmid.mon";
3927
3928         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3929             die "unable to connect to VM $vmid socket - $!\n";
3930
3931         my $timeout = 3;
3932
3933         # hack: migrate sometime blocks the monitor (when migrate_downtime
3934         # is set)
3935         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3936             $timeout = 60*60; # 1 hour
3937         }
3938
3939         # read banner;
3940         my $data = __read_avail($sock, $timeout);
3941
3942         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3943             die "got unexpected qemu monitor banner\n";
3944         }
3945
3946         my $sel = new IO::Select;
3947         $sel->add($sock);
3948
3949         if (!scalar(my @ready = $sel->can_write($timeout))) {
3950             die "monitor write error - timeout";
3951         }
3952
3953         my $fullcmd = "$cmdstr\r";
3954
3955         # syslog('info', "VM $vmid monitor command: $cmdstr");
3956
3957         my $b;
3958         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3959             die "monitor write error - $!";
3960         }
3961
3962         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3963
3964         $timeout = 20;
3965
3966         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3967             $timeout = 60*60; # 1 hour
3968         } elsif ($cmdstr =~ m/^(eject|change)/) {
3969             $timeout = 60; # note: cdrom mount command is slow
3970         }
3971         if ($res = __read_avail($sock, $timeout)) {
3972
3973             my @lines = split("\r?\n", $res);
3974
3975             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3976
3977             $res = join("\n", @lines);
3978             $res .= "\n";
3979         }
3980     };
3981
3982     my $err = $@;
3983
3984     if ($err) {
3985         syslog("err", "VM $vmid monitor command failed - $err");
3986         die $err;
3987     }
3988
3989     return $res;
3990 }
3991
3992 sub qemu_block_resize {
3993     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3994
3995     my $running = check_running($vmid);
3996
3997     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3998
3999     return if !$running;
4000
4001     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4002
4003 }
4004
4005 sub qemu_volume_snapshot {
4006     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4007
4008     my $running = check_running($vmid);
4009
4010     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4011         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4012     } else {
4013         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4014     }
4015 }
4016
4017 sub qemu_volume_snapshot_delete {
4018     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4019
4020     my $running = check_running($vmid);
4021
4022     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4023
4024     return if !$running;
4025
4026     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4027 }
4028
4029 sub set_migration_caps {
4030     my ($vmid) = @_;
4031
4032     my $cap_ref = [];
4033
4034     my $enabled_cap = {
4035         "auto-converge" => 1,
4036         "xbzrle" => 1,
4037         "x-rdma-pin-all" => 0,
4038         "zero-blocks" => 0,
4039         "compress" => 0
4040     };
4041
4042     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4043
4044     for my $supported_capability (@$supported_capabilities) {
4045         push @$cap_ref, {
4046             capability => $supported_capability->{capability},
4047             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4048         };
4049     }
4050
4051     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4052 }
4053
4054 my $fast_plug_option = {
4055     'lock' => 1,
4056     'name' => 1,
4057     'onboot' => 1,
4058     'shares' => 1,
4059     'startup' => 1,
4060     'description' => 1,
4061 };
4062
4063 # hotplug changes in [PENDING]
4064 # $selection hash can be used to only apply specified options, for
4065 # example: { cores => 1 } (only apply changed 'cores')
4066 # $errors ref is used to return error messages
4067 sub vmconfig_hotplug_pending {
4068     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4069
4070     my $defaults = load_defaults();
4071
4072     # commit values which do not have any impact on running VM first
4073     # Note: those option cannot raise errors, we we do not care about
4074     # $selection and always apply them.
4075
4076     my $add_error = sub {
4077         my ($opt, $msg) = @_;
4078         $errors->{$opt} = "hotplug problem - $msg";
4079     };
4080
4081     my $changes = 0;
4082     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4083         if ($fast_plug_option->{$opt}) {
4084             $conf->{$opt} = $conf->{pending}->{$opt};
4085             delete $conf->{pending}->{$opt};
4086             $changes = 1;
4087         }
4088     }
4089
4090     if ($changes) {
4091         PVE::QemuConfig->write_config($vmid, $conf);
4092         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4093     }
4094
4095     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4096
4097     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4098     while (my ($opt, $force) = each %$pending_delete_hash) {
4099         next if $selection && !$selection->{$opt};
4100         eval {
4101             if ($opt eq 'hotplug') {
4102                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4103             } elsif ($opt eq 'tablet') {
4104                 die "skip\n" if !$hotplug_features->{usb};
4105                 if ($defaults->{tablet}) {
4106                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4107                 } else {
4108                     vm_deviceunplug($vmid, $conf, $opt);
4109                 }
4110             } elsif ($opt eq 'vcpus') {
4111                 die "skip\n" if !$hotplug_features->{cpu};
4112                 qemu_cpu_hotplug($vmid, $conf, undef);
4113             } elsif ($opt eq 'balloon') {
4114                 # enable balloon device is not hotpluggable
4115                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4116             } elsif ($fast_plug_option->{$opt}) {
4117                 # do nothing
4118             } elsif ($opt =~ m/^net(\d+)$/) {
4119                 die "skip\n" if !$hotplug_features->{network};
4120                 vm_deviceunplug($vmid, $conf, $opt);
4121             } elsif (is_valid_drivename($opt)) {
4122                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4123                 vm_deviceunplug($vmid, $conf, $opt);
4124                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4125             } elsif ($opt =~ m/^memory$/) {
4126                 die "skip\n" if !$hotplug_features->{memory};
4127                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4128             } elsif ($opt eq 'cpuunits') {
4129                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4130             } elsif ($opt eq 'cpulimit') {
4131                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4132             } else {
4133                 die "skip\n";
4134             }
4135         };
4136         if (my $err = $@) {
4137             &$add_error($opt, $err) if $err ne "skip\n";
4138         } else {
4139             # save new config if hotplug was successful
4140             delete $conf->{$opt};
4141             vmconfig_undelete_pending_option($conf, $opt);
4142             PVE::QemuConfig->write_config($vmid, $conf);
4143             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4144         }
4145     }
4146
4147     foreach my $opt (keys %{$conf->{pending}}) {
4148         next if $selection && !$selection->{$opt};
4149         my $value = $conf->{pending}->{$opt};
4150         eval {
4151             if ($opt eq 'hotplug') {
4152                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4153             } elsif ($opt eq 'tablet') {
4154                 die "skip\n" if !$hotplug_features->{usb};
4155                 if ($value == 1) {
4156                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4157                 } elsif ($value == 0) {
4158                     vm_deviceunplug($vmid, $conf, $opt);
4159                 }
4160             } elsif ($opt eq 'vcpus') {
4161                 die "skip\n" if !$hotplug_features->{cpu};
4162                 qemu_cpu_hotplug($vmid, $conf, $value);
4163             } elsif ($opt eq 'balloon') {
4164                 # enable/disable balloning device is not hotpluggable
4165                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4166                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4167                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4168
4169                 # allow manual ballooning if shares is set to zero
4170                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4171                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4172                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4173                 }
4174             } elsif ($opt =~ m/^net(\d+)$/) {
4175                 # some changes can be done without hotplug
4176                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4177                                     $vmid, $opt, $value);
4178             } elsif (is_valid_drivename($opt)) {
4179                 # some changes can be done without hotplug
4180                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4181                                      $vmid, $opt, $value, 1);
4182             } elsif ($opt =~ m/^memory$/) { #dimms
4183                 die "skip\n" if !$hotplug_features->{memory};
4184                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4185             } elsif ($opt eq 'cpuunits') {
4186                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4187             } elsif ($opt eq 'cpulimit') {
4188                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4189                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4190             } else {
4191                 die "skip\n";  # skip non-hot-pluggable options
4192             }
4193         };
4194         if (my $err = $@) {
4195             &$add_error($opt, $err) if $err ne "skip\n";
4196         } else {
4197             # save new config if hotplug was successful
4198             $conf->{$opt} = $value;
4199             delete $conf->{pending}->{$opt};
4200             PVE::QemuConfig->write_config($vmid, $conf);
4201             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4202         }
4203     }
4204 }
4205
4206 sub try_deallocate_drive {
4207     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4208
4209     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4210         my $volid = $drive->{file};
4211         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4212             my $sid = PVE::Storage::parse_volume_id($volid);
4213             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4214
4215             # check if the disk is really unused
4216             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4217                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4218             PVE::Storage::vdisk_free($storecfg, $volid);
4219             return 1;
4220         } else {
4221             # If vm is not owner of this disk remove from config
4222             return 1;
4223         }
4224     }
4225
4226     return undef;
4227 }
4228
4229 sub vmconfig_delete_or_detach_drive {
4230     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4231
4232     my $drive = parse_drive($opt, $conf->{$opt});
4233
4234     my $rpcenv = PVE::RPCEnvironment::get();
4235     my $authuser = $rpcenv->get_user();
4236
4237     if ($force) {
4238         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4239         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4240     } else {
4241         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4242     }
4243 }
4244
4245 sub vmconfig_apply_pending {
4246     my ($vmid, $conf, $storecfg) = @_;
4247
4248     # cold plug
4249
4250     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4251     while (my ($opt, $force) = each %$pending_delete_hash) {
4252         die "internal error" if $opt =~ m/^unused/;
4253         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4254         if (!defined($conf->{$opt})) {
4255             vmconfig_undelete_pending_option($conf, $opt);
4256             PVE::QemuConfig->write_config($vmid, $conf);
4257         } elsif (is_valid_drivename($opt)) {
4258             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4259             vmconfig_undelete_pending_option($conf, $opt);
4260             delete $conf->{$opt};
4261             PVE::QemuConfig->write_config($vmid, $conf);
4262         } else {
4263             vmconfig_undelete_pending_option($conf, $opt);
4264             delete $conf->{$opt};
4265             PVE::QemuConfig->write_config($vmid, $conf);
4266         }
4267     }
4268
4269     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4270
4271     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4272         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4273
4274         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4275             # skip if nothing changed
4276         } elsif (is_valid_drivename($opt)) {
4277             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4278                 if defined($conf->{$opt});
4279             $conf->{$opt} = $conf->{pending}->{$opt};
4280         } else {
4281             $conf->{$opt} = $conf->{pending}->{$opt};
4282         }
4283
4284         delete $conf->{pending}->{$opt};
4285         PVE::QemuConfig->write_config($vmid, $conf);
4286     }
4287 }
4288
4289 my $safe_num_ne = sub {
4290     my ($a, $b) = @_;
4291
4292     return 0 if !defined($a) && !defined($b);
4293     return 1 if !defined($a);
4294     return 1 if !defined($b);
4295
4296     return $a != $b;
4297 };
4298
4299 my $safe_string_ne = sub {
4300     my ($a, $b) = @_;
4301
4302     return 0 if !defined($a) && !defined($b);
4303     return 1 if !defined($a);
4304     return 1 if !defined($b);
4305
4306     return $a ne $b;
4307 };
4308
4309 sub vmconfig_update_net {
4310     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4311
4312     my $newnet = parse_net($value);
4313
4314     if ($conf->{$opt}) {
4315         my $oldnet = parse_net($conf->{$opt});
4316
4317         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4318             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4319             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4320             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4321
4322             # for non online change, we try to hot-unplug
4323             die "skip\n" if !$hotplug;
4324             vm_deviceunplug($vmid, $conf, $opt);
4325         } else {
4326
4327             die "internal error" if $opt !~ m/net(\d+)/;
4328             my $iface = "tap${vmid}i$1";
4329
4330             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4331                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4332                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4333                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4334                 PVE::Network::tap_unplug($iface);
4335                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4336             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4337                 # Rate can be applied on its own but any change above needs to
4338                 # include the rate in tap_plug since OVS resets everything.
4339                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4340             }
4341
4342             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4343                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4344             }
4345
4346             return 1;
4347         }
4348     }
4349
4350     if ($hotplug) {
4351         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4352     } else {
4353         die "skip\n";
4354     }
4355 }
4356
4357 sub vmconfig_update_disk {
4358     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4359
4360     # fixme: do we need force?
4361
4362     my $drive = parse_drive($opt, $value);
4363
4364     if ($conf->{$opt}) {
4365
4366         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4367
4368             my $media = $drive->{media} || 'disk';
4369             my $oldmedia = $old_drive->{media} || 'disk';
4370             die "unable to change media type\n" if $media ne $oldmedia;
4371
4372             if (!drive_is_cdrom($old_drive)) {
4373
4374                 if ($drive->{file} ne $old_drive->{file}) {
4375
4376                     die "skip\n" if !$hotplug;
4377
4378                     # unplug and register as unused
4379                     vm_deviceunplug($vmid, $conf, $opt);
4380                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4381
4382                 } else {
4383                     # update existing disk
4384
4385                     # skip non hotpluggable value
4386                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4387                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4388                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4389                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4390                         die "skip\n";
4391                     }
4392
4393                     # apply throttle
4394                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4395                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4396                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4397                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4398                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4399                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4400                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4401                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4402                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4403                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4404                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4405                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4406
4407                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4408                                                    ($drive->{mbps} || 0)*1024*1024,
4409                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4410                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4411                                                    $drive->{iops} || 0,
4412                                                    $drive->{iops_rd} || 0,
4413                                                    $drive->{iops_wr} || 0,
4414                                                    ($drive->{mbps_max} || 0)*1024*1024,
4415                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4416                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4417                                                    $drive->{iops_max} || 0,
4418                                                    $drive->{iops_rd_max} || 0,
4419                                                    $drive->{iops_wr_max} || 0);
4420
4421                     }
4422
4423                     return 1;
4424                 }
4425
4426             } else { # cdrom
4427
4428                 if ($drive->{file} eq 'none') {
4429                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4430                 } else {
4431                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4432                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4433                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4434                 }
4435
4436                 return 1;
4437             }
4438         }
4439     }
4440
4441     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4442     # hotplug new disks
4443     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4444     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4445 }
4446
4447 sub vm_start {
4448     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4449         $forcemachine, $spice_ticket) = @_;
4450
4451     PVE::QemuConfig->lock_config($vmid, sub {
4452         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4453
4454         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4455
4456         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4457
4458         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4459
4460         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4461             vmconfig_apply_pending($vmid, $conf, $storecfg);
4462             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4463         }
4464
4465         my $defaults = load_defaults();
4466
4467         # set environment variable useful inside network script
4468         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4469
4470         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4471
4472         my $migrate_port = 0;
4473         my $migrate_uri;
4474         if ($statefile) {
4475             if ($statefile eq 'tcp') {
4476                 my $localip = "localhost";
4477                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4478                 my $nodename = PVE::INotify::nodename();
4479                 if ($datacenterconf->{migration_unsecure}) {
4480                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4481                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4482                 }
4483                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4484                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4485                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4486                 push @$cmd, '-incoming', $migrate_uri;
4487                 push @$cmd, '-S';
4488             } else {
4489                 push @$cmd, '-loadstate', $statefile;
4490             }
4491         } elsif ($paused) {
4492             push @$cmd, '-S';
4493         }
4494
4495         # host pci devices
4496         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4497           my $d = parse_hostpci($conf->{"hostpci$i"});
4498           next if !$d;
4499           my $pcidevices = $d->{pciid};
4500           foreach my $pcidevice (@$pcidevices) {
4501                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4502
4503                 my $info = pci_device_info("0000:$pciid");
4504                 die "IOMMU not present\n" if !check_iommu_support();
4505                 die "no pci device info for device '$pciid'\n" if !$info;
4506                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4507                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4508           }
4509         }
4510
4511         PVE::Storage::activate_volumes($storecfg, $vollist);
4512
4513         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4514                     umask => 0077); };
4515
4516         if (my $err = $@) {
4517             # deactivate volumes if start fails
4518             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4519             die "start failed: $err";
4520         }
4521
4522         print "migration listens on $migrate_uri\n" if $migrate_uri;
4523
4524         if ($statefile && $statefile ne 'tcp')  {
4525             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4526             warn $@ if $@;
4527         }
4528
4529         if ($migratedfrom) {
4530
4531             eval {
4532                 set_migration_caps($vmid);
4533             };
4534             warn $@ if $@;
4535
4536             if ($spice_port) {
4537                 print "spice listens on port $spice_port\n";
4538                 if ($spice_ticket) {
4539                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4540                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4541                 }
4542             }
4543
4544         } else {
4545
4546             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4547                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4548                     if $conf->{balloon};
4549             }
4550
4551             foreach my $opt (keys %$conf) {
4552                 next if $opt !~  m/^net\d+$/;
4553                 my $nicconf = parse_net($conf->{$opt});
4554                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4555             }
4556         }
4557
4558         vm_mon_cmd_nocheck($vmid, 'qom-set',
4559                     path => "machine/peripheral/balloon0",
4560                     property => "guest-stats-polling-interval",
4561                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4562
4563     });
4564 }
4565
4566 sub vm_mon_cmd {
4567     my ($vmid, $execute, %params) = @_;
4568
4569     my $cmd = { execute => $execute, arguments => \%params };
4570     vm_qmp_command($vmid, $cmd);
4571 }
4572
4573 sub vm_mon_cmd_nocheck {
4574     my ($vmid, $execute, %params) = @_;
4575
4576     my $cmd = { execute => $execute, arguments => \%params };
4577     vm_qmp_command($vmid, $cmd, 1);
4578 }
4579
4580 sub vm_qmp_command {
4581     my ($vmid, $cmd, $nocheck) = @_;
4582
4583     my $res;
4584
4585     my $timeout;
4586     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4587         $timeout = $cmd->{arguments}->{timeout};
4588         delete $cmd->{arguments}->{timeout};
4589     }
4590
4591     eval {
4592         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4593         my $sname = qmp_socket($vmid);
4594         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4595             my $qmpclient = PVE::QMPClient->new();
4596
4597             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4598         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4599             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4600                 if scalar(%{$cmd->{arguments}});
4601             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4602         } else {
4603             die "unable to open monitor socket\n";
4604         }
4605     };
4606     if (my $err = $@) {
4607         syslog("err", "VM $vmid qmp command failed - $err");
4608         die $err;
4609     }
4610
4611     return $res;
4612 }
4613
4614 sub vm_human_monitor_command {
4615     my ($vmid, $cmdline) = @_;
4616
4617     my $res;
4618
4619     my $cmd = {
4620         execute => 'human-monitor-command',
4621         arguments => { 'command-line' => $cmdline},
4622     };
4623
4624     return vm_qmp_command($vmid, $cmd);
4625 }
4626
4627 sub vm_commandline {
4628     my ($storecfg, $vmid) = @_;
4629
4630     my $conf = PVE::QemuConfig->load_config($vmid);
4631
4632     my $defaults = load_defaults();
4633
4634     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4635
4636     return join(' ', @$cmd);
4637 }
4638
4639 sub vm_reset {
4640     my ($vmid, $skiplock) = @_;
4641
4642     PVE::QemuConfig->lock_config($vmid, sub {
4643
4644         my $conf = PVE::QemuConfig->load_config($vmid);
4645
4646         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4647
4648         vm_mon_cmd($vmid, "system_reset");
4649     });
4650 }
4651
4652 sub get_vm_volumes {
4653     my ($conf) = @_;
4654
4655     my $vollist = [];
4656     foreach_volid($conf, sub {
4657         my ($volid, $is_cdrom) = @_;
4658
4659         return if $volid =~ m|^/|;
4660
4661         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4662         return if !$sid;
4663
4664         push @$vollist, $volid;
4665     });
4666
4667     return $vollist;
4668 }
4669
4670 sub vm_stop_cleanup {
4671     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4672
4673     eval {
4674
4675         if (!$keepActive) {
4676             my $vollist = get_vm_volumes($conf);
4677             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4678         }
4679
4680         foreach my $ext (qw(mon qmp pid vnc qga)) {
4681             unlink "/var/run/qemu-server/${vmid}.$ext";
4682         }
4683
4684         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4685     };
4686     warn $@ if $@; # avoid errors - just warn
4687 }
4688
4689 # Note: use $nockeck to skip tests if VM configuration file exists.
4690 # We need that when migration VMs to other nodes (files already moved)
4691 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4692 sub vm_stop {
4693     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4694
4695     $force = 1 if !defined($force) && !$shutdown;
4696
4697     if ($migratedfrom){
4698         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4699         kill 15, $pid if $pid;
4700         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4701         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4702         return;
4703     }
4704
4705     PVE::QemuConfig->lock_config($vmid, sub {
4706
4707         my $pid = check_running($vmid, $nocheck);
4708         return if !$pid;
4709
4710         my $conf;
4711         if (!$nocheck) {
4712             $conf = PVE::QemuConfig->load_config($vmid);
4713             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4714             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4715                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4716                 $timeout = $opts->{down} if $opts->{down};
4717             }
4718         }
4719
4720         $timeout = 60 if !defined($timeout);
4721
4722         eval {
4723             if ($shutdown) {
4724                 if (defined($conf) && $conf->{agent}) {
4725                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4726                 } else {
4727                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4728                 }
4729             } else {
4730                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4731             }
4732         };
4733         my $err = $@;
4734
4735         if (!$err) {
4736             my $count = 0;
4737             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4738                 $count++;
4739                 sleep 1;
4740             }
4741
4742             if ($count >= $timeout) {
4743                 if ($force) {
4744                     warn "VM still running - terminating now with SIGTERM\n";
4745                     kill 15, $pid;
4746                 } else {
4747                     die "VM quit/powerdown failed - got timeout\n";
4748                 }
4749             } else {
4750                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4751                 return;
4752             }
4753         } else {
4754             if ($force) {
4755                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4756                 kill 15, $pid;
4757             } else {
4758                 die "VM quit/powerdown failed\n";
4759             }
4760         }
4761
4762         # wait again
4763         $timeout = 10;
4764
4765         my $count = 0;
4766         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4767             $count++;
4768             sleep 1;
4769         }
4770
4771         if ($count >= $timeout) {
4772             warn "VM still running - terminating now with SIGKILL\n";
4773             kill 9, $pid;
4774             sleep 1;
4775         }
4776
4777         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4778    });
4779 }
4780
4781 sub vm_suspend {
4782     my ($vmid, $skiplock) = @_;
4783
4784     PVE::QemuConfig->lock_config($vmid, sub {
4785
4786         my $conf = PVE::QemuConfig->load_config($vmid);
4787
4788         PVE::QemuConfig->check_lock($conf)
4789             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4790
4791         vm_mon_cmd($vmid, "stop");
4792     });
4793 }
4794
4795 sub vm_resume {
4796     my ($vmid, $skiplock, $nocheck) = @_;
4797
4798     PVE::QemuConfig->lock_config($vmid, sub {
4799
4800         if (!$nocheck) {
4801
4802             my $conf = PVE::QemuConfig->load_config($vmid);
4803
4804             PVE::QemuConfig->check_lock($conf)
4805                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4806
4807             vm_mon_cmd($vmid, "cont");
4808
4809         } else {
4810             vm_mon_cmd_nocheck($vmid, "cont");
4811         }
4812     });
4813 }
4814
4815 sub vm_sendkey {
4816     my ($vmid, $skiplock, $key) = @_;
4817
4818     PVE::QemuConfig->lock_config($vmid, sub {
4819
4820         my $conf = PVE::QemuConfig->load_config($vmid);
4821
4822         # there is no qmp command, so we use the human monitor command
4823         vm_human_monitor_command($vmid, "sendkey $key");
4824     });
4825 }
4826
4827 sub vm_destroy {
4828     my ($storecfg, $vmid, $skiplock) = @_;
4829
4830     PVE::QemuConfig->lock_config($vmid, sub {
4831
4832         my $conf = PVE::QemuConfig->load_config($vmid);
4833
4834         if (!check_running($vmid)) {
4835             destroy_vm($storecfg, $vmid, undef, $skiplock);
4836         } else {
4837             die "VM $vmid is running - destroy failed\n";
4838         }
4839     });
4840 }
4841
4842 # pci helpers
4843
4844 sub file_write {
4845     my ($filename, $buf) = @_;
4846
4847     my $fh = IO::File->new($filename, "w");
4848     return undef if !$fh;
4849
4850     my $res = print $fh $buf;
4851
4852     $fh->close();
4853
4854     return $res;
4855 }
4856
4857 sub pci_device_info {
4858     my ($name) = @_;
4859
4860     my $res;
4861
4862     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4863     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4864
4865     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4866     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4867
4868     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4869     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4870
4871     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4872     return undef if !defined($product) || $product !~ s/^0x//;
4873
4874     $res = {
4875         name => $name,
4876         vendor => $vendor,
4877         product => $product,
4878         domain => $domain,
4879         bus => $bus,
4880         slot => $slot,
4881         func => $func,
4882         irq => $irq,
4883         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4884     };
4885
4886     return $res;
4887 }
4888
4889 sub pci_dev_reset {
4890     my ($dev) = @_;
4891
4892     my $name = $dev->{name};
4893
4894     my $fn = "$pcisysfs/devices/$name/reset";
4895
4896     return file_write($fn, "1");
4897 }
4898
4899 sub pci_dev_bind_to_vfio {
4900     my ($dev) = @_;
4901
4902     my $name = $dev->{name};
4903
4904     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4905
4906     if (!-d $vfio_basedir) {
4907         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4908     }
4909     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4910
4911     my $testdir = "$vfio_basedir/$name";
4912     return 1 if -d $testdir;
4913
4914     my $data = "$dev->{vendor} $dev->{product}";
4915     return undef if !file_write("$vfio_basedir/new_id", $data);
4916
4917     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4918     if (!file_write($fn, $name)) {
4919         return undef if -f $fn;
4920     }
4921
4922     $fn = "$vfio_basedir/bind";
4923     if (! -d $testdir) {
4924         return undef if !file_write($fn, $name);
4925     }
4926
4927     return -d $testdir;
4928 }
4929
4930 sub pci_dev_group_bind_to_vfio {
4931     my ($pciid) = @_;
4932
4933     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4934
4935     if (!-d $vfio_basedir) {
4936         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4937     }
4938     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4939
4940     # get IOMMU group devices
4941     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4942       my @devs = grep /^0000:/, readdir($D);
4943     closedir($D);
4944
4945     foreach my $pciid (@devs) {
4946         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4947
4948         # pci bridges, switches or root ports are not supported
4949         # they have a pci_bus subdirectory so skip them
4950         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4951
4952         my $info = pci_device_info($1);
4953         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4954     }
4955
4956     return 1;
4957 }
4958
4959 sub print_pci_addr {
4960     my ($id, $bridges) = @_;
4961
4962     my $res = '';
4963     my $devices = {
4964         piix3 => { bus => 0, addr => 1 },
4965         #addr2 : first videocard
4966         balloon0 => { bus => 0, addr => 3 },
4967         watchdog => { bus => 0, addr => 4 },
4968         scsihw0 => { bus => 0, addr => 5 },
4969         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4970         scsihw1 => { bus => 0, addr => 6 },
4971         ahci0 => { bus => 0, addr => 7 },
4972         qga0 => { bus => 0, addr => 8 },
4973         spice => { bus => 0, addr => 9 },
4974         virtio0 => { bus => 0, addr => 10 },
4975         virtio1 => { bus => 0, addr => 11 },
4976         virtio2 => { bus => 0, addr => 12 },
4977         virtio3 => { bus => 0, addr => 13 },
4978         virtio4 => { bus => 0, addr => 14 },
4979         virtio5 => { bus => 0, addr => 15 },
4980         hostpci0 => { bus => 0, addr => 16 },
4981         hostpci1 => { bus => 0, addr => 17 },
4982         net0 => { bus => 0, addr => 18 },
4983         net1 => { bus => 0, addr => 19 },
4984         net2 => { bus => 0, addr => 20 },
4985         net3 => { bus => 0, addr => 21 },
4986         net4 => { bus => 0, addr => 22 },
4987         net5 => { bus => 0, addr => 23 },
4988         vga1 => { bus => 0, addr => 24 },
4989         vga2 => { bus => 0, addr => 25 },
4990         vga3 => { bus => 0, addr => 26 },
4991         hostpci2 => { bus => 0, addr => 27 },
4992         hostpci3 => { bus => 0, addr => 28 },
4993         #addr29 : usb-host (pve-usb.cfg)
4994         'pci.1' => { bus => 0, addr => 30 },
4995         'pci.2' => { bus => 0, addr => 31 },
4996         'net6' => { bus => 1, addr => 1 },
4997         'net7' => { bus => 1, addr => 2 },
4998         'net8' => { bus => 1, addr => 3 },
4999         'net9' => { bus => 1, addr => 4 },
5000         'net10' => { bus => 1, addr => 5 },
5001         'net11' => { bus => 1, addr => 6 },
5002         'net12' => { bus => 1, addr => 7 },
5003         'net13' => { bus => 1, addr => 8 },
5004         'net14' => { bus => 1, addr => 9 },
5005         'net15' => { bus => 1, addr => 10 },
5006         'net16' => { bus => 1, addr => 11 },
5007         'net17' => { bus => 1, addr => 12 },
5008         'net18' => { bus => 1, addr => 13 },
5009         'net19' => { bus => 1, addr => 14 },
5010         'net20' => { bus => 1, addr => 15 },
5011         'net21' => { bus => 1, addr => 16 },
5012         'net22' => { bus => 1, addr => 17 },
5013         'net23' => { bus => 1, addr => 18 },
5014         'net24' => { bus => 1, addr => 19 },
5015         'net25' => { bus => 1, addr => 20 },
5016         'net26' => { bus => 1, addr => 21 },
5017         'net27' => { bus => 1, addr => 22 },
5018         'net28' => { bus => 1, addr => 23 },
5019         'net29' => { bus => 1, addr => 24 },
5020         'net30' => { bus => 1, addr => 25 },
5021         'net31' => { bus => 1, addr => 26 },
5022         'xhci' => { bus => 1, addr => 27 },
5023         'virtio6' => { bus => 2, addr => 1 },
5024         'virtio7' => { bus => 2, addr => 2 },
5025         'virtio8' => { bus => 2, addr => 3 },
5026         'virtio9' => { bus => 2, addr => 4 },
5027         'virtio10' => { bus => 2, addr => 5 },
5028         'virtio11' => { bus => 2, addr => 6 },
5029         'virtio12' => { bus => 2, addr => 7 },
5030         'virtio13' => { bus => 2, addr => 8 },
5031         'virtio14' => { bus => 2, addr => 9 },
5032         'virtio15' => { bus => 2, addr => 10 },
5033         'virtioscsi0' => { bus => 3, addr => 1 },
5034         'virtioscsi1' => { bus => 3, addr => 2 },
5035         'virtioscsi2' => { bus => 3, addr => 3 },
5036         'virtioscsi3' => { bus => 3, addr => 4 },
5037         'virtioscsi4' => { bus => 3, addr => 5 },
5038         'virtioscsi5' => { bus => 3, addr => 6 },
5039         'virtioscsi6' => { bus => 3, addr => 7 },
5040         'virtioscsi7' => { bus => 3, addr => 8 },
5041         'virtioscsi8' => { bus => 3, addr => 9 },
5042         'virtioscsi9' => { bus => 3, addr => 10 },
5043         'virtioscsi10' => { bus => 3, addr => 11 },
5044         'virtioscsi11' => { bus => 3, addr => 12 },
5045         'virtioscsi12' => { bus => 3, addr => 13 },
5046         'virtioscsi13' => { bus => 3, addr => 14 },
5047         'virtioscsi14' => { bus => 3, addr => 15 },
5048         'virtioscsi15' => { bus => 3, addr => 16 },
5049         'virtioscsi16' => { bus => 3, addr => 17 },
5050         'virtioscsi17' => { bus => 3, addr => 18 },
5051         'virtioscsi18' => { bus => 3, addr => 19 },
5052         'virtioscsi19' => { bus => 3, addr => 20 },
5053         'virtioscsi20' => { bus => 3, addr => 21 },
5054         'virtioscsi21' => { bus => 3, addr => 22 },
5055         'virtioscsi22' => { bus => 3, addr => 23 },
5056         'virtioscsi23' => { bus => 3, addr => 24 },
5057         'virtioscsi24' => { bus => 3, addr => 25 },
5058         'virtioscsi25' => { bus => 3, addr => 26 },
5059         'virtioscsi26' => { bus => 3, addr => 27 },
5060         'virtioscsi27' => { bus => 3, addr => 28 },
5061         'virtioscsi28' => { bus => 3, addr => 29 },
5062         'virtioscsi29' => { bus => 3, addr => 30 },
5063         'virtioscsi30' => { bus => 3, addr => 31 },
5064
5065     };
5066
5067     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5068            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5069            my $bus = $devices->{$id}->{bus};
5070            $res = ",bus=pci.$bus,addr=$addr";
5071            $bridges->{$bus} = 1 if $bridges;
5072     }
5073     return $res;
5074
5075 }
5076
5077 sub print_pcie_addr {
5078     my ($id) = @_;
5079
5080     my $res = '';
5081     my $devices = {
5082         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5083         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5084         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5085         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5086     };
5087
5088     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5089            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5090            my $bus = $devices->{$id}->{bus};
5091            $res = ",bus=$bus,addr=$addr";
5092     }
5093     return $res;
5094
5095 }
5096
5097 # vzdump restore implementaion
5098
5099 sub tar_archive_read_firstfile {
5100     my $archive = shift;
5101
5102     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5103
5104     # try to detect archive type first
5105     my $pid = open (TMP, "tar tf '$archive'|") ||
5106         die "unable to open file '$archive'\n";
5107     my $firstfile = <TMP>;
5108     kill 15, $pid;
5109     close TMP;
5110
5111     die "ERROR: archive contaions no data\n" if !$firstfile;
5112     chomp $firstfile;
5113
5114     return $firstfile;
5115 }
5116
5117 sub tar_restore_cleanup {
5118     my ($storecfg, $statfile) = @_;
5119
5120     print STDERR "starting cleanup\n";
5121
5122     if (my $fd = IO::File->new($statfile, "r")) {
5123         while (defined(my $line = <$fd>)) {
5124             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5125                 my $volid = $2;
5126                 eval {
5127                     if ($volid =~ m|^/|) {
5128                         unlink $volid || die 'unlink failed\n';
5129                     } else {
5130                         PVE::Storage::vdisk_free($storecfg, $volid);
5131                     }
5132                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5133                 };
5134                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5135             } else {
5136                 print STDERR "unable to parse line in statfile - $line";
5137             }
5138         }
5139         $fd->close();
5140     }
5141 }
5142
5143 sub restore_archive {
5144     my ($archive, $vmid, $user, $opts) = @_;
5145
5146     my $format = $opts->{format};
5147     my $comp;
5148
5149     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5150         $format = 'tar' if !$format;
5151         $comp = 'gzip';
5152     } elsif ($archive =~ m/\.tar$/) {
5153         $format = 'tar' if !$format;
5154     } elsif ($archive =~ m/.tar.lzo$/) {
5155         $format = 'tar' if !$format;
5156         $comp = 'lzop';
5157     } elsif ($archive =~ m/\.vma$/) {
5158         $format = 'vma' if !$format;
5159     } elsif ($archive =~ m/\.vma\.gz$/) {
5160         $format = 'vma' if !$format;
5161         $comp = 'gzip';
5162     } elsif ($archive =~ m/\.vma\.lzo$/) {
5163         $format = 'vma' if !$format;
5164         $comp = 'lzop';
5165     } else {
5166         $format = 'vma' if !$format; # default
5167     }
5168
5169     # try to detect archive format
5170     if ($format eq 'tar') {
5171         return restore_tar_archive($archive, $vmid, $user, $opts);
5172     } else {
5173         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5174     }
5175 }
5176
5177 sub restore_update_config_line {
5178     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5179
5180     return if $line =~ m/^\#qmdump\#/;
5181     return if $line =~ m/^\#vzdump\#/;
5182     return if $line =~ m/^lock:/;
5183     return if $line =~ m/^unused\d+:/;
5184     return if $line =~ m/^parent:/;
5185     return if $line =~ m/^template:/; # restored VM is never a template
5186
5187     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5188         # try to convert old 1.X settings
5189         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5190         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5191             my ($model, $macaddr) = split(/\=/, $devconfig);
5192             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5193             my $net = {
5194                 model => $model,
5195                 bridge => "vmbr$ind",
5196                 macaddr => $macaddr,
5197             };
5198             my $netstr = print_net($net);
5199
5200             print $outfd "net$cookie->{netcount}: $netstr\n";
5201             $cookie->{netcount}++;
5202         }
5203     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5204         my ($id, $netstr) = ($1, $2);
5205         my $net = parse_net($netstr);
5206         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5207         $netstr = print_net($net);
5208         print $outfd "$id: $netstr\n";
5209     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5210         my $virtdev = $1;
5211         my $value = $3;
5212         my $di = parse_drive($virtdev, $value);
5213         if (defined($di->{backup}) && !$di->{backup}) {
5214             print $outfd "#$line";
5215         } elsif ($map->{$virtdev}) {
5216             delete $di->{format}; # format can change on restore
5217             $di->{file} = $map->{$virtdev};
5218             $value = print_drive($vmid, $di);
5219             print $outfd "$virtdev: $value\n";
5220         } else {
5221             print $outfd $line;
5222         }
5223     } else {
5224         print $outfd $line;
5225     }
5226 }
5227
5228 sub scan_volids {
5229     my ($cfg, $vmid) = @_;
5230
5231     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5232
5233     my $volid_hash = {};
5234     foreach my $storeid (keys %$info) {
5235         foreach my $item (@{$info->{$storeid}}) {
5236             next if !($item->{volid} && $item->{size});
5237             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5238             $volid_hash->{$item->{volid}} = $item;
5239         }
5240     }
5241
5242     return $volid_hash;
5243 }
5244
5245 sub is_volume_in_use {
5246     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5247
5248     my $path = PVE::Storage::path($storecfg, $volid);
5249
5250     my $scan_config = sub {
5251         my ($cref, $snapname) = @_;
5252
5253         foreach my $key (keys %$cref) {
5254             my $value = $cref->{$key};
5255             if (is_valid_drivename($key)) {
5256                 next if $skip_drive && $key eq $skip_drive;
5257                 my $drive = parse_drive($key, $value);
5258                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5259                 return 1 if $volid eq $drive->{file};
5260                 if ($drive->{file} =~ m!^/!) {
5261                     return 1 if $drive->{file} eq $path;
5262                 } else {
5263                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5264                     next if !$storeid;
5265                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5266                     next if !$scfg;
5267                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5268                 }
5269             }
5270         }
5271
5272         return 0;
5273     };
5274
5275     return 1 if &$scan_config($conf);
5276
5277     undef $skip_drive;
5278
5279     foreach my $snapname (keys %{$conf->{snapshots}}) {
5280         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5281     }
5282
5283     return 0;
5284 }
5285
5286 sub update_disksize {
5287     my ($vmid, $conf, $volid_hash) = @_;
5288
5289     my $changes;
5290
5291     my $used = {};
5292
5293     # Note: it is allowed to define multiple storages with same path (alias), so
5294     # we need to check both 'volid' and real 'path' (two different volid can point
5295     # to the same path).
5296
5297     my $usedpath = {};
5298
5299     # update size info
5300     foreach my $opt (keys %$conf) {
5301         if (is_valid_drivename($opt)) {
5302             my $drive = parse_drive($opt, $conf->{$opt});
5303             my $volid = $drive->{file};
5304             next if !$volid;
5305
5306             $used->{$volid} = 1;
5307             if ($volid_hash->{$volid} &&
5308                 (my $path = $volid_hash->{$volid}->{path})) {
5309                 $usedpath->{$path} = 1;
5310             }
5311
5312             next if drive_is_cdrom($drive);
5313             next if !$volid_hash->{$volid};
5314
5315             $drive->{size} = $volid_hash->{$volid}->{size};
5316             my $new = print_drive($vmid, $drive);
5317             if ($new ne $conf->{$opt}) {
5318                 $changes = 1;
5319                 $conf->{$opt} = $new;
5320             }
5321         }
5322     }
5323
5324     # remove 'unusedX' entry if volume is used
5325     foreach my $opt (keys %$conf) {
5326         next if $opt !~ m/^unused\d+$/;
5327         my $volid = $conf->{$opt};
5328         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5329         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5330             $changes = 1;
5331             delete $conf->{$opt};
5332         }
5333     }
5334
5335     foreach my $volid (sort keys %$volid_hash) {
5336         next if $volid =~ m/vm-$vmid-state-/;
5337         next if $used->{$volid};
5338         my $path = $volid_hash->{$volid}->{path};
5339         next if !$path; # just to be sure
5340         next if $usedpath->{$path};
5341         $changes = 1;
5342         PVE::QemuConfig->add_unused_volume($conf, $volid);
5343         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5344     }
5345
5346     return $changes;
5347 }
5348
5349 sub rescan {
5350     my ($vmid, $nolock) = @_;
5351
5352     my $cfg = PVE::Storage::config();
5353
5354     my $volid_hash = scan_volids($cfg, $vmid);
5355
5356     my $updatefn =  sub {
5357         my ($vmid) = @_;
5358
5359         my $conf = PVE::QemuConfig->load_config($vmid);
5360
5361         PVE::QemuConfig->check_lock($conf);
5362
5363         my $vm_volids = {};
5364         foreach my $volid (keys %$volid_hash) {
5365             my $info = $volid_hash->{$volid};
5366             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5367         }
5368
5369         my $changes = update_disksize($vmid, $conf, $vm_volids);
5370
5371         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5372     };
5373
5374     if (defined($vmid)) {
5375         if ($nolock) {
5376             &$updatefn($vmid);
5377         } else {
5378             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5379         }
5380     } else {
5381         my $vmlist = config_list();
5382         foreach my $vmid (keys %$vmlist) {
5383             if ($nolock) {
5384                 &$updatefn($vmid);
5385             } else {
5386                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5387             }
5388         }
5389     }
5390 }
5391
5392 sub restore_vma_archive {
5393     my ($archive, $vmid, $user, $opts, $comp) = @_;
5394
5395     my $input = $archive eq '-' ? "<&STDIN" : undef;
5396     my $readfrom = $archive;
5397
5398     my $uncomp = '';
5399     if ($comp) {
5400         $readfrom = '-';
5401         my $qarchive = PVE::Tools::shellquote($archive);
5402         if ($comp eq 'gzip') {
5403             $uncomp = "zcat $qarchive|";
5404         } elsif ($comp eq 'lzop') {
5405             $uncomp = "lzop -d -c $qarchive|";
5406         } else {
5407             die "unknown compression method '$comp'\n";
5408         }
5409
5410     }
5411
5412     my $tmpdir = "/var/tmp/vzdumptmp$$";
5413     rmtree $tmpdir;
5414
5415     # disable interrupts (always do cleanups)
5416     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5417         warn "got interrupt - ignored\n";
5418     };
5419
5420     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5421     POSIX::mkfifo($mapfifo, 0600);
5422     my $fifofh;
5423
5424     my $openfifo = sub {
5425         open($fifofh, '>', $mapfifo) || die $!;
5426     };
5427
5428     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5429
5430     my $oldtimeout;
5431     my $timeout = 5;
5432
5433     my $devinfo = {};
5434
5435     my $rpcenv = PVE::RPCEnvironment::get();
5436
5437     my $conffile = PVE::QemuConfig->config_file($vmid);
5438     my $tmpfn = "$conffile.$$.tmp";
5439
5440     # Note: $oldconf is undef if VM does not exists
5441     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5442     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5443
5444     my $print_devmap = sub {
5445         my $virtdev_hash = {};
5446
5447         my $cfgfn = "$tmpdir/qemu-server.conf";
5448
5449         # we can read the config - that is already extracted
5450         my $fh = IO::File->new($cfgfn, "r") ||
5451             "unable to read qemu-server.conf - $!\n";
5452
5453         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5454         if (-f $fwcfgfn) {
5455             my $pve_firewall_dir = '/etc/pve/firewall';
5456             mkdir $pve_firewall_dir; # make sure the dir exists
5457             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5458         }
5459
5460         while (defined(my $line = <$fh>)) {
5461             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5462                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5463                 die "archive does not contain data for drive '$virtdev'\n"
5464                     if !$devinfo->{$devname};
5465                 if (defined($opts->{storage})) {
5466                     $storeid = $opts->{storage} || 'local';
5467                 } elsif (!$storeid) {
5468                     $storeid = 'local';
5469                 }
5470                 $format = 'raw' if !$format;
5471                 $devinfo->{$devname}->{devname} = $devname;
5472                 $devinfo->{$devname}->{virtdev} = $virtdev;
5473                 $devinfo->{$devname}->{format} = $format;
5474                 $devinfo->{$devname}->{storeid} = $storeid;
5475
5476                 # check permission on storage
5477                 my $pool = $opts->{pool}; # todo: do we need that?
5478                 if ($user ne 'root@pam') {
5479                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5480                 }
5481
5482                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5483             }
5484         }
5485
5486         foreach my $devname (keys %$devinfo) {
5487             die "found no device mapping information for device '$devname'\n"
5488                 if !$devinfo->{$devname}->{virtdev};
5489         }
5490
5491         my $cfg = PVE::Storage::config();
5492
5493         # create empty/temp config
5494         if ($oldconf) {
5495             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5496             foreach_drive($oldconf, sub {
5497                 my ($ds, $drive) = @_;
5498
5499                 return if drive_is_cdrom($drive);
5500
5501                 my $volid = $drive->{file};
5502
5503                 return if !$volid || $volid =~ m|^/|;
5504
5505                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5506                 return if !$path || !$owner || ($owner != $vmid);
5507
5508                 # Note: only delete disk we want to restore
5509                 # other volumes will become unused
5510                 if ($virtdev_hash->{$ds}) {
5511                     PVE::Storage::vdisk_free($cfg, $volid);
5512                 }
5513             });
5514
5515             # delete vmstate files
5516             # since after the restore we have no snapshots anymore
5517             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5518                 my $snap = $oldconf->{snapshots}->{$snapname};
5519                 if ($snap->{vmstate}) {
5520                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5521                     if (my $err = $@) {
5522                         warn $err;
5523                     }
5524                 }
5525             }
5526         }
5527
5528         my $map = {};
5529         foreach my $virtdev (sort keys %$virtdev_hash) {
5530             my $d = $virtdev_hash->{$virtdev};
5531             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5532             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5533
5534             # test if requested format is supported
5535             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5536             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5537             $d->{format} = $defFormat if !$supported;
5538
5539             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5540                                                   $d->{format}, undef, $alloc_size);
5541             print STDERR "new volume ID is '$volid'\n";
5542             $d->{volid} = $volid;
5543             my $path = PVE::Storage::path($cfg, $volid);
5544
5545             PVE::Storage::activate_volumes($cfg,[$volid]);
5546
5547             my $write_zeros = 1;
5548             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5549                 $write_zeros = 0;
5550             }
5551
5552             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5553
5554             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5555             $map->{$virtdev} = $volid;
5556         }
5557
5558         $fh->seek(0, 0) || die "seek failed - $!\n";
5559
5560         my $outfd = new IO::File ($tmpfn, "w") ||
5561             die "unable to write config for VM $vmid\n";
5562
5563         my $cookie = { netcount => 0 };
5564         while (defined(my $line = <$fh>)) {
5565             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5566         }
5567
5568         $fh->close();
5569         $outfd->close();
5570     };
5571
5572     eval {
5573         # enable interrupts
5574         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5575             die "interrupted by signal\n";
5576         };
5577         local $SIG{ALRM} = sub { die "got timeout\n"; };
5578
5579         $oldtimeout = alarm($timeout);
5580
5581         my $parser = sub {
5582             my $line = shift;
5583
5584             print "$line\n";
5585
5586             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5587                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5588                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5589             } elsif ($line =~ m/^CTIME: /) {
5590                 # we correctly received the vma config, so we can disable
5591                 # the timeout now for disk allocation (set to 10 minutes, so
5592                 # that we always timeout if something goes wrong)
5593                 alarm(600);
5594                 &$print_devmap();
5595                 print $fifofh "done\n";
5596                 my $tmp = $oldtimeout || 0;
5597                 $oldtimeout = undef;
5598                 alarm($tmp);
5599                 close($fifofh);
5600             }
5601         };
5602
5603         print "restore vma archive: $cmd\n";
5604         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5605     };
5606     my $err = $@;
5607
5608     alarm($oldtimeout) if $oldtimeout;
5609
5610     my $vollist = [];
5611     foreach my $devname (keys %$devinfo) {
5612         my $volid = $devinfo->{$devname}->{volid};
5613         push @$vollist, $volid if $volid;
5614     }
5615
5616     my $cfg = PVE::Storage::config();
5617     PVE::Storage::deactivate_volumes($cfg, $vollist);
5618
5619     unlink $mapfifo;
5620
5621     if ($err) {
5622         rmtree $tmpdir;
5623         unlink $tmpfn;
5624
5625         foreach my $devname (keys %$devinfo) {
5626             my $volid = $devinfo->{$devname}->{volid};
5627             next if !$volid;
5628             eval {
5629                 if ($volid =~ m|^/|) {
5630                     unlink $volid || die 'unlink failed\n';
5631                 } else {
5632                     PVE::Storage::vdisk_free($cfg, $volid);
5633                 }
5634                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5635             };
5636             print STDERR "unable to cleanup '$volid' - $@" if $@;
5637         }
5638         die $err;
5639     }
5640
5641     rmtree $tmpdir;
5642
5643     rename($tmpfn, $conffile) ||
5644         die "unable to commit configuration file '$conffile'\n";
5645
5646     PVE::Cluster::cfs_update(); # make sure we read new file
5647
5648     eval { rescan($vmid, 1); };
5649     warn $@ if $@;
5650 }
5651
5652 sub restore_tar_archive {
5653     my ($archive, $vmid, $user, $opts) = @_;
5654
5655     if ($archive ne '-') {
5656         my $firstfile = tar_archive_read_firstfile($archive);
5657         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5658             if $firstfile ne 'qemu-server.conf';
5659     }
5660
5661     my $storecfg = PVE::Storage::config();
5662
5663     # destroy existing data - keep empty config
5664     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5665     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5666
5667     my $tocmd = "/usr/lib/qemu-server/qmextract";
5668
5669     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5670     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5671     $tocmd .= ' --prealloc' if $opts->{prealloc};
5672     $tocmd .= ' --info' if $opts->{info};
5673
5674     # tar option "xf" does not autodetect compression when read from STDIN,
5675     # so we pipe to zcat
5676     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5677         PVE::Tools::shellquote("--to-command=$tocmd");
5678
5679     my $tmpdir = "/var/tmp/vzdumptmp$$";
5680     mkpath $tmpdir;
5681
5682     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5683     local $ENV{VZDUMP_VMID} = $vmid;
5684     local $ENV{VZDUMP_USER} = $user;
5685
5686     my $conffile = PVE::QemuConfig->config_file($vmid);
5687     my $tmpfn = "$conffile.$$.tmp";
5688
5689     # disable interrupts (always do cleanups)
5690     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5691         print STDERR "got interrupt - ignored\n";
5692     };
5693
5694     eval {
5695         # enable interrupts
5696         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5697             die "interrupted by signal\n";
5698         };
5699
5700         if ($archive eq '-') {
5701             print "extracting archive from STDIN\n";
5702             run_command($cmd, input => "<&STDIN");
5703         } else {
5704             print "extracting archive '$archive'\n";
5705             run_command($cmd);
5706         }
5707
5708         return if $opts->{info};
5709
5710         # read new mapping
5711         my $map = {};
5712         my $statfile = "$tmpdir/qmrestore.stat";
5713         if (my $fd = IO::File->new($statfile, "r")) {
5714             while (defined (my $line = <$fd>)) {
5715                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5716                     $map->{$1} = $2 if $1;
5717                 } else {
5718                     print STDERR "unable to parse line in statfile - $line\n";
5719                 }
5720             }
5721             $fd->close();
5722         }
5723
5724         my $confsrc = "$tmpdir/qemu-server.conf";
5725
5726         my $srcfd = new IO::File($confsrc, "r") ||
5727             die "unable to open file '$confsrc'\n";
5728
5729         my $outfd = new IO::File ($tmpfn, "w") ||
5730             die "unable to write config for VM $vmid\n";
5731
5732         my $cookie = { netcount => 0 };
5733         while (defined (my $line = <$srcfd>)) {
5734             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5735         }
5736
5737         $srcfd->close();
5738         $outfd->close();
5739     };
5740     my $err = $@;
5741
5742     if ($err) {
5743
5744         unlink $tmpfn;
5745
5746         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5747
5748         die $err;
5749     }
5750
5751     rmtree $tmpdir;
5752
5753     rename $tmpfn, $conffile ||
5754         die "unable to commit configuration file '$conffile'\n";
5755
5756     PVE::Cluster::cfs_update(); # make sure we read new file
5757
5758     eval { rescan($vmid, 1); };
5759     warn $@ if $@;
5760 };
5761
5762 sub foreach_writable_storage {
5763     my ($conf, $func) = @_;
5764
5765     my $sidhash = {};
5766
5767     foreach my $ds (keys %$conf) {
5768         next if !is_valid_drivename($ds);
5769
5770         my $drive = parse_drive($ds, $conf->{$ds});
5771         next if !$drive;
5772         next if drive_is_cdrom($drive);
5773
5774         my $volid = $drive->{file};
5775
5776         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5777         $sidhash->{$sid} = $sid if $sid;
5778     }
5779
5780     foreach my $sid (sort keys %$sidhash) {
5781         &$func($sid);
5782     }
5783 }
5784
5785 sub do_snapshots_with_qemu {
5786     my ($storecfg, $volid) = @_;
5787
5788     my $storage_name = PVE::Storage::parse_volume_id($volid);
5789
5790     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5791         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5792         return 1;
5793     }
5794
5795     if ($volid =~ m/\.(qcow2|qed)$/){
5796         return 1;
5797     }
5798
5799     return undef;
5800 }
5801
5802 sub qga_check_running {
5803     my ($vmid) = @_;
5804
5805     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5806     if ($@) {
5807         warn "Qemu Guest Agent are not running - $@";
5808         return 0;
5809     }
5810     return 1;
5811 }
5812
5813 sub template_create {
5814     my ($vmid, $conf, $disk) = @_;
5815
5816     my $storecfg = PVE::Storage::config();
5817
5818     foreach_drive($conf, sub {
5819         my ($ds, $drive) = @_;
5820
5821         return if drive_is_cdrom($drive);
5822         return if $disk && $ds ne $disk;
5823
5824         my $volid = $drive->{file};
5825         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5826
5827         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5828         $drive->{file} = $voliddst;
5829         $conf->{$ds} = print_drive($vmid, $drive);
5830         PVE::QemuConfig->write_config($vmid, $conf);
5831     });
5832 }
5833
5834 sub qemu_img_convert {
5835     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5836
5837     my $storecfg = PVE::Storage::config();
5838     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5839     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5840
5841     if ($src_storeid && $dst_storeid) {
5842
5843         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5844
5845         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5846         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5847
5848         my $src_format = qemu_img_format($src_scfg, $src_volname);
5849         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5850
5851         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5852         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5853
5854         my $cmd = [];
5855         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5856         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5857         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5858         if ($is_zero_initialized) {
5859             push @$cmd, "zeroinit:$dst_path";
5860         } else {
5861             push @$cmd, $dst_path;
5862         }
5863
5864         my $parser = sub {
5865             my $line = shift;
5866             if($line =~ m/\((\S+)\/100\%\)/){
5867                 my $percent = $1;
5868                 my $transferred = int($size * $percent / 100);
5869                 my $remaining = $size - $transferred;
5870
5871                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5872             }
5873
5874         };
5875
5876         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5877         my $err = $@;
5878         die "copy failed: $err" if $err;
5879     }
5880 }
5881
5882 sub qemu_img_format {
5883     my ($scfg, $volname) = @_;
5884
5885     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5886         return $1;
5887     } else {
5888         return "raw";
5889     }
5890 }
5891
5892 sub qemu_drive_mirror {
5893     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5894
5895     my $storecfg = PVE::Storage::config();
5896     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5897
5898     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5899
5900     my $format = qemu_img_format($dst_scfg, $dst_volname);
5901
5902     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5903
5904     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5905
5906     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5907     $opts->{format} = $format if $format;
5908
5909     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5910
5911     eval {
5912     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5913         while (1) {
5914             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5915             my $stat = @$stats[0];
5916             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5917             die "error job is not mirroring" if $stat->{type} ne "mirror";
5918
5919             my $busy = $stat->{busy};
5920             my $ready = $stat->{ready};
5921
5922             if (my $total = $stat->{len}) {
5923                 my $transferred = $stat->{offset} || 0;
5924                 my $remaining = $total - $transferred;
5925                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5926
5927                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5928             }
5929
5930
5931             if ($stat->{ready} eq 'true') {
5932
5933                 last if $vmiddst != $vmid;
5934
5935                 # try to switch the disk if source and destination are on the same guest
5936                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5937                 last if !$@;
5938                 die $@ if $@ !~ m/cannot be completed/;
5939             }
5940             sleep 1;
5941         }
5942
5943
5944     };
5945     my $err = $@;
5946
5947     my $cancel_job = sub {
5948         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5949         while (1) {
5950             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5951             my $stat = @$stats[0];
5952             last if !$stat;
5953             sleep 1;
5954         }
5955     };
5956
5957     if ($err) {
5958         eval { &$cancel_job(); };
5959         die "mirroring error: $err";
5960     }
5961
5962     if ($vmiddst != $vmid) {
5963         # if we clone a disk for a new target vm, we don't switch the disk
5964         &$cancel_job(); # so we call block-job-cancel
5965     }
5966 }
5967
5968 sub clone_disk {
5969     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5970         $newvmid, $storage, $format, $full, $newvollist) = @_;
5971
5972     my $newvolid;
5973
5974     if (!$full) {
5975         print "create linked clone of drive $drivename ($drive->{file})\n";
5976         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5977         push @$newvollist, $newvolid;
5978     } else {
5979         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5980         $storeid = $storage if $storage;
5981
5982         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5983         if (!$format) {
5984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5985             $format = qemu_img_format($scfg, $volname);
5986         }
5987
5988         # test if requested format is supported - else use default
5989         my $supported = grep { $_ eq $format } @$validFormats;
5990         $format = $defFormat if !$supported;
5991
5992         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5993
5994         print "create full clone of drive $drivename ($drive->{file})\n";
5995         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5996         push @$newvollist, $newvolid;
5997
5998         PVE::Storage::activate_volumes($storecfg, $newvollist);
5999
6000         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6001         if (!$running || $snapname) {
6002             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6003         } else {
6004             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6005         }
6006     }
6007
6008     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6009
6010     my $disk = $drive;
6011     $disk->{format} = undef;
6012     $disk->{file} = $newvolid;
6013     $disk->{size} = $size;
6014
6015     return $disk;
6016 }
6017
6018 # this only works if VM is running
6019 sub get_current_qemu_machine {
6020     my ($vmid) = @_;
6021
6022     my $cmd = { execute => 'query-machines', arguments => {} };
6023     my $res = vm_qmp_command($vmid, $cmd);
6024
6025     my ($current, $default);
6026     foreach my $e (@$res) {
6027         $default = $e->{name} if $e->{'is-default'};
6028         $current = $e->{name} if $e->{'is-current'};
6029     }
6030
6031     # fallback to the default machine if current is not supported by qemu
6032     return $current || $default || 'pc';
6033 }
6034
6035 sub qemu_machine_feature_enabled {
6036     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6037
6038     my $current_major;
6039     my $current_minor;
6040
6041     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6042
6043         $current_major = $3;
6044         $current_minor = $4;
6045
6046     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6047
6048         $current_major = $1;
6049         $current_minor = $2;
6050     }
6051
6052     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6053
6054
6055 }
6056
6057 sub qemu_machine_pxe {
6058     my ($vmid, $conf, $machine) = @_;
6059
6060     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6061
6062     foreach my $opt (keys %$conf) {
6063         next if $opt !~ m/^net(\d+)$/;
6064         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6065         next if !$net;
6066         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6067         return $machine.".pxe" if $romfile =~ m/pxe/;
6068         last;
6069     }
6070
6071     return $machine;
6072 }
6073
6074 sub qemu_use_old_bios_files {
6075     my ($machine_type) = @_;
6076
6077     return if !$machine_type;
6078
6079     my $use_old_bios_files = undef;
6080
6081     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6082         $machine_type = $1;
6083         $use_old_bios_files = 1;
6084     } else {
6085         my $kvmver = kvm_user_version();
6086         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6087         # load new efi bios files on migration. So this hack is required to allow
6088         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6089         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6090         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6091     }
6092
6093     return ($use_old_bios_files, $machine_type);
6094 }
6095
6096 sub lspci {
6097
6098     my $devices = {};
6099
6100     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6101             my (undef, $id, $function) = @_;
6102             my $res = { id => $id, function => $function};
6103             push @{$devices->{$id}}, $res;
6104     });
6105
6106     return $devices;
6107 }
6108
6109 sub vm_iothreads_list {
6110     my ($vmid) = @_;
6111
6112     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6113
6114     my $iothreads = {};
6115     foreach my $iothread (@$res) {
6116         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6117     }
6118
6119     return $iothreads;
6120 }
6121
6122 sub scsihw_infos {
6123     my ($conf, $drive) = @_;
6124
6125     my $maxdev = 0;
6126
6127     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6128         $maxdev = 7;
6129     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6130         $maxdev = 1;
6131     } else {
6132         $maxdev = 256;
6133     }
6134
6135     my $controller = int($drive->{index} / $maxdev);
6136     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6137
6138     return ($maxdev, $controller, $controller_prefix);
6139 }
6140
6141 # bash completion helper
6142
6143 sub complete_backup_archives {
6144     my ($cmdname, $pname, $cvalue) = @_;
6145
6146     my $cfg = PVE::Storage::config();
6147
6148     my $storeid;
6149
6150     if ($cvalue =~ m/^([^:]+):/) {
6151         $storeid = $1;
6152     }
6153
6154     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6155
6156     my $res = [];
6157     foreach my $id (keys %$data) {
6158         foreach my $item (@{$data->{$id}}) {
6159             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6160             push @$res, $item->{volid} if defined($item->{volid});
6161         }
6162     }
6163
6164     return $res;
6165 }
6166
6167 my $complete_vmid_full = sub {
6168     my ($running) = @_;
6169
6170     my $idlist = vmstatus();
6171
6172     my $res = [];
6173
6174     foreach my $id (keys %$idlist) {
6175         my $d = $idlist->{$id};
6176         if (defined($running)) {
6177             next if $d->{template};
6178             next if $running && $d->{status} ne 'running';
6179             next if !$running && $d->{status} eq 'running';
6180         }
6181         push @$res, $id;
6182
6183     }
6184     return $res;
6185 };
6186
6187 sub complete_vmid {
6188     return &$complete_vmid_full();
6189 }
6190
6191 sub complete_vmid_stopped {
6192     return &$complete_vmid_full(0);
6193 }
6194
6195 sub complete_vmid_running {
6196     return &$complete_vmid_full(1);
6197 }
6198
6199 sub complete_storage {
6200
6201     my $cfg = PVE::Storage::config();
6202     my $ids = $cfg->{ids};
6203
6204     my $res = [];
6205     foreach my $sid (keys %$ids) {
6206         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6207         next if !$ids->{$sid}->{content}->{images};
6208         push @$res, $sid;
6209     }
6210
6211     return $res;
6212 }
6213
6214 1;