PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $confdesc = {
 113     onboot => {
 114         optional => 1,
 115         type => 'boolean',
 116         description => "Specifies whether a VM will be started during system bootup.",
 117         default => 0,
 118     },
 119     autostart => {
 120         optional => 1,
 121         type => 'boolean',
 122         description => "Automatic restart after crash (currently ignored).",
 123         default => 0,
 124     },
 125     hotplug => {
 126         optional => 1,
 127         type => 'string', format => 'pve-hotplug-features',
 128         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 129         default => 'network,disk,usb',
 130     },
 131     reboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 135         default => 1,
 136     },
 137     lock => {
 138         optional => 1,
 139         type => 'string',
 140         description => "Lock/unlock the VM.",
 141         enum => [qw(migrate backup snapshot rollback)],
 142     },
 143     cpulimit => {
 144         optional => 1,
 145         type => 'number',
 146         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 147         minimum => 0,
 148         maximum => 128,
 149         default => 0,
 150     },
 151     cpuunits => {
 152         optional => 1,
 153         type => 'integer',
 154         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 155         minimum => 0,
 156         maximum => 500000,
 157         default => 1000,
 158     },
 159     memory => {
 160         optional => 1,
 161         type => 'integer',
 162         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 163         minimum => 16,
 164         default => 512,
 165     },
 166     balloon => {
 167         optional => 1,
 168         type => 'integer',
 169         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 170         minimum => 0,
 171     },
 172     shares => {
 173         optional => 1,
 174         type => 'integer',
 175         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 176         minimum => 0,
 177         maximum => 50000,
 178         default => 1000,
 179     },
 180     keyboard => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 184         enum => PVE::Tools::kvmkeymaplist(),
 185         default => 'en-us',
 186     },
 187     name => {
 188         optional => 1,
 189         type => 'string', format => 'dns-name',
 190         description => "Set a name for the VM. Only used on the configuration web interface.",
 191     },
 192     scsihw => {
 193         optional => 1,
 194         type => 'string',
 195         description => "scsi controller model",
 196         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 197         default => 'lsi',
 198     },
 199     description => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 203     },
 204     ostype => {
 205         optional => 1,
 206         type => 'string',
 207         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 208         description => <<EODESC,
 209 Used to enable special optimization/features for specific
 210 operating systems:
 211
 212 other  => unspecified OS
 213 wxp    => Microsoft Windows XP
 214 w2k    => Microsoft Windows 2000
 215 w2k3   => Microsoft Windows 2003
 216 w2k8   => Microsoft Windows 2008
 217 wvista => Microsoft Windows Vista
 218 win7   => Microsoft Windows 7
 219 win8   => Microsoft Windows 8/2012
 220 l24    => Linux 2.4 Kernel
 221 l26    => Linux 2.6/3.X Kernel
 222 solaris => solaris/opensolaris/openindiania kernel
 223
 224 other|l24|l26|solaris                       ... no special behaviour
 225 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 226 EODESC
 227     },
 228     boot => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 232         pattern => '[acdn]{1,4}',
 233         default => 'cdn',
 234     },
 235     bootdisk => {
 236         optional => 1,
 237         type => 'string', format => 'pve-qm-bootdisk',
 238         description => "Enable booting from specified disk.",
 239         pattern => '(ide|sata|scsi|virtio)\d+',
 240     },
 241     smp => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "The number of CPUs. Please use option -sockets instead.",
 245         minimum => 1,
 246         default => 1,
 247     },
 248     sockets => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "The number of CPU sockets.",
 252         minimum => 1,
 253         default => 1,
 254     },
 255     cores => {
 256         optional => 1,
 257         type => 'integer',
 258         description => "The number of cores per socket.",
 259         minimum => 1,
 260         default => 1,
 261     },
 262     numa => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable NUMA.",
 266         default => 0,
 267     },
 268     vcpus => {
 269         optional => 1,
 270         type => 'integer',
 271         description => "Number of hotplugged vcpus.",
 272         minimum => 1,
 273         default => 0,
 274     },
 275     acpi => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Enable/disable ACPI.",
 279         default => 1,
 280     },
 281     agent => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Enable/disable Qemu GuestAgent.",
 285         default => 0,
 286     },
 287     kvm => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Enable/disable KVM hardware virtualization.",
 291         default => 1,
 292     },
 293     tdf => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable time drift fix.",
 297         default => 0,
 298     },
 299     localtime => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 303     },
 304     freeze => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 308     },
 309     vga => {
 310         optional => 1,
 311         type => 'string',
 312         description => "Select the VGA type. If you want to use high resolution" .
 313             " modes (>= 1280x1024x16) then you should use the options " .
 314             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 315             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 316             "display sever. For win* OS you can select how many independent " .
 317             "displays you want, Linux guests can add displays them self. " .
 318             "You can also run without any graphic card, using a serial device" .
 319             " as terminal.",
 320         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 321     },
 322     watchdog => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-watchdog',
 325         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 326         description => "Create a virtual hardware watchdog device. Once enabled" .
 327             " (by a guest action), the watchdog must be periodically polled " .
 328             "by an agent inside the guest or else the watchdog will reset " .
 329             "the guest (or execute the respective action specified)",
 330     },
 331     startdate => {
 332         optional => 1,
 333         type => 'string',
 334         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 335         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 336         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 337         default => 'now',
 338     },
 339     startup =>  get_standard_option('pve-startup-order'),
 340     template => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Enable/disable Template.",
 344         default => 0,
 345     },
 346     args => {
 347         optional => 1,
 348         type => 'string',
 349         description => <<EODESCR,
 350 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 351
 352 args: -no-reboot -no-hpet
 353 EODESCR
 354     },
 355     tablet => {
 356         optional => 1,
 357         type => 'boolean',
 358         default => 1,
 359         description => "Enable/disable the USB tablet device. This device is " .
 360             "usually needed to allow absolute mouse positioning with VNC. " .
 361             "Else the mouse runs out of sync with normal VNC clients. " .
 362             "If you're running lots of console-only guests on one host, " .
 363             "you may consider disabling this to save some context switches. " .
 364             "This is turned off by default if you use spice (-vga=qxl).",
 365     },
 366     migrate_speed => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 370         minimum => 0,
 371         default => 0,
 372     },
 373     migrate_downtime => {
 374         optional => 1,
 375         type => 'number',
 376         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 377         minimum => 0,
 378         default => 0.1,
 379     },
 380     cdrom => {
 381         optional => 1,
 382         type => 'string', format => 'pve-qm-drive',
 383         typetext => 'volume',
 384         description => "This is an alias for option -ide2",
 385     },
 386     cpu => {
 387         optional => 1,
 388         description => "Emulated CPU type.",
 389         type => 'string',
 390         format => $cpu_fmt,
 391     },
 392     parent => get_standard_option('pve-snapshot-name', {
 393         optional => 1,
 394         description => "Parent snapshot name. This is used internally, and should not be modified.",
 395     }),
 396     snaptime => {
 397         optional => 1,
 398         description => "Timestamp for snapshots.",
 399         type => 'integer',
 400         minimum => 0,
 401     },
 402     vmstate => {
 403         optional => 1,
 404         type => 'string', format => 'pve-volume-id',
 405         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 406     },
 407     machine => {
 408         description => "Specific the Qemu machine type.",
 409         type => 'string',
 410         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 411         maxLength => 40,
 412         optional => 1,
 413     },
 414     smbios1 => {
 415         description => "Specify SMBIOS type 1 fields.",
 416         type => 'string', format => 'pve-qm-smbios1',
 417         maxLength => 256,
 418         optional => 1,
 419     },
 420     protection => {
 421         optional => 1,
 422         type => 'boolean',
 423         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 424         default => 0,
 425     },
 426     bios => {
 427         optional => 1,
 428         type => 'string',
 429         enum => [ qw(seabios ovmf) ],
 430         description => "Select BIOS implementation.",
 431         default => 'seabios',
 432     },
 433 };
 434
 435 # what about other qemu settings ?
 436 #cpu => 'string',
 437 #machine => 'string',
 438 #fda => 'file',
 439 #fdb => 'file',
 440 #mtdblock => 'file',
 441 #sd => 'file',
 442 #pflash => 'file',
 443 #snapshot => 'bool',
 444 #bootp => 'file',
 445 ##tftp => 'dir',
 446 ##smb => 'dir',
 447 #kernel => 'file',
 448 #append => 'string',
 449 #initrd => 'file',
 450 ##soundhw => 'string',
 451
 452 while (my ($k, $v) = each %$confdesc) {
 453     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 454 }
 455
 456 my $MAX_IDE_DISKS = 4;
 457 my $MAX_SCSI_DISKS = 14;
 458 my $MAX_VIRTIO_DISKS = 16;
 459 my $MAX_SATA_DISKS = 6;
 460 my $MAX_USB_DEVICES = 5;
 461 my $MAX_NETS = 32;
 462 my $MAX_UNUSED_DISKS = 8;
 463 my $MAX_HOSTPCI_DEVICES = 4;
 464 my $MAX_SERIAL_PORTS = 4;
 465 my $MAX_PARALLEL_PORTS = 3;
 466 my $MAX_NUMA = 8;
 467 my $MAX_MEM = 4194304;
 468 my $STATICMEM = 1024;
 469
 470 my $numadesc = {
 471     optional => 1,
 472     type => 'string', format => 'pve-qm-numanode',
 473     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 474     description => "numa topology",
 475 };
 476 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 477
 478 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 479     $confdesc->{"numa$i"} = $numadesc;
 480 }
 481
 482 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 483                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 484                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 485 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 486
 487 my $netdesc = {
 488     optional => 1,
 489     type => 'string', format => 'pve-qm-net',
 490     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 491     description => <<EODESCR,
 492 Specify network devices.
 493
 494 MODEL is one of: $nic_model_list_txt
 495
 496 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 497 automatically generated if not specified.
 498
 499 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 500
 501 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 502
 503 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 504
 505 10.0.2.2   Gateway
 506 10.0.2.3   DNS Server
 507 10.0.2.4   SMB Server
 508
 509 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 510
 511 EODESCR
 512 };
 513 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 514
 515 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 516     $confdesc->{"net$i"} = $netdesc;
 517 }
 518
 519 PVE::JSONSchema::register_format('pve-volume-id-or-none', \&verify_volume_id_or_none);
 520 sub verify_volume_id_or_none {
 521     my ($volid, $noerr) = @_;
 522
 523     return $volid if $volid eq 'none';
 524     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 525     if ($@) {
 526         return undef if $noerr;
 527         die $@;
 528     }
 529     return $volid;
 530 }
 531
 532 my $drivename_hash;
 533
 534 my %drivedesc_base = (
 535     volume => { alias => 'file' },
 536     file => {
 537         type => 'string',
 538         format => 'pve-volume-id-or-none',
 539         default_key => 1,
 540         format_description => 'volume',
 541         description => "The drive's backing volume.",
 542     },
 543     media => {
 544         type => 'string',
 545         format_description => 'cdrom|disk',
 546         enum => [qw(cdrom disk)],
 547         description => "The drive's media type.",
 548         default => 'disk',
 549         optional => 1
 550     },
 551     cyls => {
 552         type => 'integer',
 553         format_description => 'count',
 554         description => "Force the drive's physical geometry to have a specific cylinder count.",
 555         optional => 1
 556     },
 557     heads => {
 558         type => 'integer',
 559         format_description => 'count',
 560         description => "Force the drive's physical geometry to have a specific head count.",
 561         optional => 1
 562     },
 563     secs => {
 564         type => 'integer',
 565         format_description => 'count',
 566         description => "Force the drive's physical geometry to have a specific sector count.",
 567         optional => 1
 568     },
 569     trans => {
 570         type => 'string',
 571         format_description => 'none|lba|auto',
 572         enum => [qw(none lba auto)],
 573         description => "Force disk geometry bios translation mode.",
 574         optional => 1,
 575     },
 576     snapshot => {
 577         type => 'boolean',
 578         format_description => 'on|off',
 579         description => "Whether the drive should be included when making snapshots.",
 580         optional => 1,
 581     },
 582     cache => {
 583         type => 'string',
 584         format_description => 'none|writethrough|writeback|unsafe|directsync',
 585         enum => [qw(none writethrough writeback unsafe directsync)],
 586         description => "The drive's cache mode",
 587         optional => 1,
 588     },
 589     format => {
 590         type => 'string',
 591         format_description => 'drive format',
 592         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 593         description => "The drive's backing file's data format.",
 594         optional => 1,
 595     },
 596     size => {
 597         type => 'string',
 598         format => 'disk-size',
 599         description => "Disk size. This is purely informational and has no effect.",
 600         optional => 1,
 601     },
 602     backup => {
 603         type => 'boolean',
 604         format_description => 'on|off',
 605         description => "Whether the drive should be included when making backups.",
 606         optional => 1,
 607     },
 608     werror => {
 609         type => 'string',
 610         format_description => 'enospc|ignore|report|stop',
 611         enum => [qw(enospc ignore report stop)],
 612         description => 'Write error action.',
 613         optional => 1,
 614     },
 615     aio => {
 616         type => 'string',
 617         format_description => 'native|threads',
 618         enum => [qw(native threads)],
 619         description => 'AIO type to use.',
 620         optional => 1,
 621     },
 622     discard => {
 623         type => 'string',
 624         format_description => 'ignore|on',
 625         enum => [qw(ignore on)],
 626         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 627         optional => 1,
 628     },
 629     detect_zeroes => {
 630         type => 'boolean',
 631         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 632         optional => 1,
 633     },
 634     serial => {
 635         type => 'string',
 636         format => 'urlencoded',
 637         format_description => 'serial',
 638         maxLength => 20*3, # *3 since it's %xx url enoded
 639         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 640         optional => 1,
 641     }
 642 );
 643
 644 my %rerror_fmt = (
 645     rerror => {
 646         type => 'string',
 647         format_description => 'ignore|report|stop',
 648         enum => [qw(ignore report stop)],
 649         description => 'Read error action.',
 650         optional => 1,
 651     },
 652 );
 653
 654 my %iothread_fmt = ( iothread => {
 655         type => 'boolean',
 656         format_description => 'off|on',
 657         description => "Whether to use iothreads for this drive",
 658         optional => 1,
 659 });
 660
 661 my %model_fmt = (
 662     model => {
 663         type => 'string',
 664         format => 'urlencoded',
 665         format_description => 'model',
 666         maxLength => 40*3, # *3 since it's %xx url enoded
 667         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 668         optional => 1,
 669     },
 670 );
 671
 672 my %queues_fmt = (
 673     queues => {
 674         type => 'integer',
 675         format_description => 'nbqueues',
 676         description => "Number of queues.",
 677         minimum => 2,
 678         optional => 1
 679     }
 680 );
 681
 682 my $add_throttle_desc = sub {
 683     my ($key, $type, $what, $size, $longsize) = @_;
 684     $drivedesc_base{$key} = {
 685         type => $type,
 686         format_description => $size,
 687         description => "Maximum $what speed in $longsize per second.",
 688         optional => 1,
 689     };
 690 };
 691 # throughput: (leaky bucket)
 692 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 693 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 694 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 695 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 696 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 697 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 698 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 699 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 700 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 701
 702 # pools: (pool of IO before throttling starts taking effect)
 703 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 704 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 705 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 706 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 707 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 708 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 709
 710 my $ide_fmt = {
 711     %drivedesc_base,
 712     %rerror_fmt,
 713     %model_fmt,
 714 };
 715
 716 my $idedesc = {
 717     optional => 1,
 718     type => 'string', format => $ide_fmt,
 719     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 720 };
 721 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 722
 723 my $scsi_fmt = {
 724     %drivedesc_base,
 725     %iothread_fmt,
 726     %queues_fmt,
 727 };
 728 my $scsidesc = {
 729     optional => 1,
 730     type => 'string', format => $scsi_fmt,
 731     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 732 };
 733 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 734
 735 my $sata_fmt = {
 736     %drivedesc_base,
 737     %rerror_fmt,
 738 };
 739 my $satadesc = {
 740     optional => 1,
 741     type => 'string', format => $sata_fmt,
 742     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 743 };
 744 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 745
 746 my $virtio_fmt = {
 747     %drivedesc_base,
 748     %iothread_fmt,
 749     %rerror_fmt,
 750 };
 751 my $virtiodesc = {
 752     optional => 1,
 753     type => 'string', format => $virtio_fmt,
 754     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 755 };
 756 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 757
 758 my $alldrive_fmt = {
 759     %drivedesc_base,
 760     %rerror_fmt,
 761     %iothread_fmt,
 762     %model_fmt,
 763     %queues_fmt,
 764 };
 765
 766 my $usb_fmt = {
 767     host => {
 768         default_key => 1,
 769         type => 'string', format => 'pve-qm-usb-device',
 770         format_description => 'HOSTUSBDEVICE|spice',
 771         description => 'The Host USB device or port or the value spice',
 772     },
 773     usb3 => {
 774         optional => 1,
 775         type => 'boolean',
 776         format_description => 'yes|no',
 777         description => 'Specifies whether if given host option is a USB3 device or port',
 778     },
 779 };
 780
 781 my $usbdesc = {
 782     optional => 1,
 783     type => 'string', format => $usb_fmt,
 784     description => <<EODESCR,
 785 Configure an USB device (n is 0 to 4). This can be used to
 786 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 787
 788 'bus-port(.port)*' (decimal numbers) or
 789 'vendor_id:product_id' (hexadeciaml numbers) or
 790 'spice'
 791
 792 You can use the 'lsusb -t' command to list existing usb devices.
 793
 794 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 795
 796 The value 'spice' can be used to add a usb redirection devices for spice.
 797
 798 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 799
 800 EODESCR
 801 };
 802 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 803
 804 my $hostpcidesc = {
 805         optional => 1,
 806         type => 'string', format => 'pve-qm-hostpci',
 807         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 808         description => <<EODESCR,
 809 Map host pci devices. HOSTPCIDEVICE syntax is:
 810
 811 'bus:dev.func' (hexadecimal numbers)
 812
 813 You can us the 'lspci' command to list existing pci devices.
 814
 815 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 816
 817 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 818
 819 Experimental: user reported problems with this option.
 820 EODESCR
 821 };
 822 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 823
 824 my $serialdesc = {
 825         optional => 1,
 826         type => 'string',
 827         pattern => '(/dev/.+|socket)',
 828         description =>  <<EODESCR,
 829 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 830
 831 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 832
 833 Experimental: user reported problems with this option.
 834 EODESCR
 835 };
 836
 837 my $paralleldesc= {
 838         optional => 1,
 839         type => 'string',
 840         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 841         description =>  <<EODESCR,
 842 Map host parallel devices (n is 0 to 2).
 843
 844 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 845
 846 Experimental: user reported problems with this option.
 847 EODESCR
 848 };
 849
 850 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 851     $confdesc->{"parallel$i"} = $paralleldesc;
 852 }
 853
 854 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 855     $confdesc->{"serial$i"} = $serialdesc;
 856 }
 857
 858 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 859     $confdesc->{"hostpci$i"} = $hostpcidesc;
 860 }
 861
 862 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 863     $drivename_hash->{"ide$i"} = 1;
 864     $confdesc->{"ide$i"} = $idedesc;
 865 }
 866
 867 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 868     $drivename_hash->{"sata$i"} = 1;
 869     $confdesc->{"sata$i"} = $satadesc;
 870 }
 871
 872 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 873     $drivename_hash->{"scsi$i"} = 1;
 874     $confdesc->{"scsi$i"} = $scsidesc ;
 875 }
 876
 877 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 878     $drivename_hash->{"virtio$i"} = 1;
 879     $confdesc->{"virtio$i"} = $virtiodesc;
 880 }
 881
 882 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 883     $confdesc->{"usb$i"} = $usbdesc;
 884 }
 885
 886 my $unuseddesc = {
 887     optional => 1,
 888     type => 'string', format => 'pve-volume-id',
 889     description => "Reference to unused volumes.",
 890 };
 891
 892 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 893     $confdesc->{"unused$i"} = $unuseddesc;
 894 }
 895
 896 my $kvm_api_version = 0;
 897
 898 sub kvm_version {
 899
 900     return $kvm_api_version if $kvm_api_version;
 901
 902     my $fh = IO::File->new("</dev/kvm") ||
 903         return 0;
 904
 905     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 906         $kvm_api_version = $v;
 907     }
 908
 909     $fh->close();
 910
 911     return  $kvm_api_version;
 912 }
 913
 914 my $kvm_user_version;
 915
 916 sub kvm_user_version {
 917
 918     return $kvm_user_version if $kvm_user_version;
 919
 920     $kvm_user_version = 'unknown';
 921
 922     my $code = sub {
 923         my $line = shift;
 924         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 925             $kvm_user_version = $2;
 926         }
 927     };
 928
 929     eval { run_command("kvm -version", outfunc => $code); };
 930     warn $@ if $@;
 931
 932     return $kvm_user_version;
 933
 934 }
 935
 936 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 937
 938 sub valid_drive_names {
 939     # order is important - used to autoselect boot disk
 940     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 941             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 942             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 943             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 944 }
 945
 946 sub is_valid_drivename {
 947     my $dev = shift;
 948
 949     return defined($drivename_hash->{$dev});
 950 }
 951
 952 sub option_exists {
 953     my $key = shift;
 954     return defined($confdesc->{$key});
 955 }
 956
 957 sub nic_models {
 958     return $nic_model_list;
 959 }
 960
 961 sub os_list_description {
 962
 963     return {
 964         other => 'Other',
 965         wxp => 'Windows XP',
 966         w2k => 'Windows 2000',
 967         w2k3 =>, 'Windows 2003',
 968         w2k8 => 'Windows 2008',
 969         wvista => 'Windows Vista',
 970         win7 => 'Windows 7',
 971         win8 => 'Windows 8/2012',
 972         l24 => 'Linux 2.4',
 973         l26 => 'Linux 2.6',
 974     };
 975 }
 976
 977 my $cdrom_path;
 978
 979 sub get_cdrom_path {
 980
 981     return  $cdrom_path if $cdrom_path;
 982
 983     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 984     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 985     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 986 }
 987
 988 sub get_iso_path {
 989     my ($storecfg, $vmid, $cdrom) = @_;
 990
 991     if ($cdrom eq 'cdrom') {
 992         return get_cdrom_path();
 993     } elsif ($cdrom eq 'none') {
 994         return '';
 995     } elsif ($cdrom =~ m|^/|) {
 996         return $cdrom;
 997     } else {
 998         return PVE::Storage::path($storecfg, $cdrom);
 999     }
1000 }
1001
1002 # try to convert old style file names to volume IDs
1003 sub filename_to_volume_id {
1004     my ($vmid, $file, $media) = @_;
1005
1006     if (!($file eq 'none' || $file eq 'cdrom' ||
1007           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1008
1009         return undef if $file =~ m|/|;
1010
1011         if ($media && $media eq 'cdrom') {
1012             $file = "local:iso/$file";
1013         } else {
1014             $file = "local:$vmid/$file";
1015         }
1016     }
1017
1018     return $file;
1019 }
1020
1021 sub verify_media_type {
1022     my ($opt, $vtype, $media) = @_;
1023
1024     return if !$media;
1025
1026     my $etype;
1027     if ($media eq 'disk') {
1028         $etype = 'images';
1029     } elsif ($media eq 'cdrom') {
1030         $etype = 'iso';
1031     } else {
1032         die "internal error";
1033     }
1034
1035     return if ($vtype eq $etype);
1036
1037     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1038 }
1039
1040 sub cleanup_drive_path {
1041     my ($opt, $storecfg, $drive) = @_;
1042
1043     # try to convert filesystem paths to volume IDs
1044
1045     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1046         ($drive->{file} !~ m|^/dev/.+|) &&
1047         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1048         ($drive->{file} !~ m/^\d+$/)) {
1049         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1050         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1051         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1052         verify_media_type($opt, $vtype, $drive->{media});
1053         $drive->{file} = $volid;
1054     }
1055
1056     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1057 }
1058
1059 sub parse_hotplug_features {
1060     my ($data) = @_;
1061
1062     my $res = {};
1063
1064     return $res if $data eq '0';
1065
1066     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1067
1068     foreach my $feature (PVE::Tools::split_list($data)) {
1069         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1070             $res->{$1} = 1;
1071         } else {
1072             warn "ignoring unknown hotplug feature '$feature'\n";
1073         }
1074     }
1075     return $res;
1076 }
1077
1078 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1079 sub pve_verify_hotplug_features {
1080     my ($value, $noerr) = @_;
1081
1082     return $value if parse_hotplug_features($value);
1083
1084     return undef if $noerr;
1085
1086     die "unable to parse hotplug option\n";
1087 }
1088
1089 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1090 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1091 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1092 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1093 #        [,iothread=on][,serial=serial][,model=model]
1094
1095 sub parse_drive {
1096     my ($key, $data) = @_;
1097
1098     my ($interface, $index);
1099
1100     if ($key =~ m/^([^\d]+)(\d+)$/) {
1101         $interface = $1;
1102         $index = $2;
1103     } else {
1104         return undef;
1105     }
1106
1107     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1108                                      : $confdesc->{$key}->{format};
1109     if (!$desc) {
1110         warn "invalid drive key: $key\n";
1111         return undef;
1112     }
1113     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1114     return undef if !$res;
1115     $res->{interface} = $interface;
1116     $res->{index} = $index;
1117
1118     my $error = 0;
1119     foreach my $opt (qw(bps bps_rd bps_wr)) {
1120         if (my $bps = defined(delete $res->{$opt})) {
1121             if (defined($res->{"m$opt"})) {
1122                 warn "both $opt and m$opt specified\n";
1123                 ++$error;
1124                 next;
1125             }
1126             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1127         }
1128     }
1129     return undef if $error;
1130
1131     return undef if $res->{mbps_rd} && $res->{mbps};
1132     return undef if $res->{mbps_wr} && $res->{mbps};
1133     return undef if $res->{iops_rd} && $res->{iops};
1134     return undef if $res->{iops_wr} && $res->{iops};
1135
1136     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1137         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1138         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1139         return undef if $res->{interface} eq 'virtio';
1140     }
1141
1142     if (my $size = $res->{size}) {
1143         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1144     }
1145
1146     return $res;
1147 }
1148
1149 sub print_drive {
1150     my ($vmid, $drive) = @_;
1151     my $data = { %$drive };
1152     delete $data->{$_} for qw(index interface);
1153     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1154 }
1155
1156 sub scsi_inquiry {
1157     my($fh, $noerr) = @_;
1158
1159     my $SG_IO = 0x2285;
1160     my $SG_GET_VERSION_NUM = 0x2282;
1161
1162     my $versionbuf = "\x00" x 8;
1163     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1164     if (!$ret) {
1165         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1166         return undef;
1167     }
1168     my $version = unpack("I", $versionbuf);
1169     if ($version < 30000) {
1170         die "scsi generic interface too old\n"  if !$noerr;
1171         return undef;
1172     }
1173
1174     my $buf = "\x00" x 36;
1175     my $sensebuf = "\x00" x 8;
1176     my $cmd = pack("C x3 C x1", 0x12, 36);
1177
1178     # see /usr/include/scsi/sg.h
1179     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1180
1181     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1182                       length($sensebuf), 0, length($buf), $buf,
1183                       $cmd, $sensebuf, 6000);
1184
1185     $ret = ioctl($fh, $SG_IO, $packet);
1186     if (!$ret) {
1187         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1188         return undef;
1189     }
1190
1191     my @res = unpack($sg_io_hdr_t, $packet);
1192     if ($res[17] || $res[18]) {
1193         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1194         return undef;
1195     }
1196
1197     my $res = {};
1198     (my $byte0, my $byte1, $res->{vendor},
1199      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1200
1201     $res->{removable} = $byte1 & 128 ? 1 : 0;
1202     $res->{type} = $byte0 & 31;
1203
1204     return $res;
1205 }
1206
1207 sub path_is_scsi {
1208     my ($path) = @_;
1209
1210     my $fh = IO::File->new("+<$path") || return undef;
1211     my $res = scsi_inquiry($fh, 1);
1212     close($fh);
1213
1214     return $res;
1215 }
1216
1217 sub machine_type_is_q35 {
1218     my ($conf) = @_;
1219
1220     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1221 }
1222
1223 sub print_tabletdevice_full {
1224     my ($conf) = @_;
1225
1226     my $q35 = machine_type_is_q35($conf);
1227
1228     # we use uhci for old VMs because tablet driver was buggy in older qemu
1229     my $usbbus = $q35 ? "ehci" : "uhci";
1230
1231     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1232 }
1233
1234 sub print_drivedevice_full {
1235     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1236
1237     my $device = '';
1238     my $maxdev = 0;
1239
1240     if ($drive->{interface} eq 'virtio') {
1241         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1242         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1243         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1244     } elsif ($drive->{interface} eq 'scsi') {
1245
1246         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1247         my $unit = $drive->{index} % $maxdev;
1248         my $devicetype = 'hd';
1249         my $path = '';
1250         if (drive_is_cdrom($drive)) {
1251             $devicetype = 'cd';
1252         } else {
1253             if ($drive->{file} =~ m|^/|) {
1254                 $path = $drive->{file};
1255                 if (my $info = path_is_scsi($path)) {
1256                     if ($info->{type} == 0) {
1257                         $devicetype = 'block';
1258                     } elsif ($info->{type} == 1) { # tape
1259                         $devicetype = 'generic';
1260                     }
1261                 }
1262             } else {
1263                  $path = PVE::Storage::path($storecfg, $drive->{file});
1264             }
1265
1266             if($path =~ m/^iscsi\:\/\//){
1267                 $devicetype = 'generic';
1268             }
1269         }
1270
1271         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1272             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1273         } else {
1274             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1275         }
1276
1277     } elsif ($drive->{interface} eq 'ide'){
1278         $maxdev = 2;
1279         my $controller = int($drive->{index} / $maxdev);
1280         my $unit = $drive->{index} % $maxdev;
1281         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1282
1283         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1284         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1285             $model = URI::Escape::uri_unescape($model);
1286             $device .= ",model=$model";
1287         }
1288     } elsif ($drive->{interface} eq 'sata'){
1289         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1290         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1291         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1292     } elsif ($drive->{interface} eq 'usb') {
1293         die "implement me";
1294         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1295     } else {
1296         die "unsupported interface type";
1297     }
1298
1299     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1300
1301     return $device;
1302 }
1303
1304 sub get_initiator_name {
1305     my $initiator;
1306
1307     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1308     while (defined(my $line = <$fh>)) {
1309         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1310         $initiator = $1;
1311         last;
1312     }
1313     $fh->close();
1314
1315     return $initiator;
1316 }
1317
1318 sub print_drive_full {
1319     my ($storecfg, $vmid, $drive) = @_;
1320
1321     my $path;
1322     my $volid = $drive->{file};
1323     my $format;
1324
1325     if (drive_is_cdrom($drive)) {
1326         $path = get_iso_path($storecfg, $vmid, $volid);
1327     } else {
1328         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1329         if ($storeid) {
1330             $path = PVE::Storage::path($storecfg, $volid);
1331             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1332             $format = qemu_img_format($scfg, $volname);
1333         } else {
1334             $path = $volid;
1335             $format = "raw";
1336         }
1337    }
1338
1339     my $opts = '';
1340     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1341     foreach my $o (@qemu_drive_options) {
1342         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1343     }
1344     if (my $serial = $drive->{serial}) {
1345         $serial = URI::Escape::uri_unescape($serial);
1346         $opts .= ",serial=$serial";
1347     }
1348
1349     $opts .= ",format=$format" if $format && !$drive->{format};
1350
1351     foreach my $o (qw(bps bps_rd bps_wr)) {
1352         my $v = $drive->{"m$o"};
1353         $opts .= ",$o=" . int($v*1024*1024) if $v;
1354     }
1355
1356     my $cache_direct = 0;
1357
1358     if (my $cache = $drive->{cache}) {
1359         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1360     } elsif (!drive_is_cdrom($drive)) {
1361         $opts .= ",cache=none";
1362         $cache_direct = 1;
1363     }
1364
1365     # aio native works only with O_DIRECT
1366     if (!$drive->{aio}) {
1367         if($cache_direct) {
1368             $opts .= ",aio=native";
1369         } else {
1370             $opts .= ",aio=threads";
1371         }
1372     }
1373
1374     if (!drive_is_cdrom($drive)) {
1375         my $detectzeroes;
1376         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1377             $detectzeroes = 'off';
1378         } elsif ($drive->{discard}) {
1379             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1380         } else {
1381             # This used to be our default with discard not being specified:
1382             $detectzeroes = 'on';
1383         }
1384         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1385     }
1386
1387     my $pathinfo = $path ? "file=$path," : '';
1388
1389     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1390 }
1391
1392 sub print_netdevice_full {
1393     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1394
1395     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1396
1397     my $device = $net->{model};
1398     if ($net->{model} eq 'virtio') {
1399          $device = 'virtio-net-pci';
1400      };
1401
1402     my $pciaddr = print_pci_addr("$netid", $bridges);
1403     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1404     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1405         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1406         my $vectors = $net->{queues} * 2 + 2;
1407         $tmpstr .= ",vectors=$vectors,mq=on";
1408     }
1409     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1410
1411     if ($use_old_bios_files) {
1412         my $romfile;
1413         if ($device eq 'virtio-net-pci') {
1414             $romfile = 'pxe-virtio.rom';
1415         } elsif ($device eq 'e1000') {
1416             $romfile = 'pxe-e1000.rom';
1417         } elsif ($device eq 'ne2k') {
1418             $romfile = 'pxe-ne2k_pci.rom';
1419         } elsif ($device eq 'pcnet') {
1420             $romfile = 'pxe-pcnet.rom';
1421         } elsif ($device eq 'rtl8139') {
1422             $romfile = 'pxe-rtl8139.rom';
1423         }
1424         $tmpstr .= ",romfile=$romfile" if $romfile;
1425     }
1426
1427     return $tmpstr;
1428 }
1429
1430 sub print_netdev_full {
1431     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1432
1433     my $i = '';
1434     if ($netid =~ m/^net(\d+)$/) {
1435         $i = int($1);
1436     }
1437
1438     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1439
1440     my $ifname = "tap${vmid}i$i";
1441
1442     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1443     die "interface name '$ifname' is too long (max 15 character)\n"
1444         if length($ifname) >= 16;
1445
1446     my $vhostparam = '';
1447     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1448
1449     my $vmname = $conf->{name} || "vm$vmid";
1450
1451     my $netdev = "";
1452     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1453
1454     if ($net->{bridge}) {
1455         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1456     } else {
1457         $netdev = "type=user,id=$netid,hostname=$vmname";
1458     }
1459
1460     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1461
1462     return $netdev;
1463 }
1464
1465 sub drive_is_cdrom {
1466     my ($drive) = @_;
1467
1468     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1469
1470 }
1471
1472 sub parse_numa {
1473     my ($data) = @_;
1474
1475     my $res = {};
1476
1477     foreach my $kvp (split(/,/, $data)) {
1478
1479         if ($kvp =~ m/^memory=(\S+)$/) {
1480             $res->{memory} = $1;
1481         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1482             $res->{policy} = $1;
1483         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1484             $res->{cpus}->{start} = $1;
1485             $res->{cpus}->{end} = $3;
1486         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1487             $res->{hostnodes}->{start} = $1;
1488             $res->{hostnodes}->{end} = $3;
1489         } else {
1490             return undef;
1491         }
1492     }
1493
1494     return $res;
1495 }
1496
1497 sub parse_hostpci {
1498     my ($value) = @_;
1499
1500     return undef if !$value;
1501
1502
1503     my @list = split(/,/, $value);
1504     my $found;
1505
1506     my $res = {};
1507     foreach my $kv (@list) {
1508
1509         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1510             $found = 1;
1511             if(defined($4)){
1512                 push @{$res->{pciid}}, { id => $2 , function => $4};
1513
1514             }else{
1515                 my $pcidevices = lspci($2);
1516                 $res->{pciid} = $pcidevices->{$2};
1517             }
1518         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1519             $res->{rombar} = $1;
1520         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1521             $res->{'x-vga'} = $1;
1522         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1523             $res->{pcie} = 1 if $1 == 1;
1524         } else {
1525             warn "unknown hostpci setting '$kv'\n";
1526         }
1527     }
1528
1529     return undef if !$found;
1530
1531     return $res;
1532 }
1533
1534 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1535 sub parse_net {
1536     my ($data) = @_;
1537
1538     my $res = {};
1539
1540     foreach my $kvp (split(/,/, $data)) {
1541
1542         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1543             my $model = lc($1);
1544             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1545             $res->{model} = $model;
1546             $res->{macaddr} = $mac;
1547         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1548             $res->{bridge} = $1;
1549         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1550             $res->{queues} = $1;
1551         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1552             $res->{rate} = $1;
1553         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1554             $res->{tag} = $1;
1555         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1556             $res->{trunks} = $1;
1557         } elsif ($kvp =~ m/^firewall=([01])$/) {
1558             $res->{firewall} = $1;
1559         } elsif ($kvp =~ m/^link_down=([01])$/) {
1560             $res->{link_down} = $1;
1561         } else {
1562             return undef;
1563         }
1564
1565     }
1566
1567     return undef if !$res->{model};
1568
1569     return $res;
1570 }
1571
1572 sub print_net {
1573     my $net = shift;
1574
1575     my $res = "$net->{model}";
1576     $res .= "=$net->{macaddr}" if $net->{macaddr};
1577     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1578     $res .= ",rate=$net->{rate}" if $net->{rate};
1579     $res .= ",tag=$net->{tag}" if $net->{tag};
1580     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1581     $res .= ",firewall=1" if $net->{firewall};
1582     $res .= ",link_down=1" if $net->{link_down};
1583     $res .= ",queues=$net->{queues}" if $net->{queues};
1584
1585     return $res;
1586 }
1587
1588 sub add_random_macs {
1589     my ($settings) = @_;
1590
1591     foreach my $opt (keys %$settings) {
1592         next if $opt !~ m/^net(\d+)$/;
1593         my $net = parse_net($settings->{$opt});
1594         next if !$net;
1595         $settings->{$opt} = print_net($net);
1596     }
1597 }
1598
1599 sub vm_is_volid_owner {
1600     my ($storecfg, $vmid, $volid) = @_;
1601
1602     if ($volid !~  m|^/|) {
1603         my ($path, $owner);
1604         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1605         if ($owner && ($owner == $vmid)) {
1606             return 1;
1607         }
1608     }
1609
1610     return undef;
1611 }
1612
1613 sub split_flagged_list {
1614     my $text = shift || '';
1615     $text =~ s/[,;]/ /g;
1616     $text =~ s/^\s+//;
1617     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1618 }
1619
1620 sub join_flagged_list {
1621     my ($how, $lst) = @_;
1622     join $how, map { $lst->{$_} . $_ } keys %$lst;
1623 }
1624
1625 sub vmconfig_delete_pending_option {
1626     my ($conf, $key, $force) = @_;
1627
1628     delete $conf->{pending}->{$key};
1629     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1630     $pending_delete_hash->{$key} = $force ? '!' : '';
1631     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1632 }
1633
1634 sub vmconfig_undelete_pending_option {
1635     my ($conf, $key) = @_;
1636
1637     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1638     delete $pending_delete_hash->{$key};
1639
1640     if (%$pending_delete_hash) {
1641         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1642     } else {
1643         delete $conf->{pending}->{delete};
1644     }
1645 }
1646
1647 sub vmconfig_register_unused_drive {
1648     my ($storecfg, $vmid, $conf, $drive) = @_;
1649
1650     if (!drive_is_cdrom($drive)) {
1651         my $volid = $drive->{file};
1652         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1653             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1654         }
1655     }
1656 }
1657
1658 sub vmconfig_cleanup_pending {
1659     my ($conf) = @_;
1660
1661     # remove pending changes when nothing changed
1662     my $changes;
1663     foreach my $opt (keys %{$conf->{pending}}) {
1664         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1665             $changes = 1;
1666             delete $conf->{pending}->{$opt};
1667         }
1668     }
1669
1670     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1671     my $pending_delete_hash = {};
1672     while (my ($opt, $force) = each %$current_delete_hash) {
1673         if (defined($conf->{$opt})) {
1674             $pending_delete_hash->{$opt} = $force;
1675         } else {
1676             $changes = 1;
1677         }
1678     }
1679
1680     if (%$pending_delete_hash) {
1681         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1682     } else {
1683         delete $conf->{pending}->{delete};
1684     }
1685
1686     return $changes;
1687 }
1688
1689 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1690 my $smbios1_fmt = {
1691     uuid => {
1692         type => 'string',
1693         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1694         format_description => 'UUID',
1695         optional => 1,
1696     },
1697     version => {
1698         type => 'string',
1699         pattern => '\S+',
1700         format_description => 'str',
1701         optional => 1,
1702     },
1703     serial => {
1704         type => 'string',
1705         pattern => '\S+',
1706         format_description => 'str',
1707         optional => 1,
1708     },
1709     manufacturer => {
1710         type => 'string',
1711         pattern => '\S+',
1712         format_description => 'name',
1713         optional => 1,
1714     },
1715     product => {
1716         type => 'string',
1717         pattern => '\S+',
1718         format_description => 'name',
1719         optional => 1,
1720     },
1721     sku => {
1722         type => 'string',
1723         pattern => '\S+',
1724         format_description => 'str',
1725         optional => 1,
1726     },
1727     family => {
1728         type => 'string',
1729         pattern => '\S+',
1730         format_description => 'str',
1731         optional => 1,
1732     },
1733 };
1734
1735 sub parse_smbios1 {
1736     my ($data) = @_;
1737
1738     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1739     warn $@ if $@;
1740     return $res;
1741 }
1742
1743 sub print_smbios1 {
1744     my ($smbios1) = @_;
1745     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1746 }
1747
1748 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1749
1750 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1751 sub verify_bootdisk {
1752     my ($value, $noerr) = @_;
1753
1754     return $value if is_valid_drivename($value);
1755
1756     return undef if $noerr;
1757
1758     die "invalid boot disk '$value'\n";
1759 }
1760
1761 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1762 sub verify_numa {
1763     my ($value, $noerr) = @_;
1764
1765     return $value if parse_numa($value);
1766
1767     return undef if $noerr;
1768
1769     die "unable to parse numa options\n";
1770 }
1771
1772 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1773 sub verify_net {
1774     my ($value, $noerr) = @_;
1775
1776     return $value if parse_net($value);
1777
1778     return undef if $noerr;
1779
1780     die "unable to parse network options\n";
1781 }
1782
1783 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1784 sub verify_hostpci {
1785     my ($value, $noerr) = @_;
1786
1787     return $value if parse_hostpci($value);
1788
1789     return undef if $noerr;
1790
1791     die "unable to parse pci id\n";
1792 }
1793
1794 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1795 sub verify_watchdog {
1796     my ($value, $noerr) = @_;
1797
1798     return $value if parse_watchdog($value);
1799
1800     return undef if $noerr;
1801
1802     die "unable to parse watchdog options\n";
1803 }
1804
1805 sub parse_watchdog {
1806     my ($value) = @_;
1807
1808     return undef if !$value;
1809
1810     my $res = {};
1811
1812     foreach my $p (split(/,/, $value)) {
1813         next if $p =~ m/^\s*$/;
1814
1815         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1816             $res->{model} = $2;
1817         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1818             $res->{action} = $2;
1819         } else {
1820             return undef;
1821         }
1822     }
1823
1824     return $res;
1825 }
1826
1827 sub parse_usb_device {
1828     my ($value) = @_;
1829
1830     return undef if !$value;
1831
1832     my $res = {};
1833     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1834         $res->{vendorid} = $2;
1835         $res->{productid} = $4;
1836     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1837         $res->{hostbus} = $1;
1838         $res->{hostport} = $2;
1839     } elsif ($value =~ m/^spice$/i) {
1840         $res->{spice} = 1;
1841     } else {
1842         return undef;
1843     }
1844
1845     return $res;
1846 }
1847
1848 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1849 sub verify_usb_device {
1850     my ($value, $noerr) = @_;
1851
1852     return $value if parse_usb_device($value);
1853
1854     return undef if $noerr;
1855
1856     die "unable to parse usb device\n";
1857 }
1858
1859 # add JSON properties for create and set function
1860 sub json_config_properties {
1861     my $prop = shift;
1862
1863     foreach my $opt (keys %$confdesc) {
1864         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1865         $prop->{$opt} = $confdesc->{$opt};
1866     }
1867
1868     return $prop;
1869 }
1870
1871 sub check_type {
1872     my ($key, $value) = @_;
1873
1874     die "unknown setting '$key'\n" if !$confdesc->{$key};
1875
1876     my $type = $confdesc->{$key}->{type};
1877
1878     if (!defined($value)) {
1879         die "got undefined value\n";
1880     }
1881
1882     if ($value =~ m/[\n\r]/) {
1883         die "property contains a line feed\n";
1884     }
1885
1886     if ($type eq 'boolean') {
1887         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1888         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1889         die "type check ('boolean') failed - got '$value'\n";
1890     } elsif ($type eq 'integer') {
1891         return int($1) if $value =~ m/^(\d+)$/;
1892         die "type check ('integer') failed - got '$value'\n";
1893     } elsif ($type eq 'number') {
1894         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1895         die "type check ('number') failed - got '$value'\n";
1896     } elsif ($type eq 'string') {
1897         if (my $fmt = $confdesc->{$key}->{format}) {
1898             if ($fmt eq 'pve-qm-drive') {
1899                 # special case - we need to pass $key to parse_drive()
1900                 my $drive = parse_drive($key, $value);
1901                 return $value if $drive;
1902                 die "unable to parse drive options\n";
1903             }
1904             PVE::JSONSchema::check_format($fmt, $value);
1905             return $value;
1906         }
1907         $value =~ s/^\"(.*)\"$/$1/;
1908         return $value;
1909     } else {
1910         die "internal error"
1911     }
1912 }
1913
1914 sub check_iommu_support{
1915     #fixme : need to check IOMMU support
1916     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1917
1918     my $iommu=1;
1919     return $iommu;
1920
1921 }
1922
1923 sub touch_config {
1924     my ($vmid) = @_;
1925
1926     my $conf = PVE::QemuConfig->config_file($vmid);
1927     utime undef, undef, $conf;
1928 }
1929
1930 sub destroy_vm {
1931     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1932
1933     my $conffile = PVE::QemuConfig->config_file($vmid);
1934
1935     my $conf = PVE::QemuConfig->load_config($vmid);
1936
1937     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1938
1939     # only remove disks owned by this VM
1940     foreach_drive($conf, sub {
1941         my ($ds, $drive) = @_;
1942
1943         return if drive_is_cdrom($drive);
1944
1945         my $volid = $drive->{file};
1946
1947         return if !$volid || $volid =~ m|^/|;
1948
1949         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1950         return if !$path || !$owner || ($owner != $vmid);
1951
1952         PVE::Storage::vdisk_free($storecfg, $volid);
1953     });
1954
1955     if ($keep_empty_config) {
1956         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1957     } else {
1958         unlink $conffile;
1959     }
1960
1961     # also remove unused disk
1962     eval {
1963         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1964
1965         eval {
1966             PVE::Storage::foreach_volid($dl, sub {
1967                 my ($volid, $sid, $volname, $d) = @_;
1968                 PVE::Storage::vdisk_free($storecfg, $volid);
1969             });
1970         };
1971         warn $@ if $@;
1972
1973     };
1974     warn $@ if $@;
1975 }
1976
1977 sub parse_vm_config {
1978     my ($filename, $raw) = @_;
1979
1980     return undef if !defined($raw);
1981
1982     my $res = {
1983         digest => Digest::SHA::sha1_hex($raw),
1984         snapshots => {},
1985         pending => {},
1986     };
1987
1988     $filename =~ m|/qemu-server/(\d+)\.conf$|
1989         || die "got strange filename '$filename'";
1990
1991     my $vmid = $1;
1992
1993     my $conf = $res;
1994     my $descr;
1995     my $section = '';
1996
1997     my @lines = split(/\n/, $raw);
1998     foreach my $line (@lines) {
1999         next if $line =~ m/^\s*$/;
2000
2001         if ($line =~ m/^\[PENDING\]\s*$/i) {
2002             $section = 'pending';
2003             if (defined($descr)) {
2004                 $descr =~ s/\s+$//;
2005                 $conf->{description} = $descr;
2006             }
2007             $descr = undef;
2008             $conf = $res->{$section} = {};
2009             next;
2010
2011         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2012             $section = $1;
2013             if (defined($descr)) {
2014                 $descr =~ s/\s+$//;
2015                 $conf->{description} = $descr;
2016             }
2017             $descr = undef;
2018             $conf = $res->{snapshots}->{$section} = {};
2019             next;
2020         }
2021
2022         if ($line =~ m/^\#(.*)\s*$/) {
2023             $descr = '' if !defined($descr);
2024             $descr .= PVE::Tools::decode_text($1) . "\n";
2025             next;
2026         }
2027
2028         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2029             $descr = '' if !defined($descr);
2030             $descr .= PVE::Tools::decode_text($2);
2031         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2032             $conf->{snapstate} = $1;
2033         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2034             my $key = $1;
2035             my $value = $2;
2036             $conf->{$key} = $value;
2037         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2038             my $value = $1;
2039             if ($section eq 'pending') {
2040                 $conf->{delete} = $value; # we parse this later
2041             } else {
2042                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2043             }
2044         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2045             my $key = $1;
2046             my $value = $2;
2047             eval { $value = check_type($key, $value); };
2048             if ($@) {
2049                 warn "vm $vmid - unable to parse value of '$key' - $@";
2050             } else {
2051                 my $fmt = $confdesc->{$key}->{format};
2052                 if ($fmt && $fmt eq 'pve-qm-drive') {
2053                     my $v = parse_drive($key, $value);
2054                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2055                         $v->{file} = $volid;
2056                         $value = print_drive($vmid, $v);
2057                     } else {
2058                         warn "vm $vmid - unable to parse value of '$key'\n";
2059                         next;
2060                     }
2061                 }
2062
2063                 if ($key eq 'cdrom') {
2064                     $conf->{ide2} = $value;
2065                 } else {
2066                     $conf->{$key} = $value;
2067                 }
2068             }
2069         }
2070     }
2071
2072     if (defined($descr)) {
2073         $descr =~ s/\s+$//;
2074         $conf->{description} = $descr;
2075     }
2076     delete $res->{snapstate}; # just to be sure
2077
2078     return $res;
2079 }
2080
2081 sub write_vm_config {
2082     my ($filename, $conf) = @_;
2083
2084     delete $conf->{snapstate}; # just to be sure
2085
2086     if ($conf->{cdrom}) {
2087         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2088         $conf->{ide2} = $conf->{cdrom};
2089         delete $conf->{cdrom};
2090     }
2091
2092     # we do not use 'smp' any longer
2093     if ($conf->{sockets}) {
2094         delete $conf->{smp};
2095     } elsif ($conf->{smp}) {
2096         $conf->{sockets} = $conf->{smp};
2097         delete $conf->{cores};
2098         delete $conf->{smp};
2099     }
2100
2101     my $used_volids = {};
2102
2103     my $cleanup_config = sub {
2104         my ($cref, $pending, $snapname) = @_;
2105
2106         foreach my $key (keys %$cref) {
2107             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2108                 $key eq 'snapstate' || $key eq 'pending';
2109             my $value = $cref->{$key};
2110             if ($key eq 'delete') {
2111                 die "propertry 'delete' is only allowed in [PENDING]\n"
2112                     if !$pending;
2113                 # fixme: check syntax?
2114                 next;
2115             }
2116             eval { $value = check_type($key, $value); };
2117             die "unable to parse value of '$key' - $@" if $@;
2118
2119             $cref->{$key} = $value;
2120
2121             if (!$snapname && is_valid_drivename($key)) {
2122                 my $drive = parse_drive($key, $value);
2123                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2124             }
2125         }
2126     };
2127
2128     &$cleanup_config($conf);
2129
2130     &$cleanup_config($conf->{pending}, 1);
2131
2132     foreach my $snapname (keys %{$conf->{snapshots}}) {
2133         die "internal error" if $snapname eq 'pending';
2134         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2135     }
2136
2137     # remove 'unusedX' settings if we re-add a volume
2138     foreach my $key (keys %$conf) {
2139         my $value = $conf->{$key};
2140         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2141             delete $conf->{$key};
2142         }
2143     }
2144
2145     my $generate_raw_config = sub {
2146         my ($conf, $pending) = @_;
2147
2148         my $raw = '';
2149
2150         # add description as comment to top of file
2151         if (defined(my $descr = $conf->{description})) {
2152             if ($descr) {
2153                 foreach my $cl (split(/\n/, $descr)) {
2154                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2155                 }
2156             } else {
2157                 $raw .= "#\n" if $pending;
2158             }
2159         }
2160
2161         foreach my $key (sort keys %$conf) {
2162             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2163             $raw .= "$key: $conf->{$key}\n";
2164         }
2165         return $raw;
2166     };
2167
2168     my $raw = &$generate_raw_config($conf);
2169
2170     if (scalar(keys %{$conf->{pending}})){
2171         $raw .= "\n[PENDING]\n";
2172         $raw .= &$generate_raw_config($conf->{pending}, 1);
2173     }
2174
2175     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2176         $raw .= "\n[$snapname]\n";
2177         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2178     }
2179
2180     return $raw;
2181 }
2182
2183 sub load_defaults {
2184
2185     my $res = {};
2186
2187     # we use static defaults from our JSON schema configuration
2188     foreach my $key (keys %$confdesc) {
2189         if (defined(my $default = $confdesc->{$key}->{default})) {
2190             $res->{$key} = $default;
2191         }
2192     }
2193
2194     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2195     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2196
2197     return $res;
2198 }
2199
2200 sub config_list {
2201     my $vmlist = PVE::Cluster::get_vmlist();
2202     my $res = {};
2203     return $res if !$vmlist || !$vmlist->{ids};
2204     my $ids = $vmlist->{ids};
2205
2206     foreach my $vmid (keys %$ids) {
2207         my $d = $ids->{$vmid};
2208         next if !$d->{node} || $d->{node} ne $nodename;
2209         next if !$d->{type} || $d->{type} ne 'qemu';
2210         $res->{$vmid}->{exists} = 1;
2211     }
2212     return $res;
2213 }
2214
2215 # test if VM uses local resources (to prevent migration)
2216 sub check_local_resources {
2217     my ($conf, $noerr) = @_;
2218
2219     my $loc_res = 0;
2220
2221     $loc_res = 1 if $conf->{hostusb}; # old syntax
2222     $loc_res = 1 if $conf->{hostpci}; # old syntax
2223
2224     foreach my $k (keys %$conf) {
2225         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2226         # sockets are safe: they will recreated be on the target side post-migrate
2227         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2228         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2229     }
2230
2231     die "VM uses local resources\n" if $loc_res && !$noerr;
2232
2233     return $loc_res;
2234 }
2235
2236 # check if used storages are available on all nodes (use by migrate)
2237 sub check_storage_availability {
2238     my ($storecfg, $conf, $node) = @_;
2239
2240     foreach_drive($conf, sub {
2241         my ($ds, $drive) = @_;
2242
2243         my $volid = $drive->{file};
2244         return if !$volid;
2245
2246         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2247         return if !$sid;
2248
2249         # check if storage is available on both nodes
2250         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2251         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2252    });
2253 }
2254
2255 # list nodes where all VM images are available (used by has_feature API)
2256 sub shared_nodes {
2257     my ($conf, $storecfg) = @_;
2258
2259     my $nodelist = PVE::Cluster::get_nodelist();
2260     my $nodehash = { map { $_ => 1 } @$nodelist };
2261     my $nodename = PVE::INotify::nodename();
2262
2263     foreach_drive($conf, sub {
2264         my ($ds, $drive) = @_;
2265
2266         my $volid = $drive->{file};
2267         return if !$volid;
2268
2269         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2270         if ($storeid) {
2271             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2272             if ($scfg->{disable}) {
2273                 $nodehash = {};
2274             } elsif (my $avail = $scfg->{nodes}) {
2275                 foreach my $node (keys %$nodehash) {
2276                     delete $nodehash->{$node} if !$avail->{$node};
2277                 }
2278             } elsif (!$scfg->{shared}) {
2279                 foreach my $node (keys %$nodehash) {
2280                     delete $nodehash->{$node} if $node ne $nodename
2281                 }
2282             }
2283         }
2284     });
2285
2286     return $nodehash
2287 }
2288
2289 sub check_cmdline {
2290     my ($pidfile, $pid) = @_;
2291
2292     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2293     if (defined($fh)) {
2294         my $line = <$fh>;
2295         $fh->close;
2296         return undef if !$line;
2297         my @param = split(/\0/, $line);
2298
2299         my $cmd = $param[0];
2300         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2301
2302         for (my $i = 0; $i < scalar (@param); $i++) {
2303             my $p = $param[$i];
2304             next if !$p;
2305             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2306                 my $p = $param[$i+1];
2307                 return 1 if $p && ($p eq $pidfile);
2308                 return undef;
2309             }
2310         }
2311     }
2312     return undef;
2313 }
2314
2315 sub check_running {
2316     my ($vmid, $nocheck, $node) = @_;
2317
2318     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2319
2320     die "unable to find configuration file for VM $vmid - no such machine\n"
2321         if !$nocheck && ! -f $filename;
2322
2323     my $pidfile = pidfile_name($vmid);
2324
2325     if (my $fd = IO::File->new("<$pidfile")) {
2326         my $st = stat($fd);
2327         my $line = <$fd>;
2328         close($fd);
2329
2330         my $mtime = $st->mtime;
2331         if ($mtime > time()) {
2332             warn "file '$filename' modified in future\n";
2333         }
2334
2335         if ($line =~ m/^(\d+)$/) {
2336             my $pid = $1;
2337             if (check_cmdline($pidfile, $pid)) {
2338                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2339                     return $pid;
2340                 }
2341             }
2342         }
2343     }
2344
2345     return undef;
2346 }
2347
2348 sub vzlist {
2349
2350     my $vzlist = config_list();
2351
2352     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2353
2354     while (defined(my $de = $fd->read)) {
2355         next if $de !~ m/^(\d+)\.pid$/;
2356         my $vmid = $1;
2357         next if !defined($vzlist->{$vmid});
2358         if (my $pid = check_running($vmid)) {
2359             $vzlist->{$vmid}->{pid} = $pid;
2360         }
2361     }
2362
2363     return $vzlist;
2364 }
2365
2366 sub disksize {
2367     my ($storecfg, $conf) = @_;
2368
2369     my $bootdisk = $conf->{bootdisk};
2370     return undef if !$bootdisk;
2371     return undef if !is_valid_drivename($bootdisk);
2372
2373     return undef if !$conf->{$bootdisk};
2374
2375     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2376     return undef if !defined($drive);
2377
2378     return undef if drive_is_cdrom($drive);
2379
2380     my $volid = $drive->{file};
2381     return undef if !$volid;
2382
2383     return $drive->{size};
2384 }
2385
2386 my $last_proc_pid_stat;
2387
2388 # get VM status information
2389 # This must be fast and should not block ($full == false)
2390 # We only query KVM using QMP if $full == true (this can be slow)
2391 sub vmstatus {
2392     my ($opt_vmid, $full) = @_;
2393
2394     my $res = {};
2395
2396     my $storecfg = PVE::Storage::config();
2397
2398     my $list = vzlist();
2399     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2400
2401     my $cpucount = $cpuinfo->{cpus} || 1;
2402
2403     foreach my $vmid (keys %$list) {
2404         next if $opt_vmid && ($vmid ne $opt_vmid);
2405
2406         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2407         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2408
2409         my $d = {};
2410         $d->{pid} = $list->{$vmid}->{pid};
2411
2412         # fixme: better status?
2413         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2414
2415         my $size = disksize($storecfg, $conf);
2416         if (defined($size)) {
2417             $d->{disk} = 0; # no info available
2418             $d->{maxdisk} = $size;
2419         } else {
2420             $d->{disk} = 0;
2421             $d->{maxdisk} = 0;
2422         }
2423
2424         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2425         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2426         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2427
2428         $d->{name} = $conf->{name} || "VM $vmid";
2429         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2430
2431         if ($conf->{balloon}) {
2432             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2433             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2434         }
2435
2436         $d->{uptime} = 0;
2437         $d->{cpu} = 0;
2438         $d->{mem} = 0;
2439
2440         $d->{netout} = 0;
2441         $d->{netin} = 0;
2442
2443         $d->{diskread} = 0;
2444         $d->{diskwrite} = 0;
2445
2446         $d->{template} = PVE::QemuConfig->is_template($conf);
2447
2448         $res->{$vmid} = $d;
2449     }
2450
2451     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2452     foreach my $dev (keys %$netdev) {
2453         next if $dev !~ m/^tap([1-9]\d*)i/;
2454         my $vmid = $1;
2455         my $d = $res->{$vmid};
2456         next if !$d;
2457
2458         $d->{netout} += $netdev->{$dev}->{receive};
2459         $d->{netin} += $netdev->{$dev}->{transmit};
2460
2461         if ($full) {
2462             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2463             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2464         }
2465
2466     }
2467
2468     my $ctime = gettimeofday;
2469
2470     foreach my $vmid (keys %$list) {
2471
2472         my $d = $res->{$vmid};
2473         my $pid = $d->{pid};
2474         next if !$pid;
2475
2476         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2477         next if !$pstat; # not running
2478
2479         my $used = $pstat->{utime} + $pstat->{stime};
2480
2481         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2482
2483         if ($pstat->{vsize}) {
2484             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2485         }
2486
2487         my $old = $last_proc_pid_stat->{$pid};
2488         if (!$old) {
2489             $last_proc_pid_stat->{$pid} = {
2490                 time => $ctime,
2491                 used => $used,
2492                 cpu => 0,
2493             };
2494             next;
2495         }
2496
2497         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2498
2499         if ($dtime > 1000) {
2500             my $dutime = $used -  $old->{used};
2501
2502             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2503             $last_proc_pid_stat->{$pid} = {
2504                 time => $ctime,
2505                 used => $used,
2506                 cpu => $d->{cpu},
2507             };
2508         } else {
2509             $d->{cpu} = $old->{cpu};
2510         }
2511     }
2512
2513     return $res if !$full;
2514
2515     my $qmpclient = PVE::QMPClient->new();
2516
2517     my $ballooncb = sub {
2518         my ($vmid, $resp) = @_;
2519
2520         my $info = $resp->{'return'};
2521         return if !$info->{max_mem};
2522
2523         my $d = $res->{$vmid};
2524
2525         # use memory assigned to VM
2526         $d->{maxmem} = $info->{max_mem};
2527         $d->{balloon} = $info->{actual};
2528
2529         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2530             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2531             $d->{freemem} = $info->{free_mem};
2532         }
2533
2534         $d->{ballooninfo} = $info;
2535     };
2536
2537     my $blockstatscb = sub {
2538         my ($vmid, $resp) = @_;
2539         my $data = $resp->{'return'} || [];
2540         my $totalrdbytes = 0;
2541         my $totalwrbytes = 0;
2542
2543         for my $blockstat (@$data) {
2544             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2545             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2546
2547             $blockstat->{device} =~ s/drive-//;
2548             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2549         }
2550         $res->{$vmid}->{diskread} = $totalrdbytes;
2551         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2552     };
2553
2554     my $statuscb = sub {
2555         my ($vmid, $resp) = @_;
2556
2557         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2558         # this fails if ballon driver is not loaded, so this must be
2559         # the last commnand (following command are aborted if this fails).
2560         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2561
2562         my $status = 'unknown';
2563         if (!defined($status = $resp->{'return'}->{status})) {
2564             warn "unable to get VM status\n";
2565             return;
2566         }
2567
2568         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2569     };
2570
2571     foreach my $vmid (keys %$list) {
2572         next if $opt_vmid && ($vmid ne $opt_vmid);
2573         next if !$res->{$vmid}->{pid}; # not running
2574         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2575     }
2576
2577     $qmpclient->queue_execute(undef, 1);
2578
2579     foreach my $vmid (keys %$list) {
2580         next if $opt_vmid && ($vmid ne $opt_vmid);
2581         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2582     }
2583
2584     return $res;
2585 }
2586
2587 sub foreach_dimm {
2588     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2589
2590     my $dimm_id = 0;
2591     my $current_size = 1024;
2592     my $dimm_size = 512;
2593     return if $current_size == $memory;
2594
2595     for (my $j = 0; $j < 8; $j++) {
2596         for (my $i = 0; $i < 32; $i++) {
2597             my $name = "dimm${dimm_id}";
2598             $dimm_id++;
2599             my $numanode = $i % $sockets;
2600             $current_size += $dimm_size;
2601             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2602             return  $current_size if $current_size >= $memory;
2603         }
2604         $dimm_size *= 2;
2605     }
2606 }
2607
2608 sub foreach_reverse_dimm {
2609     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2610
2611     my $dimm_id = 253;
2612     my $current_size = 4177920;
2613     my $dimm_size = 65536;
2614     return if $current_size == $memory;
2615
2616     for (my $j = 0; $j < 8; $j++) {
2617         for (my $i = 0; $i < 32; $i++) {
2618             my $name = "dimm${dimm_id}";
2619             $dimm_id--;
2620             my $numanode = $i % $sockets;
2621             $current_size -= $dimm_size;
2622             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2623             return  $current_size if $current_size <= $memory;
2624         }
2625         $dimm_size /= 2;
2626     }
2627 }
2628
2629 sub foreach_drive {
2630     my ($conf, $func) = @_;
2631
2632     foreach my $ds (valid_drive_names()) {
2633         next if !defined($conf->{$ds});
2634
2635         my $drive = parse_drive($ds, $conf->{$ds});
2636         next if !$drive;
2637
2638         &$func($ds, $drive);
2639     }
2640 }
2641
2642 sub foreach_volid {
2643     my ($conf, $func) = @_;
2644
2645     my $volhash = {};
2646
2647     my $test_volid = sub {
2648         my ($volid, $is_cdrom) = @_;
2649
2650         return if !$volid;
2651
2652         $volhash->{$volid} = $is_cdrom || 0;
2653     };
2654
2655     foreach_drive($conf, sub {
2656         my ($ds, $drive) = @_;
2657         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2658     });
2659
2660     foreach my $snapname (keys %{$conf->{snapshots}}) {
2661         my $snap = $conf->{snapshots}->{$snapname};
2662         &$test_volid($snap->{vmstate}, 0);
2663         foreach_drive($snap, sub {
2664             my ($ds, $drive) = @_;
2665             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2666         });
2667     }
2668
2669     foreach my $volid (keys %$volhash) {
2670         &$func($volid, $volhash->{$volid});
2671     }
2672 }
2673
2674 sub vga_conf_has_spice {
2675     my ($vga) = @_;
2676
2677     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2678
2679     return $1 || 1;
2680 }
2681
2682 sub config_to_command {
2683     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2684
2685     my $cmd = [];
2686     my $globalFlags = [];
2687     my $machineFlags = [];
2688     my $rtcFlags = [];
2689     my $cpuFlags = [];
2690     my $devices = [];
2691     my $pciaddr = '';
2692     my $bridges = {};
2693     my $kvmver = kvm_user_version();
2694     my $vernum = 0; # unknown
2695     my $ostype = $conf->{ostype};
2696     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2697         $vernum = $1*1000000+$2*1000;
2698     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2699         $vernum = $1*1000000+$2*1000+$3;
2700     }
2701
2702     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2703
2704     my $have_ovz = -f '/proc/vz/vestat';
2705
2706     my $q35 = machine_type_is_q35($conf);
2707     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2708     my $machine_type = $forcemachine || $conf->{machine};
2709     my $use_old_bios_files = undef;
2710     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2711
2712     my $cpuunits = defined($conf->{cpuunits}) ?
2713             $conf->{cpuunits} : $defaults->{cpuunits};
2714
2715     push @$cmd, '/usr/bin/systemd-run';
2716     push @$cmd, '--scope';
2717     push @$cmd, '--slice', "qemu";
2718     push @$cmd, '--unit', $vmid;
2719     # set KillMode=none, so that systemd don't kill those scopes
2720     # at shutdown (pve-manager service should stop the VMs instead)
2721     push @$cmd, '-p', "KillMode=none";
2722     push @$cmd, '-p', "CPUShares=$cpuunits";
2723     if ($conf->{cpulimit}) {
2724         my $cpulimit = int($conf->{cpulimit} * 100);
2725         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2726     }
2727
2728     push @$cmd, '/usr/bin/kvm';
2729
2730     push @$cmd, '-id', $vmid;
2731
2732     my $use_virtio = 0;
2733
2734     my $qmpsocket = qmp_socket($vmid);
2735     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2736     push @$cmd, '-mon', "chardev=qmp,mode=control";
2737
2738
2739     push @$cmd, '-pidfile' , pidfile_name($vmid);
2740
2741     push @$cmd, '-daemonize';
2742
2743     if ($conf->{smbios1}) {
2744         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2745     }
2746
2747     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2748         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2749         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2750         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2751         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2752         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2753         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2754     }
2755
2756     if ($q35) {
2757         # the q35 chipset support native usb2, so we enable usb controller
2758         # by default for this machine type
2759         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2760     } else {
2761         $pciaddr = print_pci_addr("piix3", $bridges);
2762         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2763
2764         my $use_usb2 = 0;
2765         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2766             next if !$conf->{"usb$i"};
2767             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2768             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2769             $use_usb2 = 1;
2770         }
2771         # include usb device config
2772         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2773     }
2774
2775     # add usb3 controller if needed
2776
2777     my $use_usb3 = 0;
2778     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2779         next if !$conf->{"usb$i"};
2780         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2781         next if !$d || !$d->{usb3};
2782         $use_usb3 = 1;
2783     }
2784
2785     $pciaddr = print_pci_addr("xhci", $bridges);
2786     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2787
2788     my $vga = $conf->{vga};
2789
2790     my $qxlnum = vga_conf_has_spice($vga);
2791     $vga = 'qxl' if $qxlnum;
2792
2793     if (!$vga) {
2794         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2795                                 $conf->{ostype} eq 'win7' ||
2796                                 $conf->{ostype} eq 'w2k8')) {
2797             $vga = 'std';
2798         } else {
2799             $vga = 'cirrus';
2800         }
2801     }
2802
2803     # enable absolute mouse coordinates (needed by vnc)
2804     my $tablet;
2805     if (defined($conf->{tablet})) {
2806         $tablet = $conf->{tablet};
2807     } else {
2808         $tablet = $defaults->{tablet};
2809         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2810         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2811     }
2812
2813     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2814
2815     my $kvm_off = 0;
2816     # host pci devices
2817     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2818         my $d = parse_hostpci($conf->{"hostpci$i"});
2819         next if !$d;
2820
2821         my $pcie = $d->{pcie};
2822         if($pcie){
2823             die "q35 machine model is not enabled" if !$q35;
2824             $pciaddr = print_pcie_addr("hostpci$i");
2825         }else{
2826             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2827         }
2828
2829         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2830         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2831         if ($xvga && $xvga ne '') {
2832             $kvm_off = 1;
2833             $vga = 'none';
2834             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2835                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2836             }
2837             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2838                 $xvga = "";
2839             }
2840         }
2841         my $pcidevices = $d->{pciid};
2842         my $multifunction = 1 if @$pcidevices > 1;
2843
2844         my $j=0;
2845         foreach my $pcidevice (@$pcidevices) {
2846
2847             my $id = "hostpci$i";
2848             $id .= ".$j" if $multifunction;
2849             my $addr = $pciaddr;
2850             $addr .= ".$j" if $multifunction;
2851             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2852
2853             if($j == 0){
2854                 $devicestr .= "$rombar$xvga";
2855                 $devicestr .= ",multifunction=on" if $multifunction;
2856             }
2857
2858             push @$devices, '-device', $devicestr;
2859             $j++;
2860         }
2861     }
2862
2863     # usb devices
2864     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2865         next if !$conf->{"usb$i"};
2866         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2867         next if !$d;
2868
2869         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2870         my $usbbus = '';
2871         if (defined($d->{usb3}) && $d->{usb3}) {
2872             $usbbus = ',bus=xhci.0';
2873         }
2874
2875         if (defined($d->{host})) {
2876             $d = parse_usb_device($d->{host});
2877             if (defined($d->{vendorid}) && defined($d->{productid})) {
2878                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2879             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2880                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2881             } elsif (defined($d->{spice}) && $d->{spice}) {
2882                 # usb redir support for spice, currently no usb3
2883                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2884                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2885             }
2886         }
2887     }
2888
2889     # serial devices
2890     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2891         if (my $path = $conf->{"serial$i"}) {
2892             if ($path eq 'socket') {
2893                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2894                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2895                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2896             } else {
2897                 die "no such serial device\n" if ! -c $path;
2898                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2899                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2900             }
2901         }
2902     }
2903
2904     # parallel devices
2905     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2906         if (my $path = $conf->{"parallel$i"}) {
2907             die "no such parallel device\n" if ! -c $path;
2908             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2909             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2910             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2911         }
2912     }
2913
2914     my $vmname = $conf->{name} || "vm$vmid";
2915
2916     push @$cmd, '-name', $vmname;
2917
2918     my $sockets = 1;
2919     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2920     $sockets = $conf->{sockets} if  $conf->{sockets};
2921
2922     my $cores = $conf->{cores} || 1;
2923
2924     my $maxcpus = $sockets * $cores;
2925
2926     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2927
2928     my $allowed_vcpus = $cpuinfo->{cpus};
2929
2930     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2931         if ($allowed_vcpus < $maxcpus);
2932
2933     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2934
2935     push @$cmd, '-nodefaults';
2936
2937     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2938
2939     my $bootindex_hash = {};
2940     my $i = 1;
2941     foreach my $o (split(//, $bootorder)) {
2942         $bootindex_hash->{$o} = $i*100;
2943         $i++;
2944     }
2945
2946     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2947
2948     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2949
2950     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2951
2952     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2953
2954     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2955         my $socket = vnc_socket($vmid);
2956         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2957     } else {
2958         push @$cmd, '-nographic';
2959     }
2960
2961     # time drift fix
2962     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2963
2964     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2965     my $useLocaltime = $conf->{localtime};
2966
2967     if ($ostype) {
2968         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2969
2970         if ($ostype =~ m/^w/) { # windows
2971             $useLocaltime = 1 if !defined($conf->{localtime});
2972
2973             # use time drift fix when acpi is enabled
2974             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2975                 $tdf = 1 if !defined($conf->{tdf});
2976             }
2977         }
2978
2979         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2980             $ostype eq 'wvista') {
2981             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2982             push @$cmd, '-no-hpet';
2983             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2984                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2985                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2986                 push @$cpuFlags , 'hv_time' if !$nokvm;
2987
2988             } else {
2989                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2990             }
2991         }
2992
2993         if ($ostype eq 'win7' || $ostype eq 'win8') {
2994             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2995         }
2996     }
2997
2998     push @$rtcFlags, 'driftfix=slew' if $tdf;
2999
3000     if ($nokvm) {
3001         push @$machineFlags, 'accel=tcg';
3002     } else {
3003         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3004     }
3005
3006     if ($machine_type) {
3007         push @$machineFlags, "type=${machine_type}";
3008     }
3009
3010     if ($conf->{startdate}) {
3011         push @$rtcFlags, "base=$conf->{startdate}";
3012     } elsif ($useLocaltime) {
3013         push @$rtcFlags, 'base=localtime';
3014     }
3015
3016     my $cpu = $nokvm ? "qemu64" : "kvm64";
3017     if (my $cputype = $conf->{cpu}) {
3018         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3019             or die "Cannot parse cpu description: $cputype\n";
3020         $cpu = $cpuconf->{cputype};
3021         $kvm_off = 1 if $cpuconf->{hidden};
3022     }
3023
3024     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3025
3026     push @$cpuFlags , '-x2apic'
3027         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3028
3029     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3030
3031     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3032
3033     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3034
3035         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3036         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3037     }
3038
3039     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3040
3041     push @$cpuFlags, 'kvm=off' if $kvm_off;
3042
3043     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3044
3045     push @$cmd, '-cpu', $cpu;
3046
3047     my $memory = $conf->{memory} || $defaults->{memory};
3048     my $static_memory = 0;
3049     my $dimm_memory = 0;
3050
3051     if ($hotplug_features->{memory}) {
3052         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3053         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3054         $static_memory = $STATICMEM;
3055         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3056         $dimm_memory = $memory - $static_memory;
3057         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3058
3059     } else {
3060
3061         $static_memory = $memory;
3062         push @$cmd, '-m', $static_memory;
3063     }
3064
3065     if ($conf->{numa}) {
3066
3067         my $numa_totalmemory = undef;
3068         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3069             next if !$conf->{"numa$i"};
3070             my $numa = parse_numa($conf->{"numa$i"});
3071             next if !$numa;
3072             # memory
3073             die "missing numa node$i memory value\n" if !$numa->{memory};
3074             my $numa_memory = $numa->{memory};
3075             $numa_totalmemory += $numa_memory;
3076             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3077
3078             # cpus
3079             my $cpus_start = $numa->{cpus}->{start};
3080             die "missing numa node$i cpus\n" if !defined($cpus_start);
3081             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3082             my $cpus = $cpus_start;
3083             if (defined($cpus_end)) {
3084                 $cpus .= "-$cpus_end";
3085                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3086             }
3087
3088             # hostnodes
3089             my $hostnodes_start = $numa->{hostnodes}->{start};
3090             if (defined($hostnodes_start)) {
3091                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3092                 my $hostnodes = $hostnodes_start;
3093                 if (defined($hostnodes_end)) {
3094                     $hostnodes .= "-$hostnodes_end";
3095                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3096                 }
3097
3098                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3099                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3100                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3101                 }
3102
3103                 # policy
3104                 my $policy = $numa->{policy};
3105                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3106                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3107             }
3108
3109             push @$cmd, '-object', $numa_object;
3110             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3111         }
3112
3113         die "total memory for NUMA nodes must be equal to vm static memory\n"
3114             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3115
3116         #if no custom tology, we split memory and cores across numa nodes
3117         if(!$numa_totalmemory) {
3118
3119             my $numa_memory = ($static_memory / $sockets) . "M";
3120
3121             for (my $i = 0; $i < $sockets; $i++)  {
3122
3123                 my $cpustart = ($cores * $i);
3124                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3125                 my $cpus = $cpustart;
3126                 $cpus .= "-$cpuend" if $cpuend;
3127
3128                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3129                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3130             }
3131         }
3132     }
3133
3134     if ($hotplug_features->{memory}) {
3135         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3136             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3137             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3138             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3139
3140             #if dimm_memory is not aligned to dimm map
3141             if($current_size > $memory) {
3142                  $conf->{memory} = $current_size;
3143                  PVE::QemuConfig->write_config($vmid, $conf);
3144             }
3145         });
3146     }
3147
3148     push @$cmd, '-S' if $conf->{freeze};
3149
3150     # set keyboard layout
3151     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3152     push @$cmd, '-k', $kb if $kb;
3153
3154     # enable sound
3155     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3156     #push @$cmd, '-soundhw', 'es1370';
3157     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3158
3159     if($conf->{agent}) {
3160         my $qgasocket = qmp_socket($vmid, 1);
3161         my $pciaddr = print_pci_addr("qga0", $bridges);
3162         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3163         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3164         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3165     }
3166
3167     my $spice_port;
3168
3169     if ($qxlnum) {
3170         if ($qxlnum > 1) {
3171             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3172                 for(my $i = 1; $i < $qxlnum; $i++){
3173                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3174                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3175                 }
3176             } else {
3177                 # assume other OS works like Linux
3178                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3179                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3180             }
3181         }
3182
3183         my $pciaddr = print_pci_addr("spice", $bridges);
3184
3185         my $nodename = PVE::INotify::nodename();
3186         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3187         $spice_port = PVE::Tools::next_spice_port($pfamily);
3188
3189         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3190
3191         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3192         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3193         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3194     }
3195
3196     # enable balloon by default, unless explicitly disabled
3197     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3198         $pciaddr = print_pci_addr("balloon0", $bridges);
3199         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3200     }
3201
3202     if ($conf->{watchdog}) {
3203         my $wdopts = parse_watchdog($conf->{watchdog});
3204         $pciaddr = print_pci_addr("watchdog", $bridges);
3205         my $watchdog = $wdopts->{model} || 'i6300esb';
3206         push @$devices, '-device', "$watchdog$pciaddr";
3207         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3208     }
3209
3210     my $vollist = [];
3211     my $scsicontroller = {};
3212     my $ahcicontroller = {};
3213     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3214
3215     # Add iscsi initiator name if available
3216     if (my $initiator = get_initiator_name()) {
3217         push @$devices, '-iscsi', "initiator-name=$initiator";
3218     }
3219
3220     foreach_drive($conf, sub {
3221         my ($ds, $drive) = @_;
3222
3223         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3224             push @$vollist, $drive->{file};
3225         }
3226
3227         $use_virtio = 1 if $ds =~ m/^virtio/;
3228
3229         if (drive_is_cdrom ($drive)) {
3230             if ($bootindex_hash->{d}) {
3231                 $drive->{bootindex} = $bootindex_hash->{d};
3232                 $bootindex_hash->{d} += 1;
3233             }
3234         } else {
3235             if ($bootindex_hash->{c}) {
3236                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3237                 $bootindex_hash->{c} += 1;
3238             }
3239         }
3240
3241         if($drive->{interface} eq 'virtio'){
3242            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3243         }
3244
3245         if ($drive->{interface} eq 'scsi') {
3246
3247             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3248
3249             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3250             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3251
3252             my $iothread = '';
3253             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3254                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3255                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3256             }
3257
3258             my $queues = '';
3259             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3260                 $queues = ",num_queues=$drive->{queues}";
3261             }
3262
3263             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3264             $scsicontroller->{$controller}=1;
3265         }
3266
3267         if ($drive->{interface} eq 'sata') {
3268            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3269            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3270            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3271            $ahcicontroller->{$controller}=1;
3272         }
3273
3274         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3275         push @$devices, '-drive',$drive_cmd;
3276         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3277     });
3278
3279     for (my $i = 0; $i < $MAX_NETS; $i++) {
3280          next if !$conf->{"net$i"};
3281          my $d = parse_net($conf->{"net$i"});
3282          next if !$d;
3283
3284          $use_virtio = 1 if $d->{model} eq 'virtio';
3285
3286          if ($bootindex_hash->{n}) {
3287             $d->{bootindex} = $bootindex_hash->{n};
3288             $bootindex_hash->{n} += 1;
3289          }
3290
3291          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3292          push @$devices, '-netdev', $netdevfull;
3293
3294          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3295          push @$devices, '-device', $netdevicefull;
3296     }
3297
3298     if (!$q35) {
3299         # add pci bridges
3300         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3301            $bridges->{1} = 1;
3302            $bridges->{2} = 1;
3303         }
3304
3305         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3306
3307         while (my ($k, $v) = each %$bridges) {
3308             $pciaddr = print_pci_addr("pci.$k");
3309             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3310         }
3311     }
3312
3313     # add custom args
3314     if ($conf->{args}) {
3315         my $aa = PVE::Tools::split_args($conf->{args});
3316         push @$cmd, @$aa;
3317     }
3318
3319     push @$cmd, @$devices;
3320     push @$cmd, '-rtc', join(',', @$rtcFlags)
3321         if scalar(@$rtcFlags);
3322     push @$cmd, '-machine', join(',', @$machineFlags)
3323         if scalar(@$machineFlags);
3324     push @$cmd, '-global', join(',', @$globalFlags)
3325         if scalar(@$globalFlags);
3326
3327     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3328 }
3329
3330 sub vnc_socket {
3331     my ($vmid) = @_;
3332     return "${var_run_tmpdir}/$vmid.vnc";
3333 }
3334
3335 sub spice_port {
3336     my ($vmid) = @_;
3337
3338     my $res = vm_mon_cmd($vmid, 'query-spice');
3339
3340     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3341 }
3342
3343 sub qmp_socket {
3344     my ($vmid, $qga) = @_;
3345     my $sockettype = $qga ? 'qga' : 'qmp';
3346     return "${var_run_tmpdir}/$vmid.$sockettype";
3347 }
3348
3349 sub pidfile_name {
3350     my ($vmid) = @_;
3351     return "${var_run_tmpdir}/$vmid.pid";
3352 }
3353
3354 sub vm_devices_list {
3355     my ($vmid) = @_;
3356
3357     my $res = vm_mon_cmd($vmid, 'query-pci');
3358     my $devices = {};
3359     foreach my $pcibus (@$res) {
3360         foreach my $device (@{$pcibus->{devices}}) {
3361             next if !$device->{'qdev_id'};
3362             if ($device->{'pci_bridge'}) {
3363                 $devices->{$device->{'qdev_id'}} = 1;
3364                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3365                     next if !$bridge_device->{'qdev_id'};
3366                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3367                     $devices->{$device->{'qdev_id'}}++;
3368                 }
3369             } else {
3370                 $devices->{$device->{'qdev_id'}} = 1;
3371             }
3372         }
3373     }
3374
3375     my $resblock = vm_mon_cmd($vmid, 'query-block');
3376     foreach my $block (@$resblock) {
3377         if($block->{device} =~ m/^drive-(\S+)/){
3378                 $devices->{$1} = 1;
3379         }
3380     }
3381
3382     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3383     foreach my $mice (@$resmice) {
3384         if ($mice->{name} eq 'QEMU HID Tablet') {
3385             $devices->{tablet} = 1;
3386             last;
3387         }
3388     }
3389
3390     return $devices;
3391 }
3392
3393 sub vm_deviceplug {
3394     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3395
3396     my $q35 = machine_type_is_q35($conf);
3397
3398     my $devices_list = vm_devices_list($vmid);
3399     return 1 if defined($devices_list->{$deviceid});
3400
3401     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3402
3403     if ($deviceid eq 'tablet') {
3404
3405         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3406
3407     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3408
3409         qemu_iothread_add($vmid, $deviceid, $device);
3410
3411         qemu_driveadd($storecfg, $vmid, $device);
3412         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3413
3414         qemu_deviceadd($vmid, $devicefull);
3415         eval { qemu_deviceaddverify($vmid, $deviceid); };
3416         if (my $err = $@) {
3417             eval { qemu_drivedel($vmid, $deviceid); };
3418             warn $@ if $@;
3419             die $err;
3420         }
3421
3422     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3423
3424
3425         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3426         my $pciaddr = print_pci_addr($deviceid);
3427         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3428
3429         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3430
3431         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3432             qemu_iothread_add($vmid, $deviceid, $device);
3433             $devicefull .= ",iothread=iothread-$deviceid";
3434         }
3435
3436         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3437             $devicefull .= ",num_queues=$device->{queues}";
3438         }
3439
3440         qemu_deviceadd($vmid, $devicefull);
3441         qemu_deviceaddverify($vmid, $deviceid);
3442
3443     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3444
3445         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3446         qemu_driveadd($storecfg, $vmid, $device);
3447
3448         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3449         eval { qemu_deviceadd($vmid, $devicefull); };
3450         if (my $err = $@) {
3451             eval { qemu_drivedel($vmid, $deviceid); };
3452             warn $@ if $@;
3453             die $err;
3454         }
3455
3456     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3457
3458         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3459
3460         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3461         my $use_old_bios_files = undef;
3462         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3463
3464         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3465         qemu_deviceadd($vmid, $netdevicefull);
3466         eval { qemu_deviceaddverify($vmid, $deviceid); };
3467         if (my $err = $@) {
3468             eval { qemu_netdevdel($vmid, $deviceid); };
3469             warn $@ if $@;
3470             die $err;
3471         }
3472
3473     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3474
3475         my $bridgeid = $2;
3476         my $pciaddr = print_pci_addr($deviceid);
3477         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3478
3479         qemu_deviceadd($vmid, $devicefull);
3480         qemu_deviceaddverify($vmid, $deviceid);
3481
3482     } else {
3483         die "can't hotplug device '$deviceid'\n";
3484     }
3485
3486     return 1;
3487 }
3488
3489 # fixme: this should raise exceptions on error!
3490 sub vm_deviceunplug {
3491     my ($vmid, $conf, $deviceid) = @_;
3492
3493     my $devices_list = vm_devices_list($vmid);
3494     return 1 if !defined($devices_list->{$deviceid});
3495
3496     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3497
3498     if ($deviceid eq 'tablet') {
3499
3500         qemu_devicedel($vmid, $deviceid);
3501
3502     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3503
3504         qemu_devicedel($vmid, $deviceid);
3505         qemu_devicedelverify($vmid, $deviceid);
3506         qemu_drivedel($vmid, $deviceid);
3507         qemu_iothread_del($conf, $vmid, $deviceid);
3508
3509     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3510
3511         qemu_devicedel($vmid, $deviceid);
3512         qemu_devicedelverify($vmid, $deviceid);
3513         qemu_iothread_del($conf, $vmid, $deviceid);
3514
3515     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3516
3517         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3518         my $device = parse_drive($deviceid, $conf->{$deviceid});
3519         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_drivedel($vmid, $deviceid);
3523         qemu_deletescsihw($conf, $vmid, $deviceid);
3524
3525     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3526
3527         qemu_devicedel($vmid, $deviceid);
3528         qemu_devicedelverify($vmid, $deviceid);
3529         qemu_netdevdel($vmid, $deviceid);
3530
3531     } else {
3532         die "can't unplug device '$deviceid'\n";
3533     }
3534
3535     return 1;
3536 }
3537
3538 sub qemu_deviceadd {
3539     my ($vmid, $devicefull) = @_;
3540
3541     $devicefull = "driver=".$devicefull;
3542     my %options =  split(/[=,]/, $devicefull);
3543
3544     vm_mon_cmd($vmid, "device_add" , %options);
3545 }
3546
3547 sub qemu_devicedel {
3548     my ($vmid, $deviceid) = @_;
3549
3550     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3551 }
3552
3553 sub qemu_iothread_add {
3554     my($vmid, $deviceid, $device) = @_;
3555
3556     if ($device->{iothread}) {
3557         my $iothreads = vm_iothreads_list($vmid);
3558         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3559     }
3560 }
3561
3562 sub qemu_iothread_del {
3563     my($conf, $vmid, $deviceid) = @_;
3564
3565     my $device = parse_drive($deviceid, $conf->{$deviceid});
3566     if ($device->{iothread}) {
3567         my $iothreads = vm_iothreads_list($vmid);
3568         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3569     }
3570 }
3571
3572 sub qemu_objectadd {
3573     my($vmid, $objectid, $qomtype) = @_;
3574
3575     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3576
3577     return 1;
3578 }
3579
3580 sub qemu_objectdel {
3581     my($vmid, $objectid) = @_;
3582
3583     vm_mon_cmd($vmid, "object-del", id => $objectid);
3584
3585     return 1;
3586 }
3587
3588 sub qemu_driveadd {
3589     my ($storecfg, $vmid, $device) = @_;
3590
3591     my $drive = print_drive_full($storecfg, $vmid, $device);
3592     $drive =~ s/\\/\\\\/g;
3593     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3594
3595     # If the command succeeds qemu prints: "OK"
3596     return 1 if $ret =~ m/OK/s;
3597
3598     die "adding drive failed: $ret\n";
3599 }
3600
3601 sub qemu_drivedel {
3602     my($vmid, $deviceid) = @_;
3603
3604     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3605     $ret =~ s/^\s+//;
3606
3607     return 1 if $ret eq "";
3608
3609     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3610     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3611
3612     die "deleting drive $deviceid failed : $ret\n";
3613 }
3614
3615 sub qemu_deviceaddverify {
3616     my ($vmid, $deviceid) = @_;
3617
3618     for (my $i = 0; $i <= 5; $i++) {
3619          my $devices_list = vm_devices_list($vmid);
3620          return 1 if defined($devices_list->{$deviceid});
3621          sleep 1;
3622     }
3623
3624     die "error on hotplug device '$deviceid'\n";
3625 }
3626
3627
3628 sub qemu_devicedelverify {
3629     my ($vmid, $deviceid) = @_;
3630
3631     # need to verify that the device is correctly removed as device_del
3632     # is async and empty return is not reliable
3633
3634     for (my $i = 0; $i <= 5; $i++) {
3635          my $devices_list = vm_devices_list($vmid);
3636          return 1 if !defined($devices_list->{$deviceid});
3637          sleep 1;
3638     }
3639
3640     die "error on hot-unplugging device '$deviceid'\n";
3641 }
3642
3643 sub qemu_findorcreatescsihw {
3644     my ($storecfg, $conf, $vmid, $device) = @_;
3645
3646     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3647
3648     my $scsihwid="$controller_prefix$controller";
3649     my $devices_list = vm_devices_list($vmid);
3650
3651     if(!defined($devices_list->{$scsihwid})) {
3652         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3653     }
3654
3655     return 1;
3656 }
3657
3658 sub qemu_deletescsihw {
3659     my ($conf, $vmid, $opt) = @_;
3660
3661     my $device = parse_drive($opt, $conf->{$opt});
3662
3663     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3664         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3665         return 1;
3666     }
3667
3668     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3669
3670     my $devices_list = vm_devices_list($vmid);
3671     foreach my $opt (keys %{$devices_list}) {
3672         if (PVE::QemuServer::is_valid_drivename($opt)) {
3673             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3674             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3675                 return 1;
3676             }
3677         }
3678     }
3679
3680     my $scsihwid="scsihw$controller";
3681
3682     vm_deviceunplug($vmid, $conf, $scsihwid);
3683
3684     return 1;
3685 }
3686
3687 sub qemu_add_pci_bridge {
3688     my ($storecfg, $conf, $vmid, $device) = @_;
3689
3690     my $bridges = {};
3691
3692     my $bridgeid;
3693
3694     print_pci_addr($device, $bridges);
3695
3696     while (my ($k, $v) = each %$bridges) {
3697         $bridgeid = $k;
3698     }
3699     return 1 if !defined($bridgeid) || $bridgeid < 1;
3700
3701     my $bridge = "pci.$bridgeid";
3702     my $devices_list = vm_devices_list($vmid);
3703
3704     if (!defined($devices_list->{$bridge})) {
3705         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3706     }
3707
3708     return 1;
3709 }
3710
3711 sub qemu_set_link_status {
3712     my ($vmid, $device, $up) = @_;
3713
3714     vm_mon_cmd($vmid, "set_link", name => $device,
3715                up => $up ? JSON::true : JSON::false);
3716 }
3717
3718 sub qemu_netdevadd {
3719     my ($vmid, $conf, $device, $deviceid) = @_;
3720
3721     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3722     my %options =  split(/[=,]/, $netdev);
3723
3724     vm_mon_cmd($vmid, "netdev_add",  %options);
3725     return 1;
3726 }
3727
3728 sub qemu_netdevdel {
3729     my ($vmid, $deviceid) = @_;
3730
3731     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3732 }
3733
3734 sub qemu_cpu_hotplug {
3735     my ($vmid, $conf, $vcpus) = @_;
3736
3737     my $sockets = 1;
3738     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3739     $sockets = $conf->{sockets} if  $conf->{sockets};
3740     my $cores = $conf->{cores} || 1;
3741     my $maxcpus = $sockets * $cores;
3742
3743     $vcpus = $maxcpus if !$vcpus;
3744
3745     die "you can't add more vcpus than maxcpus\n"
3746         if $vcpus > $maxcpus;
3747
3748     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3749     die "online cpu unplug is not yet possible\n"
3750         if $vcpus < $currentvcpus;
3751
3752     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3753     die "vcpus in running vm is different than configuration\n"
3754         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3755
3756     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3757         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3758     }
3759 }
3760
3761 sub qemu_memory_hotplug {
3762     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3763
3764     return $value if !check_running($vmid);
3765
3766     my $memory = $conf->{memory} || $defaults->{memory};
3767     $value = $defaults->{memory} if !$value;
3768     return $value if $value == $memory;
3769
3770     my $static_memory = $STATICMEM;
3771     my $dimm_memory = $memory - $static_memory;
3772
3773     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3774     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3775
3776
3777     my $sockets = 1;
3778     $sockets = $conf->{sockets} if $conf->{sockets};
3779
3780     if($value > $memory) {
3781
3782         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3783             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3784
3785                 return if $current_size <= $conf->{memory};
3786
3787                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3788                 if (my $err = $@) {
3789                     eval { qemu_objectdel($vmid, "mem-$name"); };
3790                     die $err;
3791                 }
3792
3793                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3794                 if (my $err = $@) {
3795                     eval { qemu_objectdel($vmid, "mem-$name"); };
3796                     die $err;
3797                 }
3798                 #update conf after each succesful module hotplug
3799                 $conf->{memory} = $current_size;
3800                 PVE::QemuConfig->write_config($vmid, $conf);
3801         });
3802
3803     } else {
3804
3805         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3806             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3807
3808                 return if $current_size >= $conf->{memory};
3809                 print "try to unplug memory dimm $name\n";
3810
3811                 my $retry = 0;
3812                 while (1) {
3813                     eval { qemu_devicedel($vmid, $name) };
3814                     sleep 3;
3815                     my $dimm_list = qemu_dimm_list($vmid);
3816                     last if !$dimm_list->{$name};
3817                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3818                     $retry++;
3819                 }
3820
3821                 #update conf after each succesful module unplug
3822                 $conf->{memory} = $current_size;
3823
3824                 eval { qemu_objectdel($vmid, "mem-$name"); };
3825                 PVE::QemuConfig->write_config($vmid, $conf);
3826         });
3827     }
3828 }
3829
3830 sub qemu_dimm_list {
3831     my ($vmid) = @_;
3832
3833     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3834     my $dimms = {};
3835
3836     foreach my $dimm (@$dimmarray) {
3837
3838         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3839         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3840         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3841         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3842         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3843     }
3844     return $dimms;
3845 }
3846
3847 sub qemu_block_set_io_throttle {
3848     my ($vmid, $deviceid,
3849         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3850         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3851
3852     return if !check_running($vmid) ;
3853
3854     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3855         bps => int($bps),
3856         bps_rd => int($bps_rd),
3857         bps_wr => int($bps_wr),
3858         iops => int($iops),
3859         iops_rd => int($iops_rd),
3860         iops_wr => int($iops_wr),
3861         bps_max => int($bps_max),
3862         bps_rd_max => int($bps_rd_max),
3863         bps_wr_max => int($bps_wr_max),
3864         iops_max => int($iops_max),
3865         iops_rd_max => int($iops_rd_max),
3866         iops_wr_max => int($iops_wr_max)
3867     );
3868
3869 }
3870
3871 # old code, only used to shutdown old VM after update
3872 sub __read_avail {
3873     my ($fh, $timeout) = @_;
3874
3875     my $sel = new IO::Select;
3876     $sel->add($fh);
3877
3878     my $res = '';
3879     my $buf;
3880
3881     my @ready;
3882     while (scalar (@ready = $sel->can_read($timeout))) {
3883         my $count;
3884         if ($count = $fh->sysread($buf, 8192)) {
3885             if ($buf =~ /^(.*)\(qemu\) $/s) {
3886                 $res .= $1;
3887                 last;
3888             } else {
3889                 $res .= $buf;
3890             }
3891         } else {
3892             if (!defined($count)) {
3893                 die "$!\n";
3894             }
3895             last;
3896         }
3897     }
3898
3899     die "monitor read timeout\n" if !scalar(@ready);
3900
3901     return $res;
3902 }
3903
3904 # old code, only used to shutdown old VM after update
3905 sub vm_monitor_command {
3906     my ($vmid, $cmdstr, $nocheck) = @_;
3907
3908     my $res;
3909
3910     eval {
3911         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3912
3913         my $sname = "${var_run_tmpdir}/$vmid.mon";
3914
3915         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3916             die "unable to connect to VM $vmid socket - $!\n";
3917
3918         my $timeout = 3;
3919
3920         # hack: migrate sometime blocks the monitor (when migrate_downtime
3921         # is set)
3922         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3923             $timeout = 60*60; # 1 hour
3924         }
3925
3926         # read banner;
3927         my $data = __read_avail($sock, $timeout);
3928
3929         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3930             die "got unexpected qemu monitor banner\n";
3931         }
3932
3933         my $sel = new IO::Select;
3934         $sel->add($sock);
3935
3936         if (!scalar(my @ready = $sel->can_write($timeout))) {
3937             die "monitor write error - timeout";
3938         }
3939
3940         my $fullcmd = "$cmdstr\r";
3941
3942         # syslog('info', "VM $vmid monitor command: $cmdstr");
3943
3944         my $b;
3945         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3946             die "monitor write error - $!";
3947         }
3948
3949         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3950
3951         $timeout = 20;
3952
3953         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3954             $timeout = 60*60; # 1 hour
3955         } elsif ($cmdstr =~ m/^(eject|change)/) {
3956             $timeout = 60; # note: cdrom mount command is slow
3957         }
3958         if ($res = __read_avail($sock, $timeout)) {
3959
3960             my @lines = split("\r?\n", $res);
3961
3962             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3963
3964             $res = join("\n", @lines);
3965             $res .= "\n";
3966         }
3967     };
3968
3969     my $err = $@;
3970
3971     if ($err) {
3972         syslog("err", "VM $vmid monitor command failed - $err");
3973         die $err;
3974     }
3975
3976     return $res;
3977 }
3978
3979 sub qemu_block_resize {
3980     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3981
3982     my $running = check_running($vmid);
3983
3984     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3985
3986     return if !$running;
3987
3988     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3989
3990 }
3991
3992 sub qemu_volume_snapshot {
3993     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3994
3995     my $running = check_running($vmid);
3996
3997     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3998         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3999     } else {
4000         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4001     }
4002 }
4003
4004 sub qemu_volume_snapshot_delete {
4005     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4006
4007     my $running = check_running($vmid);
4008
4009     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4010
4011     return if !$running;
4012
4013     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4014 }
4015
4016 sub set_migration_caps {
4017     my ($vmid) = @_;
4018
4019     my $cap_ref = [];
4020
4021     my $enabled_cap = {
4022         "auto-converge" => 1,
4023         "xbzrle" => 1,
4024         "x-rdma-pin-all" => 0,
4025         "zero-blocks" => 0,
4026         "compress" => 0
4027     };
4028
4029     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4030
4031     for my $supported_capability (@$supported_capabilities) {
4032         push @$cap_ref, {
4033             capability => $supported_capability->{capability},
4034             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4035         };
4036     }
4037
4038     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4039 }
4040
4041 my $fast_plug_option = {
4042     'lock' => 1,
4043     'name' => 1,
4044     'onboot' => 1,
4045     'shares' => 1,
4046     'startup' => 1,
4047     'description' => 1,
4048 };
4049
4050 # hotplug changes in [PENDING]
4051 # $selection hash can be used to only apply specified options, for
4052 # example: { cores => 1 } (only apply changed 'cores')
4053 # $errors ref is used to return error messages
4054 sub vmconfig_hotplug_pending {
4055     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4056
4057     my $defaults = load_defaults();
4058
4059     # commit values which do not have any impact on running VM first
4060     # Note: those option cannot raise errors, we we do not care about
4061     # $selection and always apply them.
4062
4063     my $add_error = sub {
4064         my ($opt, $msg) = @_;
4065         $errors->{$opt} = "hotplug problem - $msg";
4066     };
4067
4068     my $changes = 0;
4069     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4070         if ($fast_plug_option->{$opt}) {
4071             $conf->{$opt} = $conf->{pending}->{$opt};
4072             delete $conf->{pending}->{$opt};
4073             $changes = 1;
4074         }
4075     }
4076
4077     if ($changes) {
4078         PVE::QemuConfig->write_config($vmid, $conf);
4079         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4080     }
4081
4082     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4083
4084     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4085     while (my ($opt, $force) = each %$pending_delete_hash) {
4086         next if $selection && !$selection->{$opt};
4087         eval {
4088             if ($opt eq 'hotplug') {
4089                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4090             } elsif ($opt eq 'tablet') {
4091                 die "skip\n" if !$hotplug_features->{usb};
4092                 if ($defaults->{tablet}) {
4093                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4094                 } else {
4095                     vm_deviceunplug($vmid, $conf, $opt);
4096                 }
4097             } elsif ($opt eq 'vcpus') {
4098                 die "skip\n" if !$hotplug_features->{cpu};
4099                 qemu_cpu_hotplug($vmid, $conf, undef);
4100             } elsif ($opt eq 'balloon') {
4101                 # enable balloon device is not hotpluggable
4102                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4103             } elsif ($fast_plug_option->{$opt}) {
4104                 # do nothing
4105             } elsif ($opt =~ m/^net(\d+)$/) {
4106                 die "skip\n" if !$hotplug_features->{network};
4107                 vm_deviceunplug($vmid, $conf, $opt);
4108             } elsif (is_valid_drivename($opt)) {
4109                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4110                 vm_deviceunplug($vmid, $conf, $opt);
4111                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4112             } elsif ($opt =~ m/^memory$/) {
4113                 die "skip\n" if !$hotplug_features->{memory};
4114                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4115             } elsif ($opt eq 'cpuunits') {
4116                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4117             } elsif ($opt eq 'cpulimit') {
4118                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4119             } else {
4120                 die "skip\n";
4121             }
4122         };
4123         if (my $err = $@) {
4124             &$add_error($opt, $err) if $err ne "skip\n";
4125         } else {
4126             # save new config if hotplug was successful
4127             delete $conf->{$opt};
4128             vmconfig_undelete_pending_option($conf, $opt);
4129             PVE::QemuConfig->write_config($vmid, $conf);
4130             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4131         }
4132     }
4133
4134     foreach my $opt (keys %{$conf->{pending}}) {
4135         next if $selection && !$selection->{$opt};
4136         my $value = $conf->{pending}->{$opt};
4137         eval {
4138             if ($opt eq 'hotplug') {
4139                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4140             } elsif ($opt eq 'tablet') {
4141                 die "skip\n" if !$hotplug_features->{usb};
4142                 if ($value == 1) {
4143                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4144                 } elsif ($value == 0) {
4145                     vm_deviceunplug($vmid, $conf, $opt);
4146                 }
4147             } elsif ($opt eq 'vcpus') {
4148                 die "skip\n" if !$hotplug_features->{cpu};
4149                 qemu_cpu_hotplug($vmid, $conf, $value);
4150             } elsif ($opt eq 'balloon') {
4151                 # enable/disable balloning device is not hotpluggable
4152                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4153                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4154                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4155
4156                 # allow manual ballooning if shares is set to zero
4157                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4158                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4159                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4160                 }
4161             } elsif ($opt =~ m/^net(\d+)$/) {
4162                 # some changes can be done without hotplug
4163                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4164                                     $vmid, $opt, $value);
4165             } elsif (is_valid_drivename($opt)) {
4166                 # some changes can be done without hotplug
4167                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4168                                      $vmid, $opt, $value, 1);
4169             } elsif ($opt =~ m/^memory$/) { #dimms
4170                 die "skip\n" if !$hotplug_features->{memory};
4171                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4172             } elsif ($opt eq 'cpuunits') {
4173                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4174             } elsif ($opt eq 'cpulimit') {
4175                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4176                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4177             } else {
4178                 die "skip\n";  # skip non-hot-pluggable options
4179             }
4180         };
4181         if (my $err = $@) {
4182             &$add_error($opt, $err) if $err ne "skip\n";
4183         } else {
4184             # save new config if hotplug was successful
4185             $conf->{$opt} = $value;
4186             delete $conf->{pending}->{$opt};
4187             PVE::QemuConfig->write_config($vmid, $conf);
4188             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4189         }
4190     }
4191 }
4192
4193 sub try_deallocate_drive {
4194     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4195
4196     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4197         my $volid = $drive->{file};
4198         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4199             my $sid = PVE::Storage::parse_volume_id($volid);
4200             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4201
4202             # check if the disk is really unused
4203             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4204                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4205             PVE::Storage::vdisk_free($storecfg, $volid);
4206             return 1;
4207         } else {
4208             # If vm is not owner of this disk remove from config
4209             return 1;
4210         }
4211     }
4212
4213     return undef;
4214 }
4215
4216 sub vmconfig_delete_or_detach_drive {
4217     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4218
4219     my $drive = parse_drive($opt, $conf->{$opt});
4220
4221     my $rpcenv = PVE::RPCEnvironment::get();
4222     my $authuser = $rpcenv->get_user();
4223
4224     if ($force) {
4225         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4226         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4227     } else {
4228         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4229     }
4230 }
4231
4232 sub vmconfig_apply_pending {
4233     my ($vmid, $conf, $storecfg) = @_;
4234
4235     # cold plug
4236
4237     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4238     while (my ($opt, $force) = each %$pending_delete_hash) {
4239         die "internal error" if $opt =~ m/^unused/;
4240         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4241         if (!defined($conf->{$opt})) {
4242             vmconfig_undelete_pending_option($conf, $opt);
4243             PVE::QemuConfig->write_config($vmid, $conf);
4244         } elsif (is_valid_drivename($opt)) {
4245             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             delete $conf->{$opt};
4248             PVE::QemuConfig->write_config($vmid, $conf);
4249         } else {
4250             vmconfig_undelete_pending_option($conf, $opt);
4251             delete $conf->{$opt};
4252             PVE::QemuConfig->write_config($vmid, $conf);
4253         }
4254     }
4255
4256     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4257
4258     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4259         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4260
4261         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4262             # skip if nothing changed
4263         } elsif (is_valid_drivename($opt)) {
4264             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4265                 if defined($conf->{$opt});
4266             $conf->{$opt} = $conf->{pending}->{$opt};
4267         } else {
4268             $conf->{$opt} = $conf->{pending}->{$opt};
4269         }
4270
4271         delete $conf->{pending}->{$opt};
4272         PVE::QemuConfig->write_config($vmid, $conf);
4273     }
4274 }
4275
4276 my $safe_num_ne = sub {
4277     my ($a, $b) = @_;
4278
4279     return 0 if !defined($a) && !defined($b);
4280     return 1 if !defined($a);
4281     return 1 if !defined($b);
4282
4283     return $a != $b;
4284 };
4285
4286 my $safe_string_ne = sub {
4287     my ($a, $b) = @_;
4288
4289     return 0 if !defined($a) && !defined($b);
4290     return 1 if !defined($a);
4291     return 1 if !defined($b);
4292
4293     return $a ne $b;
4294 };
4295
4296 sub vmconfig_update_net {
4297     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4298
4299     my $newnet = parse_net($value);
4300
4301     if ($conf->{$opt}) {
4302         my $oldnet = parse_net($conf->{$opt});
4303
4304         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4305             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4306             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4307             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4308
4309             # for non online change, we try to hot-unplug
4310             die "skip\n" if !$hotplug;
4311             vm_deviceunplug($vmid, $conf, $opt);
4312         } else {
4313
4314             die "internal error" if $opt !~ m/net(\d+)/;
4315             my $iface = "tap${vmid}i$1";
4316
4317             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4318                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4319                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4320                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4321                 PVE::Network::tap_unplug($iface);
4322                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4323             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4324                 # Rate can be applied on its own but any change above needs to
4325                 # include the rate in tap_plug since OVS resets everything.
4326                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4327             }
4328
4329             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4330                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4331             }
4332
4333             return 1;
4334         }
4335     }
4336
4337     if ($hotplug) {
4338         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4339     } else {
4340         die "skip\n";
4341     }
4342 }
4343
4344 sub vmconfig_update_disk {
4345     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4346
4347     # fixme: do we need force?
4348
4349     my $drive = parse_drive($opt, $value);
4350
4351     if ($conf->{$opt}) {
4352
4353         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4354
4355             my $media = $drive->{media} || 'disk';
4356             my $oldmedia = $old_drive->{media} || 'disk';
4357             die "unable to change media type\n" if $media ne $oldmedia;
4358
4359             if (!drive_is_cdrom($old_drive)) {
4360
4361                 if ($drive->{file} ne $old_drive->{file}) {
4362
4363                     die "skip\n" if !$hotplug;
4364
4365                     # unplug and register as unused
4366                     vm_deviceunplug($vmid, $conf, $opt);
4367                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4368
4369                 } else {
4370                     # update existing disk
4371
4372                     # skip non hotpluggable value
4373                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4374                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4375                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4376                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4377                         die "skip\n";
4378                     }
4379
4380                     # apply throttle
4381                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4382                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4383                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4384                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4385                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4386                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4387                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4388                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4389                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4390                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4391                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4392                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4393
4394                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4395                                                    ($drive->{mbps} || 0)*1024*1024,
4396                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4397                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4398                                                    $drive->{iops} || 0,
4399                                                    $drive->{iops_rd} || 0,
4400                                                    $drive->{iops_wr} || 0,
4401                                                    ($drive->{mbps_max} || 0)*1024*1024,
4402                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4403                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4404                                                    $drive->{iops_max} || 0,
4405                                                    $drive->{iops_rd_max} || 0,
4406                                                    $drive->{iops_wr_max} || 0);
4407
4408                     }
4409
4410                     return 1;
4411                 }
4412
4413             } else { # cdrom
4414
4415                 if ($drive->{file} eq 'none') {
4416                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4417                 } else {
4418                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4419                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4420                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4421                 }
4422
4423                 return 1;
4424             }
4425         }
4426     }
4427
4428     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4429     # hotplug new disks
4430     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4431     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4432 }
4433
4434 sub vm_start {
4435     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4436         $forcemachine, $spice_ticket) = @_;
4437
4438     PVE::QemuConfig->lock_config($vmid, sub {
4439         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4440
4441         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4442
4443         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4444
4445         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4446
4447         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4448             vmconfig_apply_pending($vmid, $conf, $storecfg);
4449             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4450         }
4451
4452         my $defaults = load_defaults();
4453
4454         # set environment variable useful inside network script
4455         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4456
4457         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4458
4459         my $migrate_port = 0;
4460         my $migrate_uri;
4461         if ($statefile) {
4462             if ($statefile eq 'tcp') {
4463                 my $localip = "localhost";
4464                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4465                 my $nodename = PVE::INotify::nodename();
4466                 if ($datacenterconf->{migration_unsecure}) {
4467                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4468                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4469                 }
4470                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4471                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4472                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4473                 push @$cmd, '-incoming', $migrate_uri;
4474                 push @$cmd, '-S';
4475             } else {
4476                 push @$cmd, '-loadstate', $statefile;
4477             }
4478         } elsif ($paused) {
4479             push @$cmd, '-S';
4480         }
4481
4482         # host pci devices
4483         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4484           my $d = parse_hostpci($conf->{"hostpci$i"});
4485           next if !$d;
4486           my $pcidevices = $d->{pciid};
4487           foreach my $pcidevice (@$pcidevices) {
4488                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4489
4490                 my $info = pci_device_info("0000:$pciid");
4491                 die "IOMMU not present\n" if !check_iommu_support();
4492                 die "no pci device info for device '$pciid'\n" if !$info;
4493                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4494                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4495           }
4496         }
4497
4498         PVE::Storage::activate_volumes($storecfg, $vollist);
4499
4500         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4501                     umask => 0077); };
4502
4503         if (my $err = $@) {
4504             # deactivate volumes if start fails
4505             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4506             die "start failed: $err";
4507         }
4508
4509         print "migration listens on $migrate_uri\n" if $migrate_uri;
4510
4511         if ($statefile && $statefile ne 'tcp')  {
4512             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4513             warn $@ if $@;
4514         }
4515
4516         if ($migratedfrom) {
4517
4518             eval {
4519                 set_migration_caps($vmid);
4520             };
4521             warn $@ if $@;
4522
4523             if ($spice_port) {
4524                 print "spice listens on port $spice_port\n";
4525                 if ($spice_ticket) {
4526                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4527                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4528                 }
4529             }
4530
4531         } else {
4532
4533             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4534                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4535                     if $conf->{balloon};
4536             }
4537
4538             foreach my $opt (keys %$conf) {
4539                 next if $opt !~  m/^net\d+$/;
4540                 my $nicconf = parse_net($conf->{$opt});
4541                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4542             }
4543         }
4544
4545         vm_mon_cmd_nocheck($vmid, 'qom-set',
4546                     path => "machine/peripheral/balloon0",
4547                     property => "guest-stats-polling-interval",
4548                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4549
4550     });
4551 }
4552
4553 sub vm_mon_cmd {
4554     my ($vmid, $execute, %params) = @_;
4555
4556     my $cmd = { execute => $execute, arguments => \%params };
4557     vm_qmp_command($vmid, $cmd);
4558 }
4559
4560 sub vm_mon_cmd_nocheck {
4561     my ($vmid, $execute, %params) = @_;
4562
4563     my $cmd = { execute => $execute, arguments => \%params };
4564     vm_qmp_command($vmid, $cmd, 1);
4565 }
4566
4567 sub vm_qmp_command {
4568     my ($vmid, $cmd, $nocheck) = @_;
4569
4570     my $res;
4571
4572     my $timeout;
4573     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4574         $timeout = $cmd->{arguments}->{timeout};
4575         delete $cmd->{arguments}->{timeout};
4576     }
4577
4578     eval {
4579         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4580         my $sname = qmp_socket($vmid);
4581         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4582             my $qmpclient = PVE::QMPClient->new();
4583
4584             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4585         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4586             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4587                 if scalar(%{$cmd->{arguments}});
4588             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4589         } else {
4590             die "unable to open monitor socket\n";
4591         }
4592     };
4593     if (my $err = $@) {
4594         syslog("err", "VM $vmid qmp command failed - $err");
4595         die $err;
4596     }
4597
4598     return $res;
4599 }
4600
4601 sub vm_human_monitor_command {
4602     my ($vmid, $cmdline) = @_;
4603
4604     my $res;
4605
4606     my $cmd = {
4607         execute => 'human-monitor-command',
4608         arguments => { 'command-line' => $cmdline},
4609     };
4610
4611     return vm_qmp_command($vmid, $cmd);
4612 }
4613
4614 sub vm_commandline {
4615     my ($storecfg, $vmid) = @_;
4616
4617     my $conf = PVE::QemuConfig->load_config($vmid);
4618
4619     my $defaults = load_defaults();
4620
4621     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4622
4623     return join(' ', @$cmd);
4624 }
4625
4626 sub vm_reset {
4627     my ($vmid, $skiplock) = @_;
4628
4629     PVE::QemuConfig->lock_config($vmid, sub {
4630
4631         my $conf = PVE::QemuConfig->load_config($vmid);
4632
4633         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4634
4635         vm_mon_cmd($vmid, "system_reset");
4636     });
4637 }
4638
4639 sub get_vm_volumes {
4640     my ($conf) = @_;
4641
4642     my $vollist = [];
4643     foreach_volid($conf, sub {
4644         my ($volid, $is_cdrom) = @_;
4645
4646         return if $volid =~ m|^/|;
4647
4648         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4649         return if !$sid;
4650
4651         push @$vollist, $volid;
4652     });
4653
4654     return $vollist;
4655 }
4656
4657 sub vm_stop_cleanup {
4658     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4659
4660     eval {
4661
4662         if (!$keepActive) {
4663             my $vollist = get_vm_volumes($conf);
4664             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4665         }
4666
4667         foreach my $ext (qw(mon qmp pid vnc qga)) {
4668             unlink "/var/run/qemu-server/${vmid}.$ext";
4669         }
4670
4671         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4672     };
4673     warn $@ if $@; # avoid errors - just warn
4674 }
4675
4676 # Note: use $nockeck to skip tests if VM configuration file exists.
4677 # We need that when migration VMs to other nodes (files already moved)
4678 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4679 sub vm_stop {
4680     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4681
4682     $force = 1 if !defined($force) && !$shutdown;
4683
4684     if ($migratedfrom){
4685         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4686         kill 15, $pid if $pid;
4687         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4688         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4689         return;
4690     }
4691
4692     PVE::QemuConfig->lock_config($vmid, sub {
4693
4694         my $pid = check_running($vmid, $nocheck);
4695         return if !$pid;
4696
4697         my $conf;
4698         if (!$nocheck) {
4699             $conf = PVE::QemuConfig->load_config($vmid);
4700             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4701             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4702                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4703                 $timeout = $opts->{down} if $opts->{down};
4704             }
4705         }
4706
4707         $timeout = 60 if !defined($timeout);
4708
4709         eval {
4710             if ($shutdown) {
4711                 if (defined($conf) && $conf->{agent}) {
4712                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4713                 } else {
4714                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4715                 }
4716             } else {
4717                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4718             }
4719         };
4720         my $err = $@;
4721
4722         if (!$err) {
4723             my $count = 0;
4724             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4725                 $count++;
4726                 sleep 1;
4727             }
4728
4729             if ($count >= $timeout) {
4730                 if ($force) {
4731                     warn "VM still running - terminating now with SIGTERM\n";
4732                     kill 15, $pid;
4733                 } else {
4734                     die "VM quit/powerdown failed - got timeout\n";
4735                 }
4736             } else {
4737                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4738                 return;
4739             }
4740         } else {
4741             if ($force) {
4742                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4743                 kill 15, $pid;
4744             } else {
4745                 die "VM quit/powerdown failed\n";
4746             }
4747         }
4748
4749         # wait again
4750         $timeout = 10;
4751
4752         my $count = 0;
4753         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4754             $count++;
4755             sleep 1;
4756         }
4757
4758         if ($count >= $timeout) {
4759             warn "VM still running - terminating now with SIGKILL\n";
4760             kill 9, $pid;
4761             sleep 1;
4762         }
4763
4764         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4765    });
4766 }
4767
4768 sub vm_suspend {
4769     my ($vmid, $skiplock) = @_;
4770
4771     PVE::QemuConfig->lock_config($vmid, sub {
4772
4773         my $conf = PVE::QemuConfig->load_config($vmid);
4774
4775         PVE::QemuConfig->check_lock($conf)
4776             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4777
4778         vm_mon_cmd($vmid, "stop");
4779     });
4780 }
4781
4782 sub vm_resume {
4783     my ($vmid, $skiplock, $nocheck) = @_;
4784
4785     PVE::QemuConfig->lock_config($vmid, sub {
4786
4787         if (!$nocheck) {
4788
4789             my $conf = PVE::QemuConfig->load_config($vmid);
4790
4791             PVE::QemuConfig->check_lock($conf)
4792                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4793
4794             vm_mon_cmd($vmid, "cont");
4795
4796         } else {
4797             vm_mon_cmd_nocheck($vmid, "cont");
4798         }
4799     });
4800 }
4801
4802 sub vm_sendkey {
4803     my ($vmid, $skiplock, $key) = @_;
4804
4805     PVE::QemuConfig->lock_config($vmid, sub {
4806
4807         my $conf = PVE::QemuConfig->load_config($vmid);
4808
4809         # there is no qmp command, so we use the human monitor command
4810         vm_human_monitor_command($vmid, "sendkey $key");
4811     });
4812 }
4813
4814 sub vm_destroy {
4815     my ($storecfg, $vmid, $skiplock) = @_;
4816
4817     PVE::QemuConfig->lock_config($vmid, sub {
4818
4819         my $conf = PVE::QemuConfig->load_config($vmid);
4820
4821         if (!check_running($vmid)) {
4822             destroy_vm($storecfg, $vmid, undef, $skiplock);
4823         } else {
4824             die "VM $vmid is running - destroy failed\n";
4825         }
4826     });
4827 }
4828
4829 # pci helpers
4830
4831 sub file_write {
4832     my ($filename, $buf) = @_;
4833
4834     my $fh = IO::File->new($filename, "w");
4835     return undef if !$fh;
4836
4837     my $res = print $fh $buf;
4838
4839     $fh->close();
4840
4841     return $res;
4842 }
4843
4844 sub pci_device_info {
4845     my ($name) = @_;
4846
4847     my $res;
4848
4849     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4850     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4851
4852     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4853     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4854
4855     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4856     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4857
4858     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4859     return undef if !defined($product) || $product !~ s/^0x//;
4860
4861     $res = {
4862         name => $name,
4863         vendor => $vendor,
4864         product => $product,
4865         domain => $domain,
4866         bus => $bus,
4867         slot => $slot,
4868         func => $func,
4869         irq => $irq,
4870         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4871     };
4872
4873     return $res;
4874 }
4875
4876 sub pci_dev_reset {
4877     my ($dev) = @_;
4878
4879     my $name = $dev->{name};
4880
4881     my $fn = "$pcisysfs/devices/$name/reset";
4882
4883     return file_write($fn, "1");
4884 }
4885
4886 sub pci_dev_bind_to_vfio {
4887     my ($dev) = @_;
4888
4889     my $name = $dev->{name};
4890
4891     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4892
4893     if (!-d $vfio_basedir) {
4894         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4895     }
4896     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4897
4898     my $testdir = "$vfio_basedir/$name";
4899     return 1 if -d $testdir;
4900
4901     my $data = "$dev->{vendor} $dev->{product}";
4902     return undef if !file_write("$vfio_basedir/new_id", $data);
4903
4904     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4905     if (!file_write($fn, $name)) {
4906         return undef if -f $fn;
4907     }
4908
4909     $fn = "$vfio_basedir/bind";
4910     if (! -d $testdir) {
4911         return undef if !file_write($fn, $name);
4912     }
4913
4914     return -d $testdir;
4915 }
4916
4917 sub pci_dev_group_bind_to_vfio {
4918     my ($pciid) = @_;
4919
4920     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4921
4922     if (!-d $vfio_basedir) {
4923         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4924     }
4925     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4926
4927     # get IOMMU group devices
4928     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4929       my @devs = grep /^0000:/, readdir($D);
4930     closedir($D);
4931
4932     foreach my $pciid (@devs) {
4933         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4934
4935         # pci bridges, switches or root ports are not supported
4936         # they have a pci_bus subdirectory so skip them
4937         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4938
4939         my $info = pci_device_info($1);
4940         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4941     }
4942
4943     return 1;
4944 }
4945
4946 sub print_pci_addr {
4947     my ($id, $bridges) = @_;
4948
4949     my $res = '';
4950     my $devices = {
4951         piix3 => { bus => 0, addr => 1 },
4952         #addr2 : first videocard
4953         balloon0 => { bus => 0, addr => 3 },
4954         watchdog => { bus => 0, addr => 4 },
4955         scsihw0 => { bus => 0, addr => 5 },
4956         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4957         scsihw1 => { bus => 0, addr => 6 },
4958         ahci0 => { bus => 0, addr => 7 },
4959         qga0 => { bus => 0, addr => 8 },
4960         spice => { bus => 0, addr => 9 },
4961         virtio0 => { bus => 0, addr => 10 },
4962         virtio1 => { bus => 0, addr => 11 },
4963         virtio2 => { bus => 0, addr => 12 },
4964         virtio3 => { bus => 0, addr => 13 },
4965         virtio4 => { bus => 0, addr => 14 },
4966         virtio5 => { bus => 0, addr => 15 },
4967         hostpci0 => { bus => 0, addr => 16 },
4968         hostpci1 => { bus => 0, addr => 17 },
4969         net0 => { bus => 0, addr => 18 },
4970         net1 => { bus => 0, addr => 19 },
4971         net2 => { bus => 0, addr => 20 },
4972         net3 => { bus => 0, addr => 21 },
4973         net4 => { bus => 0, addr => 22 },
4974         net5 => { bus => 0, addr => 23 },
4975         vga1 => { bus => 0, addr => 24 },
4976         vga2 => { bus => 0, addr => 25 },
4977         vga3 => { bus => 0, addr => 26 },
4978         hostpci2 => { bus => 0, addr => 27 },
4979         hostpci3 => { bus => 0, addr => 28 },
4980         #addr29 : usb-host (pve-usb.cfg)
4981         'pci.1' => { bus => 0, addr => 30 },
4982         'pci.2' => { bus => 0, addr => 31 },
4983         'net6' => { bus => 1, addr => 1 },
4984         'net7' => { bus => 1, addr => 2 },
4985         'net8' => { bus => 1, addr => 3 },
4986         'net9' => { bus => 1, addr => 4 },
4987         'net10' => { bus => 1, addr => 5 },
4988         'net11' => { bus => 1, addr => 6 },
4989         'net12' => { bus => 1, addr => 7 },
4990         'net13' => { bus => 1, addr => 8 },
4991         'net14' => { bus => 1, addr => 9 },
4992         'net15' => { bus => 1, addr => 10 },
4993         'net16' => { bus => 1, addr => 11 },
4994         'net17' => { bus => 1, addr => 12 },
4995         'net18' => { bus => 1, addr => 13 },
4996         'net19' => { bus => 1, addr => 14 },
4997         'net20' => { bus => 1, addr => 15 },
4998         'net21' => { bus => 1, addr => 16 },
4999         'net22' => { bus => 1, addr => 17 },
5000         'net23' => { bus => 1, addr => 18 },
5001         'net24' => { bus => 1, addr => 19 },
5002         'net25' => { bus => 1, addr => 20 },
5003         'net26' => { bus => 1, addr => 21 },
5004         'net27' => { bus => 1, addr => 22 },
5005         'net28' => { bus => 1, addr => 23 },
5006         'net29' => { bus => 1, addr => 24 },
5007         'net30' => { bus => 1, addr => 25 },
5008         'net31' => { bus => 1, addr => 26 },
5009         'xhci' => { bus => 1, addr => 27 },
5010         'virtio6' => { bus => 2, addr => 1 },
5011         'virtio7' => { bus => 2, addr => 2 },
5012         'virtio8' => { bus => 2, addr => 3 },
5013         'virtio9' => { bus => 2, addr => 4 },
5014         'virtio10' => { bus => 2, addr => 5 },
5015         'virtio11' => { bus => 2, addr => 6 },
5016         'virtio12' => { bus => 2, addr => 7 },
5017         'virtio13' => { bus => 2, addr => 8 },
5018         'virtio14' => { bus => 2, addr => 9 },
5019         'virtio15' => { bus => 2, addr => 10 },
5020         'virtioscsi0' => { bus => 3, addr => 1 },
5021         'virtioscsi1' => { bus => 3, addr => 2 },
5022         'virtioscsi2' => { bus => 3, addr => 3 },
5023         'virtioscsi3' => { bus => 3, addr => 4 },
5024         'virtioscsi4' => { bus => 3, addr => 5 },
5025         'virtioscsi5' => { bus => 3, addr => 6 },
5026         'virtioscsi6' => { bus => 3, addr => 7 },
5027         'virtioscsi7' => { bus => 3, addr => 8 },
5028         'virtioscsi8' => { bus => 3, addr => 9 },
5029         'virtioscsi9' => { bus => 3, addr => 10 },
5030         'virtioscsi10' => { bus => 3, addr => 11 },
5031         'virtioscsi11' => { bus => 3, addr => 12 },
5032         'virtioscsi12' => { bus => 3, addr => 13 },
5033         'virtioscsi13' => { bus => 3, addr => 14 },
5034         'virtioscsi14' => { bus => 3, addr => 15 },
5035         'virtioscsi15' => { bus => 3, addr => 16 },
5036         'virtioscsi16' => { bus => 3, addr => 17 },
5037         'virtioscsi17' => { bus => 3, addr => 18 },
5038         'virtioscsi18' => { bus => 3, addr => 19 },
5039         'virtioscsi19' => { bus => 3, addr => 20 },
5040         'virtioscsi20' => { bus => 3, addr => 21 },
5041         'virtioscsi21' => { bus => 3, addr => 22 },
5042         'virtioscsi22' => { bus => 3, addr => 23 },
5043         'virtioscsi23' => { bus => 3, addr => 24 },
5044         'virtioscsi24' => { bus => 3, addr => 25 },
5045         'virtioscsi25' => { bus => 3, addr => 26 },
5046         'virtioscsi26' => { bus => 3, addr => 27 },
5047         'virtioscsi27' => { bus => 3, addr => 28 },
5048         'virtioscsi28' => { bus => 3, addr => 29 },
5049         'virtioscsi29' => { bus => 3, addr => 30 },
5050         'virtioscsi30' => { bus => 3, addr => 31 },
5051
5052     };
5053
5054     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5055            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5056            my $bus = $devices->{$id}->{bus};
5057            $res = ",bus=pci.$bus,addr=$addr";
5058            $bridges->{$bus} = 1 if $bridges;
5059     }
5060     return $res;
5061
5062 }
5063
5064 sub print_pcie_addr {
5065     my ($id) = @_;
5066
5067     my $res = '';
5068     my $devices = {
5069         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5070         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5071         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5072         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5073     };
5074
5075     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5076            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5077            my $bus = $devices->{$id}->{bus};
5078            $res = ",bus=$bus,addr=$addr";
5079     }
5080     return $res;
5081
5082 }
5083
5084 # vzdump restore implementaion
5085
5086 sub tar_archive_read_firstfile {
5087     my $archive = shift;
5088
5089     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5090
5091     # try to detect archive type first
5092     my $pid = open (TMP, "tar tf '$archive'|") ||
5093         die "unable to open file '$archive'\n";
5094     my $firstfile = <TMP>;
5095     kill 15, $pid;
5096     close TMP;
5097
5098     die "ERROR: archive contaions no data\n" if !$firstfile;
5099     chomp $firstfile;
5100
5101     return $firstfile;
5102 }
5103
5104 sub tar_restore_cleanup {
5105     my ($storecfg, $statfile) = @_;
5106
5107     print STDERR "starting cleanup\n";
5108
5109     if (my $fd = IO::File->new($statfile, "r")) {
5110         while (defined(my $line = <$fd>)) {
5111             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5112                 my $volid = $2;
5113                 eval {
5114                     if ($volid =~ m|^/|) {
5115                         unlink $volid || die 'unlink failed\n';
5116                     } else {
5117                         PVE::Storage::vdisk_free($storecfg, $volid);
5118                     }
5119                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5120                 };
5121                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5122             } else {
5123                 print STDERR "unable to parse line in statfile - $line";
5124             }
5125         }
5126         $fd->close();
5127     }
5128 }
5129
5130 sub restore_archive {
5131     my ($archive, $vmid, $user, $opts) = @_;
5132
5133     my $format = $opts->{format};
5134     my $comp;
5135
5136     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5137         $format = 'tar' if !$format;
5138         $comp = 'gzip';
5139     } elsif ($archive =~ m/\.tar$/) {
5140         $format = 'tar' if !$format;
5141     } elsif ($archive =~ m/.tar.lzo$/) {
5142         $format = 'tar' if !$format;
5143         $comp = 'lzop';
5144     } elsif ($archive =~ m/\.vma$/) {
5145         $format = 'vma' if !$format;
5146     } elsif ($archive =~ m/\.vma\.gz$/) {
5147         $format = 'vma' if !$format;
5148         $comp = 'gzip';
5149     } elsif ($archive =~ m/\.vma\.lzo$/) {
5150         $format = 'vma' if !$format;
5151         $comp = 'lzop';
5152     } else {
5153         $format = 'vma' if !$format; # default
5154     }
5155
5156     # try to detect archive format
5157     if ($format eq 'tar') {
5158         return restore_tar_archive($archive, $vmid, $user, $opts);
5159     } else {
5160         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5161     }
5162 }
5163
5164 sub restore_update_config_line {
5165     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5166
5167     return if $line =~ m/^\#qmdump\#/;
5168     return if $line =~ m/^\#vzdump\#/;
5169     return if $line =~ m/^lock:/;
5170     return if $line =~ m/^unused\d+:/;
5171     return if $line =~ m/^parent:/;
5172     return if $line =~ m/^template:/; # restored VM is never a template
5173
5174     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5175         # try to convert old 1.X settings
5176         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5177         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5178             my ($model, $macaddr) = split(/\=/, $devconfig);
5179             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5180             my $net = {
5181                 model => $model,
5182                 bridge => "vmbr$ind",
5183                 macaddr => $macaddr,
5184             };
5185             my $netstr = print_net($net);
5186
5187             print $outfd "net$cookie->{netcount}: $netstr\n";
5188             $cookie->{netcount}++;
5189         }
5190     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5191         my ($id, $netstr) = ($1, $2);
5192         my $net = parse_net($netstr);
5193         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5194         $netstr = print_net($net);
5195         print $outfd "$id: $netstr\n";
5196     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5197         my $virtdev = $1;
5198         my $value = $3;
5199         my $di = parse_drive($virtdev, $value);
5200         if (defined($di->{backup}) && !$di->{backup}) {
5201             print $outfd "#$line";
5202         } elsif ($map->{$virtdev}) {
5203             delete $di->{format}; # format can change on restore
5204             $di->{file} = $map->{$virtdev};
5205             $value = print_drive($vmid, $di);
5206             print $outfd "$virtdev: $value\n";
5207         } else {
5208             print $outfd $line;
5209         }
5210     } else {
5211         print $outfd $line;
5212     }
5213 }
5214
5215 sub scan_volids {
5216     my ($cfg, $vmid) = @_;
5217
5218     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5219
5220     my $volid_hash = {};
5221     foreach my $storeid (keys %$info) {
5222         foreach my $item (@{$info->{$storeid}}) {
5223             next if !($item->{volid} && $item->{size});
5224             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5225             $volid_hash->{$item->{volid}} = $item;
5226         }
5227     }
5228
5229     return $volid_hash;
5230 }
5231
5232 sub is_volume_in_use {
5233     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5234
5235     my $path = PVE::Storage::path($storecfg, $volid);
5236
5237     my $scan_config = sub {
5238         my ($cref, $snapname) = @_;
5239
5240         foreach my $key (keys %$cref) {
5241             my $value = $cref->{$key};
5242             if (is_valid_drivename($key)) {
5243                 next if $skip_drive && $key eq $skip_drive;
5244                 my $drive = parse_drive($key, $value);
5245                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5246                 return 1 if $volid eq $drive->{file};
5247                 if ($drive->{file} =~ m!^/!) {
5248                     return 1 if $drive->{file} eq $path;
5249                 } else {
5250                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5251                     next if !$storeid;
5252                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5253                     next if !$scfg;
5254                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5255                 }
5256             }
5257         }
5258
5259         return 0;
5260     };
5261
5262     return 1 if &$scan_config($conf);
5263
5264     undef $skip_drive;
5265
5266     foreach my $snapname (keys %{$conf->{snapshots}}) {
5267         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5268     }
5269
5270     return 0;
5271 }
5272
5273 sub update_disksize {
5274     my ($vmid, $conf, $volid_hash) = @_;
5275
5276     my $changes;
5277
5278     my $used = {};
5279
5280     # Note: it is allowed to define multiple storages with same path (alias), so
5281     # we need to check both 'volid' and real 'path' (two different volid can point
5282     # to the same path).
5283
5284     my $usedpath = {};
5285
5286     # update size info
5287     foreach my $opt (keys %$conf) {
5288         if (is_valid_drivename($opt)) {
5289             my $drive = parse_drive($opt, $conf->{$opt});
5290             my $volid = $drive->{file};
5291             next if !$volid;
5292
5293             $used->{$volid} = 1;
5294             if ($volid_hash->{$volid} &&
5295                 (my $path = $volid_hash->{$volid}->{path})) {
5296                 $usedpath->{$path} = 1;
5297             }
5298
5299             next if drive_is_cdrom($drive);
5300             next if !$volid_hash->{$volid};
5301
5302             $drive->{size} = $volid_hash->{$volid}->{size};
5303             my $new = print_drive($vmid, $drive);
5304             if ($new ne $conf->{$opt}) {
5305                 $changes = 1;
5306                 $conf->{$opt} = $new;
5307             }
5308         }
5309     }
5310
5311     # remove 'unusedX' entry if volume is used
5312     foreach my $opt (keys %$conf) {
5313         next if $opt !~ m/^unused\d+$/;
5314         my $volid = $conf->{$opt};
5315         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5316         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5317             $changes = 1;
5318             delete $conf->{$opt};
5319         }
5320     }
5321
5322     foreach my $volid (sort keys %$volid_hash) {
5323         next if $volid =~ m/vm-$vmid-state-/;
5324         next if $used->{$volid};
5325         my $path = $volid_hash->{$volid}->{path};
5326         next if !$path; # just to be sure
5327         next if $usedpath->{$path};
5328         $changes = 1;
5329         PVE::QemuConfig->add_unused_volume($conf, $volid);
5330         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5331     }
5332
5333     return $changes;
5334 }
5335
5336 sub rescan {
5337     my ($vmid, $nolock) = @_;
5338
5339     my $cfg = PVE::Storage::config();
5340
5341     my $volid_hash = scan_volids($cfg, $vmid);
5342
5343     my $updatefn =  sub {
5344         my ($vmid) = @_;
5345
5346         my $conf = PVE::QemuConfig->load_config($vmid);
5347
5348         PVE::QemuConfig->check_lock($conf);
5349
5350         my $vm_volids = {};
5351         foreach my $volid (keys %$volid_hash) {
5352             my $info = $volid_hash->{$volid};
5353             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5354         }
5355
5356         my $changes = update_disksize($vmid, $conf, $vm_volids);
5357
5358         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5359     };
5360
5361     if (defined($vmid)) {
5362         if ($nolock) {
5363             &$updatefn($vmid);
5364         } else {
5365             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5366         }
5367     } else {
5368         my $vmlist = config_list();
5369         foreach my $vmid (keys %$vmlist) {
5370             if ($nolock) {
5371                 &$updatefn($vmid);
5372             } else {
5373                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5374             }
5375         }
5376     }
5377 }
5378
5379 sub restore_vma_archive {
5380     my ($archive, $vmid, $user, $opts, $comp) = @_;
5381
5382     my $input = $archive eq '-' ? "<&STDIN" : undef;
5383     my $readfrom = $archive;
5384
5385     my $uncomp = '';
5386     if ($comp) {
5387         $readfrom = '-';
5388         my $qarchive = PVE::Tools::shellquote($archive);
5389         if ($comp eq 'gzip') {
5390             $uncomp = "zcat $qarchive|";
5391         } elsif ($comp eq 'lzop') {
5392             $uncomp = "lzop -d -c $qarchive|";
5393         } else {
5394             die "unknown compression method '$comp'\n";
5395         }
5396
5397     }
5398
5399     my $tmpdir = "/var/tmp/vzdumptmp$$";
5400     rmtree $tmpdir;
5401
5402     # disable interrupts (always do cleanups)
5403     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5404         warn "got interrupt - ignored\n";
5405     };
5406
5407     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5408     POSIX::mkfifo($mapfifo, 0600);
5409     my $fifofh;
5410
5411     my $openfifo = sub {
5412         open($fifofh, '>', $mapfifo) || die $!;
5413     };
5414
5415     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5416
5417     my $oldtimeout;
5418     my $timeout = 5;
5419
5420     my $devinfo = {};
5421
5422     my $rpcenv = PVE::RPCEnvironment::get();
5423
5424     my $conffile = PVE::QemuConfig->config_file($vmid);
5425     my $tmpfn = "$conffile.$$.tmp";
5426
5427     # Note: $oldconf is undef if VM does not exists
5428     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5429     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5430
5431     my $print_devmap = sub {
5432         my $virtdev_hash = {};
5433
5434         my $cfgfn = "$tmpdir/qemu-server.conf";
5435
5436         # we can read the config - that is already extracted
5437         my $fh = IO::File->new($cfgfn, "r") ||
5438             "unable to read qemu-server.conf - $!\n";
5439
5440         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5441         if (-f $fwcfgfn) {
5442             my $pve_firewall_dir = '/etc/pve/firewall';
5443             mkdir $pve_firewall_dir; # make sure the dir exists
5444             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5445         }
5446
5447         while (defined(my $line = <$fh>)) {
5448             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5449                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5450                 die "archive does not contain data for drive '$virtdev'\n"
5451                     if !$devinfo->{$devname};
5452                 if (defined($opts->{storage})) {
5453                     $storeid = $opts->{storage} || 'local';
5454                 } elsif (!$storeid) {
5455                     $storeid = 'local';
5456                 }
5457                 $format = 'raw' if !$format;
5458                 $devinfo->{$devname}->{devname} = $devname;
5459                 $devinfo->{$devname}->{virtdev} = $virtdev;
5460                 $devinfo->{$devname}->{format} = $format;
5461                 $devinfo->{$devname}->{storeid} = $storeid;
5462
5463                 # check permission on storage
5464                 my $pool = $opts->{pool}; # todo: do we need that?
5465                 if ($user ne 'root@pam') {
5466                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5467                 }
5468
5469                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5470             }
5471         }
5472
5473         foreach my $devname (keys %$devinfo) {
5474             die "found no device mapping information for device '$devname'\n"
5475                 if !$devinfo->{$devname}->{virtdev};
5476         }
5477
5478         my $cfg = PVE::Storage::config();
5479
5480         # create empty/temp config
5481         if ($oldconf) {
5482             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5483             foreach_drive($oldconf, sub {
5484                 my ($ds, $drive) = @_;
5485
5486                 return if drive_is_cdrom($drive);
5487
5488                 my $volid = $drive->{file};
5489
5490                 return if !$volid || $volid =~ m|^/|;
5491
5492                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5493                 return if !$path || !$owner || ($owner != $vmid);
5494
5495                 # Note: only delete disk we want to restore
5496                 # other volumes will become unused
5497                 if ($virtdev_hash->{$ds}) {
5498                     PVE::Storage::vdisk_free($cfg, $volid);
5499                 }
5500             });
5501
5502             # delete vmstate files
5503             # since after the restore we have no snapshots anymore
5504             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5505                 my $snap = $oldconf->{snapshots}->{$snapname};
5506                 if ($snap->{vmstate}) {
5507                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5508                     if (my $err = $@) {
5509                         warn $err;
5510                     }
5511                 }
5512             }
5513         }
5514
5515         my $map = {};
5516         foreach my $virtdev (sort keys %$virtdev_hash) {
5517             my $d = $virtdev_hash->{$virtdev};
5518             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5519             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5520
5521             # test if requested format is supported
5522             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5523             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5524             $d->{format} = $defFormat if !$supported;
5525
5526             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5527                                                   $d->{format}, undef, $alloc_size);
5528             print STDERR "new volume ID is '$volid'\n";
5529             $d->{volid} = $volid;
5530             my $path = PVE::Storage::path($cfg, $volid);
5531
5532             PVE::Storage::activate_volumes($cfg,[$volid]);
5533
5534             my $write_zeros = 1;
5535             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5536                 $write_zeros = 0;
5537             }
5538
5539             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5540
5541             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5542             $map->{$virtdev} = $volid;
5543         }
5544
5545         $fh->seek(0, 0) || die "seek failed - $!\n";
5546
5547         my $outfd = new IO::File ($tmpfn, "w") ||
5548             die "unable to write config for VM $vmid\n";
5549
5550         my $cookie = { netcount => 0 };
5551         while (defined(my $line = <$fh>)) {
5552             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5553         }
5554
5555         $fh->close();
5556         $outfd->close();
5557     };
5558
5559     eval {
5560         # enable interrupts
5561         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5562             die "interrupted by signal\n";
5563         };
5564         local $SIG{ALRM} = sub { die "got timeout\n"; };
5565
5566         $oldtimeout = alarm($timeout);
5567
5568         my $parser = sub {
5569             my $line = shift;
5570
5571             print "$line\n";
5572
5573             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5574                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5575                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5576             } elsif ($line =~ m/^CTIME: /) {
5577                 # we correctly received the vma config, so we can disable
5578                 # the timeout now for disk allocation (set to 10 minutes, so
5579                 # that we always timeout if something goes wrong)
5580                 alarm(600);
5581                 &$print_devmap();
5582                 print $fifofh "done\n";
5583                 my $tmp = $oldtimeout || 0;
5584                 $oldtimeout = undef;
5585                 alarm($tmp);
5586                 close($fifofh);
5587             }
5588         };
5589
5590         print "restore vma archive: $cmd\n";
5591         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5592     };
5593     my $err = $@;
5594
5595     alarm($oldtimeout) if $oldtimeout;
5596
5597     my $vollist = [];
5598     foreach my $devname (keys %$devinfo) {
5599         my $volid = $devinfo->{$devname}->{volid};
5600         push @$vollist, $volid if $volid;
5601     }
5602
5603     my $cfg = PVE::Storage::config();
5604     PVE::Storage::deactivate_volumes($cfg, $vollist);
5605
5606     unlink $mapfifo;
5607
5608     if ($err) {
5609         rmtree $tmpdir;
5610         unlink $tmpfn;
5611
5612         foreach my $devname (keys %$devinfo) {
5613             my $volid = $devinfo->{$devname}->{volid};
5614             next if !$volid;
5615             eval {
5616                 if ($volid =~ m|^/|) {
5617                     unlink $volid || die 'unlink failed\n';
5618                 } else {
5619                     PVE::Storage::vdisk_free($cfg, $volid);
5620                 }
5621                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5622             };
5623             print STDERR "unable to cleanup '$volid' - $@" if $@;
5624         }
5625         die $err;
5626     }
5627
5628     rmtree $tmpdir;
5629
5630     rename($tmpfn, $conffile) ||
5631         die "unable to commit configuration file '$conffile'\n";
5632
5633     PVE::Cluster::cfs_update(); # make sure we read new file
5634
5635     eval { rescan($vmid, 1); };
5636     warn $@ if $@;
5637 }
5638
5639 sub restore_tar_archive {
5640     my ($archive, $vmid, $user, $opts) = @_;
5641
5642     if ($archive ne '-') {
5643         my $firstfile = tar_archive_read_firstfile($archive);
5644         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5645             if $firstfile ne 'qemu-server.conf';
5646     }
5647
5648     my $storecfg = PVE::Storage::config();
5649
5650     # destroy existing data - keep empty config
5651     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5652     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5653
5654     my $tocmd = "/usr/lib/qemu-server/qmextract";
5655
5656     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5657     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5658     $tocmd .= ' --prealloc' if $opts->{prealloc};
5659     $tocmd .= ' --info' if $opts->{info};
5660
5661     # tar option "xf" does not autodetect compression when read from STDIN,
5662     # so we pipe to zcat
5663     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5664         PVE::Tools::shellquote("--to-command=$tocmd");
5665
5666     my $tmpdir = "/var/tmp/vzdumptmp$$";
5667     mkpath $tmpdir;
5668
5669     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5670     local $ENV{VZDUMP_VMID} = $vmid;
5671     local $ENV{VZDUMP_USER} = $user;
5672
5673     my $conffile = PVE::QemuConfig->config_file($vmid);
5674     my $tmpfn = "$conffile.$$.tmp";
5675
5676     # disable interrupts (always do cleanups)
5677     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5678         print STDERR "got interrupt - ignored\n";
5679     };
5680
5681     eval {
5682         # enable interrupts
5683         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5684             die "interrupted by signal\n";
5685         };
5686
5687         if ($archive eq '-') {
5688             print "extracting archive from STDIN\n";
5689             run_command($cmd, input => "<&STDIN");
5690         } else {
5691             print "extracting archive '$archive'\n";
5692             run_command($cmd);
5693         }
5694
5695         return if $opts->{info};
5696
5697         # read new mapping
5698         my $map = {};
5699         my $statfile = "$tmpdir/qmrestore.stat";
5700         if (my $fd = IO::File->new($statfile, "r")) {
5701             while (defined (my $line = <$fd>)) {
5702                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5703                     $map->{$1} = $2 if $1;
5704                 } else {
5705                     print STDERR "unable to parse line in statfile - $line\n";
5706                 }
5707             }
5708             $fd->close();
5709         }
5710
5711         my $confsrc = "$tmpdir/qemu-server.conf";
5712
5713         my $srcfd = new IO::File($confsrc, "r") ||
5714             die "unable to open file '$confsrc'\n";
5715
5716         my $outfd = new IO::File ($tmpfn, "w") ||
5717             die "unable to write config for VM $vmid\n";
5718
5719         my $cookie = { netcount => 0 };
5720         while (defined (my $line = <$srcfd>)) {
5721             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5722         }
5723
5724         $srcfd->close();
5725         $outfd->close();
5726     };
5727     my $err = $@;
5728
5729     if ($err) {
5730
5731         unlink $tmpfn;
5732
5733         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5734
5735         die $err;
5736     }
5737
5738     rmtree $tmpdir;
5739
5740     rename $tmpfn, $conffile ||
5741         die "unable to commit configuration file '$conffile'\n";
5742
5743     PVE::Cluster::cfs_update(); # make sure we read new file
5744
5745     eval { rescan($vmid, 1); };
5746     warn $@ if $@;
5747 };
5748
5749 sub foreach_writable_storage {
5750     my ($conf, $func) = @_;
5751
5752     my $sidhash = {};
5753
5754     foreach my $ds (keys %$conf) {
5755         next if !is_valid_drivename($ds);
5756
5757         my $drive = parse_drive($ds, $conf->{$ds});
5758         next if !$drive;
5759         next if drive_is_cdrom($drive);
5760
5761         my $volid = $drive->{file};
5762
5763         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5764         $sidhash->{$sid} = $sid if $sid;
5765     }
5766
5767     foreach my $sid (sort keys %$sidhash) {
5768         &$func($sid);
5769     }
5770 }
5771
5772 sub do_snapshots_with_qemu {
5773     my ($storecfg, $volid) = @_;
5774
5775     my $storage_name = PVE::Storage::parse_volume_id($volid);
5776
5777     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5778         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5779         return 1;
5780     }
5781
5782     if ($volid =~ m/\.(qcow2|qed)$/){
5783         return 1;
5784     }
5785
5786     return undef;
5787 }
5788
5789 sub qga_check_running {
5790     my ($vmid) = @_;
5791
5792     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5793     if ($@) {
5794         warn "Qemu Guest Agent are not running - $@";
5795         return 0;
5796     }
5797     return 1;
5798 }
5799
5800 sub template_create {
5801     my ($vmid, $conf, $disk) = @_;
5802
5803     my $storecfg = PVE::Storage::config();
5804
5805     foreach_drive($conf, sub {
5806         my ($ds, $drive) = @_;
5807
5808         return if drive_is_cdrom($drive);
5809         return if $disk && $ds ne $disk;
5810
5811         my $volid = $drive->{file};
5812         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5813
5814         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5815         $drive->{file} = $voliddst;
5816         $conf->{$ds} = print_drive($vmid, $drive);
5817         PVE::QemuConfig->write_config($vmid, $conf);
5818     });
5819 }
5820
5821 sub qemu_img_convert {
5822     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5823
5824     my $storecfg = PVE::Storage::config();
5825     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5826     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5827
5828     if ($src_storeid && $dst_storeid) {
5829
5830         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5831
5832         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5833         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5834
5835         my $src_format = qemu_img_format($src_scfg, $src_volname);
5836         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5837
5838         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5839         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5840
5841         my $cmd = [];
5842         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5843         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5844         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5845         if ($is_zero_initialized) {
5846             push @$cmd, "zeroinit:$dst_path";
5847         } else {
5848             push @$cmd, $dst_path;
5849         }
5850
5851         my $parser = sub {
5852             my $line = shift;
5853             if($line =~ m/\((\S+)\/100\%\)/){
5854                 my $percent = $1;
5855                 my $transferred = int($size * $percent / 100);
5856                 my $remaining = $size - $transferred;
5857
5858                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5859             }
5860
5861         };
5862
5863         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5864         my $err = $@;
5865         die "copy failed: $err" if $err;
5866     }
5867 }
5868
5869 sub qemu_img_format {
5870     my ($scfg, $volname) = @_;
5871
5872     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5873         return $1;
5874     } else {
5875         return "raw";
5876     }
5877 }
5878
5879 sub qemu_drive_mirror {
5880     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5881
5882     my $storecfg = PVE::Storage::config();
5883     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5884
5885     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5886
5887     my $format = qemu_img_format($dst_scfg, $dst_volname);
5888
5889     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5890
5891     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5892
5893     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5894     $opts->{format} = $format if $format;
5895
5896     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5897
5898     eval {
5899     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5900         while (1) {
5901             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5902             my $stat = @$stats[0];
5903             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5904             die "error job is not mirroring" if $stat->{type} ne "mirror";
5905
5906             my $busy = $stat->{busy};
5907             my $ready = $stat->{ready};
5908
5909             if (my $total = $stat->{len}) {
5910                 my $transferred = $stat->{offset} || 0;
5911                 my $remaining = $total - $transferred;
5912                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5913
5914                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5915             }
5916
5917
5918             if ($stat->{ready} eq 'true') {
5919
5920                 last if $vmiddst != $vmid;
5921
5922                 # try to switch the disk if source and destination are on the same guest
5923                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5924                 last if !$@;
5925                 die $@ if $@ !~ m/cannot be completed/;
5926             }
5927             sleep 1;
5928         }
5929
5930
5931     };
5932     my $err = $@;
5933
5934     my $cancel_job = sub {
5935         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5936         while (1) {
5937             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5938             my $stat = @$stats[0];
5939             last if !$stat;
5940             sleep 1;
5941         }
5942     };
5943
5944     if ($err) {
5945         eval { &$cancel_job(); };
5946         die "mirroring error: $err";
5947     }
5948
5949     if ($vmiddst != $vmid) {
5950         # if we clone a disk for a new target vm, we don't switch the disk
5951         &$cancel_job(); # so we call block-job-cancel
5952     }
5953 }
5954
5955 sub clone_disk {
5956     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5957         $newvmid, $storage, $format, $full, $newvollist) = @_;
5958
5959     my $newvolid;
5960
5961     if (!$full) {
5962         print "create linked clone of drive $drivename ($drive->{file})\n";
5963         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5964         push @$newvollist, $newvolid;
5965     } else {
5966         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5967         $storeid = $storage if $storage;
5968
5969         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5970         if (!$format) {
5971             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5972             $format = qemu_img_format($scfg, $volname);
5973         }
5974
5975         # test if requested format is supported - else use default
5976         my $supported = grep { $_ eq $format } @$validFormats;
5977         $format = $defFormat if !$supported;
5978
5979         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5980
5981         print "create full clone of drive $drivename ($drive->{file})\n";
5982         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5983         push @$newvollist, $newvolid;
5984
5985         PVE::Storage::activate_volumes($storecfg, $newvollist);
5986
5987         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5988         if (!$running || $snapname) {
5989             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5990         } else {
5991             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5992         }
5993     }
5994
5995     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5996
5997     my $disk = $drive;
5998     $disk->{format} = undef;
5999     $disk->{file} = $newvolid;
6000     $disk->{size} = $size;
6001
6002     return $disk;
6003 }
6004
6005 # this only works if VM is running
6006 sub get_current_qemu_machine {
6007     my ($vmid) = @_;
6008
6009     my $cmd = { execute => 'query-machines', arguments => {} };
6010     my $res = vm_qmp_command($vmid, $cmd);
6011
6012     my ($current, $default);
6013     foreach my $e (@$res) {
6014         $default = $e->{name} if $e->{'is-default'};
6015         $current = $e->{name} if $e->{'is-current'};
6016     }
6017
6018     # fallback to the default machine if current is not supported by qemu
6019     return $current || $default || 'pc';
6020 }
6021
6022 sub qemu_machine_feature_enabled {
6023     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6024
6025     my $current_major;
6026     my $current_minor;
6027
6028     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6029
6030         $current_major = $3;
6031         $current_minor = $4;
6032
6033     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6034
6035         $current_major = $1;
6036         $current_minor = $2;
6037     }
6038
6039     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6040
6041
6042 }
6043
6044 sub qemu_machine_pxe {
6045     my ($vmid, $conf, $machine) = @_;
6046
6047     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6048
6049     foreach my $opt (keys %$conf) {
6050         next if $opt !~ m/^net(\d+)$/;
6051         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6052         next if !$net;
6053         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6054         return $machine.".pxe" if $romfile =~ m/pxe/;
6055         last;
6056     }
6057
6058     return $machine;
6059 }
6060
6061 sub qemu_use_old_bios_files {
6062     my ($machine_type) = @_;
6063
6064     return if !$machine_type;
6065
6066     my $use_old_bios_files = undef;
6067
6068     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6069         $machine_type = $1;
6070         $use_old_bios_files = 1;
6071     } else {
6072         my $kvmver = kvm_user_version();
6073         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6074         # load new efi bios files on migration. So this hack is required to allow
6075         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6076         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6077         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6078     }
6079
6080     return ($use_old_bios_files, $machine_type);
6081 }
6082
6083 sub lspci {
6084
6085     my $devices = {};
6086
6087     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6088             my (undef, $id, $function) = @_;
6089             my $res = { id => $id, function => $function};
6090             push @{$devices->{$id}}, $res;
6091     });
6092
6093     return $devices;
6094 }
6095
6096 sub vm_iothreads_list {
6097     my ($vmid) = @_;
6098
6099     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6100
6101     my $iothreads = {};
6102     foreach my $iothread (@$res) {
6103         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6104     }
6105
6106     return $iothreads;
6107 }
6108
6109 sub scsihw_infos {
6110     my ($conf, $drive) = @_;
6111
6112     my $maxdev = 0;
6113
6114     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6115         $maxdev = 7;
6116     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6117         $maxdev = 1;
6118     } else {
6119         $maxdev = 256;
6120     }
6121
6122     my $controller = int($drive->{index} / $maxdev);
6123     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6124
6125     return ($maxdev, $controller, $controller_prefix);
6126 }
6127
6128 # bash completion helper
6129
6130 sub complete_backup_archives {
6131     my ($cmdname, $pname, $cvalue) = @_;
6132
6133     my $cfg = PVE::Storage::config();
6134
6135     my $storeid;
6136
6137     if ($cvalue =~ m/^([^:]+):/) {
6138         $storeid = $1;
6139     }
6140
6141     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6142
6143     my $res = [];
6144     foreach my $id (keys %$data) {
6145         foreach my $item (@{$data->{$id}}) {
6146             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6147             push @$res, $item->{volid} if defined($item->{volid});
6148         }
6149     }
6150
6151     return $res;
6152 }
6153
6154 my $complete_vmid_full = sub {
6155     my ($running) = @_;
6156
6157     my $idlist = vmstatus();
6158
6159     my $res = [];
6160
6161     foreach my $id (keys %$idlist) {
6162         my $d = $idlist->{$id};
6163         if (defined($running)) {
6164             next if $d->{template};
6165             next if $running && $d->{status} ne 'running';
6166             next if !$running && $d->{status} eq 'running';
6167         }
6168         push @$res, $id;
6169
6170     }
6171     return $res;
6172 };
6173
6174 sub complete_vmid {
6175     return &$complete_vmid_full();
6176 }
6177
6178 sub complete_vmid_stopped {
6179     return &$complete_vmid_full(0);
6180 }
6181
6182 sub complete_vmid_running {
6183     return &$complete_vmid_full(1);
6184 }
6185
6186 sub complete_storage {
6187
6188     my $cfg = PVE::Storage::config();
6189     my $ids = $cfg->{ids};
6190
6191     my $res = [];
6192     foreach my $sid (keys %$ids) {
6193         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6194         next if !$ids->{$sid}->{content}->{images};
6195         push @$res, $sid;
6196     }
6197
6198     return $res;
6199 }
6200
6201 1;