PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $confdesc = {
 113     onboot => {
 114         optional => 1,
 115         type => 'boolean',
 116         description => "Specifies whether a VM will be started during system bootup.",
 117         default => 0,
 118     },
 119     autostart => {
 120         optional => 1,
 121         type => 'boolean',
 122         description => "Automatic restart after crash (currently ignored).",
 123         default => 0,
 124     },
 125     hotplug => {
 126         optional => 1,
 127         type => 'string', format => 'pve-hotplug-features',
 128         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 129         default => 'network,disk,usb',
 130     },
 131     reboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 135         default => 1,
 136     },
 137     lock => {
 138         optional => 1,
 139         type => 'string',
 140         description => "Lock/unlock the VM.",
 141         enum => [qw(migrate backup snapshot rollback)],
 142     },
 143     cpulimit => {
 144         optional => 1,
 145         type => 'number',
 146         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 147         minimum => 0,
 148         maximum => 128,
 149         default => 0,
 150     },
 151     cpuunits => {
 152         optional => 1,
 153         type => 'integer',
 154         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 155         minimum => 0,
 156         maximum => 500000,
 157         default => 1000,
 158     },
 159     memory => {
 160         optional => 1,
 161         type => 'integer',
 162         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 163         minimum => 16,
 164         default => 512,
 165     },
 166     balloon => {
 167         optional => 1,
 168         type => 'integer',
 169         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 170         minimum => 0,
 171     },
 172     shares => {
 173         optional => 1,
 174         type => 'integer',
 175         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 176         minimum => 0,
 177         maximum => 50000,
 178         default => 1000,
 179     },
 180     keyboard => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 184         enum => PVE::Tools::kvmkeymaplist(),
 185         default => 'en-us',
 186     },
 187     name => {
 188         optional => 1,
 189         type => 'string', format => 'dns-name',
 190         description => "Set a name for the VM. Only used on the configuration web interface.",
 191     },
 192     scsihw => {
 193         optional => 1,
 194         type => 'string',
 195         description => "scsi controller model",
 196         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 197         default => 'lsi',
 198     },
 199     description => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 203     },
 204     ostype => {
 205         optional => 1,
 206         type => 'string',
 207         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 208         description => <<EODESC,
 209 Used to enable special optimization/features for specific
 210 operating systems:
 211
 212 other  => unspecified OS
 213 wxp    => Microsoft Windows XP
 214 w2k    => Microsoft Windows 2000
 215 w2k3   => Microsoft Windows 2003
 216 w2k8   => Microsoft Windows 2008
 217 wvista => Microsoft Windows Vista
 218 win7   => Microsoft Windows 7
 219 win8   => Microsoft Windows 8/2012
 220 l24    => Linux 2.4 Kernel
 221 l26    => Linux 2.6/3.X Kernel
 222 solaris => solaris/opensolaris/openindiania kernel
 223
 224 other|l24|l26|solaris                       ... no special behaviour
 225 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 226 EODESC
 227     },
 228     boot => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 232         pattern => '[acdn]{1,4}',
 233         default => 'cdn',
 234     },
 235     bootdisk => {
 236         optional => 1,
 237         type => 'string', format => 'pve-qm-bootdisk',
 238         description => "Enable booting from specified disk.",
 239         pattern => '(ide|sata|scsi|virtio)\d+',
 240     },
 241     smp => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "The number of CPUs. Please use option -sockets instead.",
 245         minimum => 1,
 246         default => 1,
 247     },
 248     sockets => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "The number of CPU sockets.",
 252         minimum => 1,
 253         default => 1,
 254     },
 255     cores => {
 256         optional => 1,
 257         type => 'integer',
 258         description => "The number of cores per socket.",
 259         minimum => 1,
 260         default => 1,
 261     },
 262     numa => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable NUMA.",
 266         default => 0,
 267     },
 268     vcpus => {
 269         optional => 1,
 270         type => 'integer',
 271         description => "Number of hotplugged vcpus.",
 272         minimum => 1,
 273         default => 0,
 274     },
 275     acpi => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Enable/disable ACPI.",
 279         default => 1,
 280     },
 281     agent => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Enable/disable Qemu GuestAgent.",
 285         default => 0,
 286     },
 287     kvm => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Enable/disable KVM hardware virtualization.",
 291         default => 1,
 292     },
 293     tdf => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable time drift fix.",
 297         default => 0,
 298     },
 299     localtime => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 303     },
 304     freeze => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 308     },
 309     vga => {
 310         optional => 1,
 311         type => 'string',
 312         description => "Select the VGA type. If you want to use high resolution" .
 313             " modes (>= 1280x1024x16) then you should use the options " .
 314             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 315             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 316             "display sever. For win* OS you can select how many independent " .
 317             "displays you want, Linux guests can add displays them self. " .
 318             "You can also run without any graphic card, using a serial device" .
 319             " as terminal.",
 320         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 321     },
 322     watchdog => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-watchdog',
 325         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 326         description => "Create a virtual hardware watchdog device. Once enabled" .
 327             " (by a guest action), the watchdog must be periodically polled " .
 328             "by an agent inside the guest or else the watchdog will reset " .
 329             "the guest (or execute the respective action specified)",
 330     },
 331     startdate => {
 332         optional => 1,
 333         type => 'string',
 334         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 335         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 336         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 337         default => 'now',
 338     },
 339     startup =>  get_standard_option('pve-startup-order'),
 340     template => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Enable/disable Template.",
 344         default => 0,
 345     },
 346     args => {
 347         optional => 1,
 348         type => 'string',
 349         description => <<EODESCR,
 350 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 351
 352 args: -no-reboot -no-hpet
 353 EODESCR
 354     },
 355     tablet => {
 356         optional => 1,
 357         type => 'boolean',
 358         default => 1,
 359         description => "Enable/disable the USB tablet device. This device is " .
 360             "usually needed to allow absolute mouse positioning with VNC. " .
 361             "Else the mouse runs out of sync with normal VNC clients. " .
 362             "If you're running lots of console-only guests on one host, " .
 363             "you may consider disabling this to save some context switches. " .
 364             "This is turned off by default if you use spice (-vga=qxl).",
 365     },
 366     migrate_speed => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 370         minimum => 0,
 371         default => 0,
 372     },
 373     migrate_downtime => {
 374         optional => 1,
 375         type => 'number',
 376         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 377         minimum => 0,
 378         default => 0.1,
 379     },
 380     cdrom => {
 381         optional => 1,
 382         type => 'string', format => 'pve-qm-drive',
 383         typetext => 'volume',
 384         description => "This is an alias for option -ide2",
 385     },
 386     cpu => {
 387         optional => 1,
 388         description => "Emulated CPU type.",
 389         type => 'string',
 390         format => $cpu_fmt,
 391     },
 392     parent => get_standard_option('pve-snapshot-name', {
 393         optional => 1,
 394         description => "Parent snapshot name. This is used internally, and should not be modified.",
 395     }),
 396     snaptime => {
 397         optional => 1,
 398         description => "Timestamp for snapshots.",
 399         type => 'integer',
 400         minimum => 0,
 401     },
 402     vmstate => {
 403         optional => 1,
 404         type => 'string', format => 'pve-volume-id',
 405         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 406     },
 407     machine => {
 408         description => "Specific the Qemu machine type.",
 409         type => 'string',
 410         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 411         maxLength => 40,
 412         optional => 1,
 413     },
 414     smbios1 => {
 415         description => "Specify SMBIOS type 1 fields.",
 416         type => 'string', format => 'pve-qm-smbios1',
 417         maxLength => 256,
 418         optional => 1,
 419     },
 420     protection => {
 421         optional => 1,
 422         type => 'boolean',
 423         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 424         default => 0,
 425     },
 426     bios => {
 427         optional => 1,
 428         type => 'string',
 429         enum => [ qw(seabios ovmf) ],
 430         description => "Select BIOS implementation.",
 431         default => 'seabios',
 432     },
 433 };
 434
 435 # what about other qemu settings ?
 436 #cpu => 'string',
 437 #machine => 'string',
 438 #fda => 'file',
 439 #fdb => 'file',
 440 #mtdblock => 'file',
 441 #sd => 'file',
 442 #pflash => 'file',
 443 #snapshot => 'bool',
 444 #bootp => 'file',
 445 ##tftp => 'dir',
 446 ##smb => 'dir',
 447 #kernel => 'file',
 448 #append => 'string',
 449 #initrd => 'file',
 450 ##soundhw => 'string',
 451
 452 while (my ($k, $v) = each %$confdesc) {
 453     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 454 }
 455
 456 my $MAX_IDE_DISKS = 4;
 457 my $MAX_SCSI_DISKS = 14;
 458 my $MAX_VIRTIO_DISKS = 16;
 459 my $MAX_SATA_DISKS = 6;
 460 my $MAX_USB_DEVICES = 5;
 461 my $MAX_NETS = 32;
 462 my $MAX_UNUSED_DISKS = 8;
 463 my $MAX_HOSTPCI_DEVICES = 4;
 464 my $MAX_SERIAL_PORTS = 4;
 465 my $MAX_PARALLEL_PORTS = 3;
 466 my $MAX_NUMA = 8;
 467 my $MAX_MEM = 4194304;
 468 my $STATICMEM = 1024;
 469
 470 my $numadesc = {
 471     optional => 1,
 472     type => 'string', format => 'pve-qm-numanode',
 473     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 474     description => "numa topology",
 475 };
 476 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 477
 478 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 479     $confdesc->{"numa$i"} = $numadesc;
 480 }
 481
 482 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 483                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 484                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 485 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 486
 487 my $netdesc = {
 488     optional => 1,
 489     type => 'string', format => 'pve-qm-net',
 490     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 491     description => <<EODESCR,
 492 Specify network devices.
 493
 494 MODEL is one of: $nic_model_list_txt
 495
 496 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 497 automatically generated if not specified.
 498
 499 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 500
 501 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 502
 503 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 504
 505 10.0.2.2   Gateway
 506 10.0.2.3   DNS Server
 507 10.0.2.4   SMB Server
 508
 509 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 510
 511 EODESCR
 512 };
 513 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 514
 515 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 516     $confdesc->{"net$i"} = $netdesc;
 517 }
 518
 519 my $drivename_hash;
 520
 521 my %drivedesc_base = (
 522     volume => { alias => 'file' },
 523     file => {
 524         type => 'string',
 525         format => 'pve-volume-id',
 526         default_key => 1,
 527         format_description => 'volume',
 528         description => "The drive's backing volume.",
 529     },
 530     media => {
 531         type => 'string',
 532         format_description => 'cdrom|disk',
 533         enum => [qw(cdrom disk)],
 534         description => "The drive's media type.",
 535         default => 'disk',
 536         optional => 1
 537     },
 538     cyls => {
 539         type => 'integer',
 540         format_description => 'count',
 541         description => "Force the drive's physical geometry to have a specific cylinder count.",
 542         optional => 1
 543     },
 544     heads => {
 545         type => 'integer',
 546         format_description => 'count',
 547         description => "Force the drive's physical geometry to have a specific head count.",
 548         optional => 1
 549     },
 550     secs => {
 551         type => 'integer',
 552         format_description => 'count',
 553         description => "Force the drive's physical geometry to have a specific sector count.",
 554         optional => 1
 555     },
 556     trans => {
 557         type => 'string',
 558         format_description => 'none|lba|auto',
 559         enum => [qw(none lba auto)],
 560         description => "Force disk geometry bios translation mode.",
 561         optional => 1,
 562     },
 563     snapshot => {
 564         type => 'boolean',
 565         format_description => 'on|off',
 566         description => "Whether the drive should be included when making snapshots.",
 567         optional => 1,
 568     },
 569     cache => {
 570         type => 'string',
 571         format_description => 'none|writethrough|writeback|unsafe|directsync',
 572         enum => [qw(none writethrough writeback unsafe directsync)],
 573         description => "The drive's cache mode",
 574         optional => 1,
 575     },
 576     format => {
 577         type => 'string',
 578         format_description => 'drive format',
 579         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 580         description => "The drive's backing file's data format.",
 581         optional => 1,
 582     },
 583     size => {
 584         type => 'string',
 585         format => 'disk-size',
 586         description => "Disk size. This is purely informational and has no effect.",
 587         optional => 1,
 588     },
 589     backup => {
 590         type => 'boolean',
 591         format_description => 'on|off',
 592         description => "Whether the drive should be included when making backups.",
 593         optional => 1,
 594     },
 595     werror => {
 596         type => 'string',
 597         format_description => 'enospc|ignore|report|stop',
 598         enum => [qw(enospc ignore report stop)],
 599         description => 'Write error action.',
 600         optional => 1,
 601     },
 602     aio => {
 603         type => 'string',
 604         format_description => 'native|threads',
 605         enum => [qw(native threads)],
 606         description => 'AIO type to use.',
 607         optional => 1,
 608     },
 609     discard => {
 610         type => 'string',
 611         format_description => 'ignore|on',
 612         enum => [qw(ignore on)],
 613         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 614         optional => 1,
 615     },
 616     detect_zeroes => {
 617         type => 'boolean',
 618         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 619         optional => 1,
 620     },
 621     serial => {
 622         type => 'string',
 623         format => 'urlencoded',
 624         format_description => 'serial',
 625         maxLength => 20*3, # *3 since it's %xx url enoded
 626         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 627         optional => 1,
 628     }
 629 );
 630
 631 my %rerror_fmt = (
 632     rerror => {
 633         type => 'string',
 634         format_description => 'ignore|report|stop',
 635         enum => [qw(ignore report stop)],
 636         description => 'Read error action.',
 637         optional => 1,
 638     },
 639 );
 640
 641 my %iothread_fmt = ( iothread => {
 642         type => 'boolean',
 643         format_description => 'off|on',
 644         description => "Whether to use iothreads for this drive",
 645         optional => 1,
 646 });
 647
 648 my %model_fmt = (
 649     model => {
 650         type => 'string',
 651         format => 'urlencoded',
 652         format_description => 'model',
 653         maxLength => 40*3, # *3 since it's %xx url enoded
 654         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 655         optional => 1,
 656     },
 657 );
 658
 659 my %queues_fmt = (
 660     queues => {
 661         type => 'integer',
 662         format_description => 'nbqueues',
 663         description => "Number of queues.",
 664         minimum => 2,
 665         optional => 1
 666     }
 667 );
 668
 669 my $add_throttle_desc = sub {
 670     my ($key, $type, $what, $size, $longsize) = @_;
 671     $drivedesc_base{$key} = {
 672         type => $type,
 673         format_description => $size,
 674         description => "Maximum $what speed in $longsize per second.",
 675         optional => 1,
 676     };
 677 };
 678 # throughput: (leaky bucket)
 679 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 680 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 681 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 682 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 683 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 684 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 685 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 686 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 687 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 688
 689 # pools: (pool of IO before throttling starts taking effect)
 690 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 691 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 692 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 693 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 694 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 695 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 696
 697 my $ide_fmt = {
 698     %drivedesc_base,
 699     %rerror_fmt,
 700     %model_fmt,
 701 };
 702
 703 my $idedesc = {
 704     optional => 1,
 705     type => 'string', format => $ide_fmt,
 706     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 709
 710 my $scsi_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %queues_fmt,
 714 };
 715 my $scsidesc = {
 716     optional => 1,
 717     type => 'string', format => $scsi_fmt,
 718     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 721
 722 my $sata_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725 };
 726 my $satadesc = {
 727     optional => 1,
 728     type => 'string', format => $sata_fmt,
 729     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 730 };
 731 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 732
 733 my $virtio_fmt = {
 734     %drivedesc_base,
 735     %iothread_fmt,
 736     %rerror_fmt,
 737 };
 738 my $virtiodesc = {
 739     optional => 1,
 740     type => 'string', format => $virtio_fmt,
 741     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 742 };
 743 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 744
 745 my $alldrive_fmt = {
 746     %drivedesc_base,
 747     %rerror_fmt,
 748     %iothread_fmt,
 749     %model_fmt,
 750     %queues_fmt,
 751 };
 752
 753 my $usb_fmt = {
 754     host => {
 755         default_key => 1,
 756         type => 'string', format => 'pve-qm-usb-device',
 757         format_description => 'HOSTUSBDEVICE|spice',
 758         description => 'The Host USB device or port or the value spice',
 759     },
 760     usb3 => {
 761         optional => 1,
 762         type => 'boolean',
 763         format_description => 'yes|no',
 764         description => 'Specifies whether if given host option is a USB3 device or port',
 765     },
 766 };
 767
 768 my $usbdesc = {
 769     optional => 1,
 770     type => 'string', format => $usb_fmt,
 771     description => <<EODESCR,
 772 Configure an USB device (n is 0 to 4). This can be used to
 773 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 774
 775 'bus-port(.port)*' (decimal numbers) or
 776 'vendor_id:product_id' (hexadeciaml numbers) or
 777 'spice'
 778
 779 You can use the 'lsusb -t' command to list existing usb devices.
 780
 781 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 782
 783 The value 'spice' can be used to add a usb redirection devices for spice.
 784
 785 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 786
 787 EODESCR
 788 };
 789 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 790
 791 my $hostpcidesc = {
 792         optional => 1,
 793         type => 'string', format => 'pve-qm-hostpci',
 794         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 795         description => <<EODESCR,
 796 Map host pci devices. HOSTPCIDEVICE syntax is:
 797
 798 'bus:dev.func' (hexadecimal numbers)
 799
 800 You can us the 'lspci' command to list existing pci devices.
 801
 802 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 803
 804 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 805
 806 Experimental: user reported problems with this option.
 807 EODESCR
 808 };
 809 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 810
 811 my $serialdesc = {
 812         optional => 1,
 813         type => 'string',
 814         pattern => '(/dev/.+|socket)',
 815         description =>  <<EODESCR,
 816 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 817
 818 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 819
 820 Experimental: user reported problems with this option.
 821 EODESCR
 822 };
 823
 824 my $paralleldesc= {
 825         optional => 1,
 826         type => 'string',
 827         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 828         description =>  <<EODESCR,
 829 Map host parallel devices (n is 0 to 2).
 830
 831 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 832
 833 Experimental: user reported problems with this option.
 834 EODESCR
 835 };
 836
 837 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 838     $confdesc->{"parallel$i"} = $paralleldesc;
 839 }
 840
 841 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 842     $confdesc->{"serial$i"} = $serialdesc;
 843 }
 844
 845 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 846     $confdesc->{"hostpci$i"} = $hostpcidesc;
 847 }
 848
 849 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 850     $drivename_hash->{"ide$i"} = 1;
 851     $confdesc->{"ide$i"} = $idedesc;
 852 }
 853
 854 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 855     $drivename_hash->{"sata$i"} = 1;
 856     $confdesc->{"sata$i"} = $satadesc;
 857 }
 858
 859 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 860     $drivename_hash->{"scsi$i"} = 1;
 861     $confdesc->{"scsi$i"} = $scsidesc ;
 862 }
 863
 864 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 865     $drivename_hash->{"virtio$i"} = 1;
 866     $confdesc->{"virtio$i"} = $virtiodesc;
 867 }
 868
 869 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 870     $confdesc->{"usb$i"} = $usbdesc;
 871 }
 872
 873 my $unuseddesc = {
 874     optional => 1,
 875     type => 'string', format => 'pve-volume-id',
 876     description => "Reference to unused volumes.",
 877 };
 878
 879 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 880     $confdesc->{"unused$i"} = $unuseddesc;
 881 }
 882
 883 my $kvm_api_version = 0;
 884
 885 sub kvm_version {
 886
 887     return $kvm_api_version if $kvm_api_version;
 888
 889     my $fh = IO::File->new("</dev/kvm") ||
 890         return 0;
 891
 892     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 893         $kvm_api_version = $v;
 894     }
 895
 896     $fh->close();
 897
 898     return  $kvm_api_version;
 899 }
 900
 901 my $kvm_user_version;
 902
 903 sub kvm_user_version {
 904
 905     return $kvm_user_version if $kvm_user_version;
 906
 907     $kvm_user_version = 'unknown';
 908
 909     my $code = sub {
 910         my $line = shift;
 911         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 912             $kvm_user_version = $2;
 913         }
 914     };
 915
 916     eval { run_command("kvm -version", outfunc => $code); };
 917     warn $@ if $@;
 918
 919     return $kvm_user_version;
 920
 921 }
 922
 923 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 924
 925 sub valid_drive_names {
 926     # order is important - used to autoselect boot disk
 927     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 928             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 929             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 930             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 931 }
 932
 933 sub is_valid_drivename {
 934     my $dev = shift;
 935
 936     return defined($drivename_hash->{$dev});
 937 }
 938
 939 sub option_exists {
 940     my $key = shift;
 941     return defined($confdesc->{$key});
 942 }
 943
 944 sub nic_models {
 945     return $nic_model_list;
 946 }
 947
 948 sub os_list_description {
 949
 950     return {
 951         other => 'Other',
 952         wxp => 'Windows XP',
 953         w2k => 'Windows 2000',
 954         w2k3 =>, 'Windows 2003',
 955         w2k8 => 'Windows 2008',
 956         wvista => 'Windows Vista',
 957         win7 => 'Windows 7',
 958         win8 => 'Windows 8/2012',
 959         l24 => 'Linux 2.4',
 960         l26 => 'Linux 2.6',
 961     };
 962 }
 963
 964 my $cdrom_path;
 965
 966 sub get_cdrom_path {
 967
 968     return  $cdrom_path if $cdrom_path;
 969
 970     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 971     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 972     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 973 }
 974
 975 sub get_iso_path {
 976     my ($storecfg, $vmid, $cdrom) = @_;
 977
 978     if ($cdrom eq 'cdrom') {
 979         return get_cdrom_path();
 980     } elsif ($cdrom eq 'none') {
 981         return '';
 982     } elsif ($cdrom =~ m|^/|) {
 983         return $cdrom;
 984     } else {
 985         return PVE::Storage::path($storecfg, $cdrom);
 986     }
 987 }
 988
 989 # try to convert old style file names to volume IDs
 990 sub filename_to_volume_id {
 991     my ($vmid, $file, $media) = @_;
 992
 993     if (!($file eq 'none' || $file eq 'cdrom' ||
 994           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 995
 996         return undef if $file =~ m|/|;
 997
 998         if ($media && $media eq 'cdrom') {
 999             $file = "local:iso/$file";
1000         } else {
1001             $file = "local:$vmid/$file";
1002         }
1003     }
1004
1005     return $file;
1006 }
1007
1008 sub verify_media_type {
1009     my ($opt, $vtype, $media) = @_;
1010
1011     return if !$media;
1012
1013     my $etype;
1014     if ($media eq 'disk') {
1015         $etype = 'images';
1016     } elsif ($media eq 'cdrom') {
1017         $etype = 'iso';
1018     } else {
1019         die "internal error";
1020     }
1021
1022     return if ($vtype eq $etype);
1023
1024     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1025 }
1026
1027 sub cleanup_drive_path {
1028     my ($opt, $storecfg, $drive) = @_;
1029
1030     # try to convert filesystem paths to volume IDs
1031
1032     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1033         ($drive->{file} !~ m|^/dev/.+|) &&
1034         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1035         ($drive->{file} !~ m/^\d+$/)) {
1036         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1037         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1038         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1039         verify_media_type($opt, $vtype, $drive->{media});
1040         $drive->{file} = $volid;
1041     }
1042
1043     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1044 }
1045
1046 sub parse_hotplug_features {
1047     my ($data) = @_;
1048
1049     my $res = {};
1050
1051     return $res if $data eq '0';
1052
1053     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1054
1055     foreach my $feature (PVE::Tools::split_list($data)) {
1056         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1057             $res->{$1} = 1;
1058         } else {
1059             warn "ignoring unknown hotplug feature '$feature'\n";
1060         }
1061     }
1062     return $res;
1063 }
1064
1065 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1066 sub pve_verify_hotplug_features {
1067     my ($value, $noerr) = @_;
1068
1069     return $value if parse_hotplug_features($value);
1070
1071     return undef if $noerr;
1072
1073     die "unable to parse hotplug option\n";
1074 }
1075
1076 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1077 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1078 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1079 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1080 #        [,iothread=on][,serial=serial][,model=model]
1081
1082 sub parse_drive {
1083     my ($key, $data) = @_;
1084
1085     my ($interface, $index);
1086
1087     if ($key =~ m/^([^\d]+)(\d+)$/) {
1088         $interface = $1;
1089         $index = $2;
1090     } else {
1091         return undef;
1092     }
1093
1094     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1095                                      : $confdesc->{$key}->{format};
1096     if (!$desc) {
1097         warn "invalid drive key: $key\n";
1098         return undef;
1099     }
1100     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1101     return undef if !$res;
1102     $res->{interface} = $interface;
1103     $res->{index} = $index;
1104
1105     my $error = 0;
1106     foreach my $opt (qw(bps bps_rd bps_wr)) {
1107         if (my $bps = defined(delete $res->{$opt})) {
1108             if (defined($res->{"m$opt"})) {
1109                 warn "both $opt and m$opt specified\n";
1110                 ++$error;
1111                 next;
1112             }
1113             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1114         }
1115     }
1116     return undef if $error;
1117
1118     return undef if $res->{mbps_rd} && $res->{mbps};
1119     return undef if $res->{mbps_wr} && $res->{mbps};
1120     return undef if $res->{iops_rd} && $res->{iops};
1121     return undef if $res->{iops_wr} && $res->{iops};
1122
1123     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1124         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1125         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1126         return undef if $res->{interface} eq 'virtio';
1127     }
1128
1129     if (my $size = $res->{size}) {
1130         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1131     }
1132
1133     return $res;
1134 }
1135
1136 sub print_drive {
1137     my ($vmid, $drive) = @_;
1138     my $data = { %$drive };
1139     delete $data->{$_} for qw(index interface);
1140     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1141 }
1142
1143 sub scsi_inquiry {
1144     my($fh, $noerr) = @_;
1145
1146     my $SG_IO = 0x2285;
1147     my $SG_GET_VERSION_NUM = 0x2282;
1148
1149     my $versionbuf = "\x00" x 8;
1150     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1151     if (!$ret) {
1152         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1153         return undef;
1154     }
1155     my $version = unpack("I", $versionbuf);
1156     if ($version < 30000) {
1157         die "scsi generic interface too old\n"  if !$noerr;
1158         return undef;
1159     }
1160
1161     my $buf = "\x00" x 36;
1162     my $sensebuf = "\x00" x 8;
1163     my $cmd = pack("C x3 C x1", 0x12, 36);
1164
1165     # see /usr/include/scsi/sg.h
1166     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1167
1168     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1169                       length($sensebuf), 0, length($buf), $buf,
1170                       $cmd, $sensebuf, 6000);
1171
1172     $ret = ioctl($fh, $SG_IO, $packet);
1173     if (!$ret) {
1174         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1175         return undef;
1176     }
1177
1178     my @res = unpack($sg_io_hdr_t, $packet);
1179     if ($res[17] || $res[18]) {
1180         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1181         return undef;
1182     }
1183
1184     my $res = {};
1185     (my $byte0, my $byte1, $res->{vendor},
1186      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1187
1188     $res->{removable} = $byte1 & 128 ? 1 : 0;
1189     $res->{type} = $byte0 & 31;
1190
1191     return $res;
1192 }
1193
1194 sub path_is_scsi {
1195     my ($path) = @_;
1196
1197     my $fh = IO::File->new("+<$path") || return undef;
1198     my $res = scsi_inquiry($fh, 1);
1199     close($fh);
1200
1201     return $res;
1202 }
1203
1204 sub machine_type_is_q35 {
1205     my ($conf) = @_;
1206
1207     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1208 }
1209
1210 sub print_tabletdevice_full {
1211     my ($conf) = @_;
1212
1213     my $q35 = machine_type_is_q35($conf);
1214
1215     # we use uhci for old VMs because tablet driver was buggy in older qemu
1216     my $usbbus = $q35 ? "ehci" : "uhci";
1217
1218     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1219 }
1220
1221 sub print_drivedevice_full {
1222     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1223
1224     my $device = '';
1225     my $maxdev = 0;
1226
1227     if ($drive->{interface} eq 'virtio') {
1228         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1229         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1230         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1231     } elsif ($drive->{interface} eq 'scsi') {
1232
1233         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1234         my $unit = $drive->{index} % $maxdev;
1235         my $devicetype = 'hd';
1236         my $path = '';
1237         if (drive_is_cdrom($drive)) {
1238             $devicetype = 'cd';
1239         } else {
1240             if ($drive->{file} =~ m|^/|) {
1241                 $path = $drive->{file};
1242                 if (my $info = path_is_scsi($path)) {
1243                     if ($info->{type} == 0) {
1244                         $devicetype = 'block';
1245                     } elsif ($info->{type} == 1) { # tape
1246                         $devicetype = 'generic';
1247                     }
1248                 }
1249             } else {
1250                  $path = PVE::Storage::path($storecfg, $drive->{file});
1251             }
1252
1253             if($path =~ m/^iscsi\:\/\//){
1254                 $devicetype = 'generic';
1255             }
1256         }
1257
1258         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1259             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1260         } else {
1261             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1262         }
1263
1264     } elsif ($drive->{interface} eq 'ide'){
1265         $maxdev = 2;
1266         my $controller = int($drive->{index} / $maxdev);
1267         my $unit = $drive->{index} % $maxdev;
1268         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1269
1270         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1271         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1272             $model = URI::Escape::uri_unescape($model);
1273             $device .= ",model=$model";
1274         }
1275     } elsif ($drive->{interface} eq 'sata'){
1276         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1277         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1278         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1279     } elsif ($drive->{interface} eq 'usb') {
1280         die "implement me";
1281         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1282     } else {
1283         die "unsupported interface type";
1284     }
1285
1286     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1287
1288     return $device;
1289 }
1290
1291 sub get_initiator_name {
1292     my $initiator;
1293
1294     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1295     while (defined(my $line = <$fh>)) {
1296         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1297         $initiator = $1;
1298         last;
1299     }
1300     $fh->close();
1301
1302     return $initiator;
1303 }
1304
1305 sub print_drive_full {
1306     my ($storecfg, $vmid, $drive) = @_;
1307
1308     my $path;
1309     my $volid = $drive->{file};
1310     my $format;
1311
1312     if (drive_is_cdrom($drive)) {
1313         $path = get_iso_path($storecfg, $vmid, $volid);
1314     } else {
1315         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1316         if ($storeid) {
1317             $path = PVE::Storage::path($storecfg, $volid);
1318             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1319             $format = qemu_img_format($scfg, $volname);
1320         } else {
1321             $path = $volid;
1322             $format = "raw";
1323         }
1324    }
1325
1326     my $opts = '';
1327     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1328     foreach my $o (@qemu_drive_options) {
1329         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1330     }
1331     if (my $serial = $drive->{serial}) {
1332         $serial = URI::Escape::uri_unescape($serial);
1333         $opts .= ",serial=$serial";
1334     }
1335
1336     $opts .= ",format=$format" if $format && !$drive->{format};
1337
1338     foreach my $o (qw(bps bps_rd bps_wr)) {
1339         my $v = $drive->{"m$o"};
1340         $opts .= ",$o=" . int($v*1024*1024) if $v;
1341     }
1342
1343     my $cache_direct = 0;
1344
1345     if (my $cache = $drive->{cache}) {
1346         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1347     } elsif (!drive_is_cdrom($drive)) {
1348         $opts .= ",cache=none";
1349         $cache_direct = 1;
1350     }
1351
1352     # aio native works only with O_DIRECT
1353     if (!$drive->{aio}) {
1354         if($cache_direct) {
1355             $opts .= ",aio=native";
1356         } else {
1357             $opts .= ",aio=threads";
1358         }
1359     }
1360
1361     if (!drive_is_cdrom($drive)) {
1362         my $detectzeroes;
1363         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1364             $detectzeroes = 'off';
1365         } elsif ($drive->{discard}) {
1366             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1367         } else {
1368             # This used to be our default with discard not being specified:
1369             $detectzeroes = 'on';
1370         }
1371         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1372     }
1373
1374     my $pathinfo = $path ? "file=$path," : '';
1375
1376     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1377 }
1378
1379 sub print_netdevice_full {
1380     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1381
1382     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1383
1384     my $device = $net->{model};
1385     if ($net->{model} eq 'virtio') {
1386          $device = 'virtio-net-pci';
1387      };
1388
1389     my $pciaddr = print_pci_addr("$netid", $bridges);
1390     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1391     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1392         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1393         my $vectors = $net->{queues} * 2 + 2;
1394         $tmpstr .= ",vectors=$vectors,mq=on";
1395     }
1396     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1397
1398     if ($use_old_bios_files) {
1399         my $romfile;
1400         if ($device eq 'virtio-net-pci') {
1401             $romfile = 'pxe-virtio.rom';
1402         } elsif ($device eq 'e1000') {
1403             $romfile = 'pxe-e1000.rom';
1404         } elsif ($device eq 'ne2k') {
1405             $romfile = 'pxe-ne2k_pci.rom';
1406         } elsif ($device eq 'pcnet') {
1407             $romfile = 'pxe-pcnet.rom';
1408         } elsif ($device eq 'rtl8139') {
1409             $romfile = 'pxe-rtl8139.rom';
1410         }
1411         $tmpstr .= ",romfile=$romfile" if $romfile;
1412     }
1413
1414     return $tmpstr;
1415 }
1416
1417 sub print_netdev_full {
1418     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1419
1420     my $i = '';
1421     if ($netid =~ m/^net(\d+)$/) {
1422         $i = int($1);
1423     }
1424
1425     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1426
1427     my $ifname = "tap${vmid}i$i";
1428
1429     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1430     die "interface name '$ifname' is too long (max 15 character)\n"
1431         if length($ifname) >= 16;
1432
1433     my $vhostparam = '';
1434     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1435
1436     my $vmname = $conf->{name} || "vm$vmid";
1437
1438     my $netdev = "";
1439     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1440
1441     if ($net->{bridge}) {
1442         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1443     } else {
1444         $netdev = "type=user,id=$netid,hostname=$vmname";
1445     }
1446
1447     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1448
1449     return $netdev;
1450 }
1451
1452 sub drive_is_cdrom {
1453     my ($drive) = @_;
1454
1455     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1456
1457 }
1458
1459 sub parse_numa {
1460     my ($data) = @_;
1461
1462     my $res = {};
1463
1464     foreach my $kvp (split(/,/, $data)) {
1465
1466         if ($kvp =~ m/^memory=(\S+)$/) {
1467             $res->{memory} = $1;
1468         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1469             $res->{policy} = $1;
1470         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1471             $res->{cpus}->{start} = $1;
1472             $res->{cpus}->{end} = $3;
1473         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1474             $res->{hostnodes}->{start} = $1;
1475             $res->{hostnodes}->{end} = $3;
1476         } else {
1477             return undef;
1478         }
1479     }
1480
1481     return $res;
1482 }
1483
1484 sub parse_hostpci {
1485     my ($value) = @_;
1486
1487     return undef if !$value;
1488
1489
1490     my @list = split(/,/, $value);
1491     my $found;
1492
1493     my $res = {};
1494     foreach my $kv (@list) {
1495
1496         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1497             $found = 1;
1498             if(defined($4)){
1499                 push @{$res->{pciid}}, { id => $2 , function => $4};
1500
1501             }else{
1502                 my $pcidevices = lspci($2);
1503                 $res->{pciid} = $pcidevices->{$2};
1504             }
1505         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1506             $res->{rombar} = $1;
1507         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1508             $res->{'x-vga'} = $1;
1509         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1510             $res->{pcie} = 1 if $1 == 1;
1511         } else {
1512             warn "unknown hostpci setting '$kv'\n";
1513         }
1514     }
1515
1516     return undef if !$found;
1517
1518     return $res;
1519 }
1520
1521 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1522 sub parse_net {
1523     my ($data) = @_;
1524
1525     my $res = {};
1526
1527     foreach my $kvp (split(/,/, $data)) {
1528
1529         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1530             my $model = lc($1);
1531             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1532             $res->{model} = $model;
1533             $res->{macaddr} = $mac;
1534         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1535             $res->{bridge} = $1;
1536         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1537             $res->{queues} = $1;
1538         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1539             $res->{rate} = $1;
1540         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1541             $res->{tag} = $1;
1542         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1543             $res->{trunks} = $1;
1544         } elsif ($kvp =~ m/^firewall=([01])$/) {
1545             $res->{firewall} = $1;
1546         } elsif ($kvp =~ m/^link_down=([01])$/) {
1547             $res->{link_down} = $1;
1548         } else {
1549             return undef;
1550         }
1551
1552     }
1553
1554     return undef if !$res->{model};
1555
1556     return $res;
1557 }
1558
1559 sub print_net {
1560     my $net = shift;
1561
1562     my $res = "$net->{model}";
1563     $res .= "=$net->{macaddr}" if $net->{macaddr};
1564     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1565     $res .= ",rate=$net->{rate}" if $net->{rate};
1566     $res .= ",tag=$net->{tag}" if $net->{tag};
1567     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1568     $res .= ",firewall=1" if $net->{firewall};
1569     $res .= ",link_down=1" if $net->{link_down};
1570     $res .= ",queues=$net->{queues}" if $net->{queues};
1571
1572     return $res;
1573 }
1574
1575 sub add_random_macs {
1576     my ($settings) = @_;
1577
1578     foreach my $opt (keys %$settings) {
1579         next if $opt !~ m/^net(\d+)$/;
1580         my $net = parse_net($settings->{$opt});
1581         next if !$net;
1582         $settings->{$opt} = print_net($net);
1583     }
1584 }
1585
1586 sub vm_is_volid_owner {
1587     my ($storecfg, $vmid, $volid) = @_;
1588
1589     if ($volid !~  m|^/|) {
1590         my ($path, $owner);
1591         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1592         if ($owner && ($owner == $vmid)) {
1593             return 1;
1594         }
1595     }
1596
1597     return undef;
1598 }
1599
1600 sub split_flagged_list {
1601     my $text = shift || '';
1602     $text =~ s/[,;]/ /g;
1603     $text =~ s/^\s+//;
1604     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1605 }
1606
1607 sub join_flagged_list {
1608     my ($how, $lst) = @_;
1609     join $how, map { $lst->{$_} . $_ } keys %$lst;
1610 }
1611
1612 sub vmconfig_delete_pending_option {
1613     my ($conf, $key, $force) = @_;
1614
1615     delete $conf->{pending}->{$key};
1616     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1617     $pending_delete_hash->{$key} = $force ? '!' : '';
1618     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1619 }
1620
1621 sub vmconfig_undelete_pending_option {
1622     my ($conf, $key) = @_;
1623
1624     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1625     delete $pending_delete_hash->{$key};
1626
1627     if (%$pending_delete_hash) {
1628         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1629     } else {
1630         delete $conf->{pending}->{delete};
1631     }
1632 }
1633
1634 sub vmconfig_register_unused_drive {
1635     my ($storecfg, $vmid, $conf, $drive) = @_;
1636
1637     if (!drive_is_cdrom($drive)) {
1638         my $volid = $drive->{file};
1639         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1640             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1641         }
1642     }
1643 }
1644
1645 sub vmconfig_cleanup_pending {
1646     my ($conf) = @_;
1647
1648     # remove pending changes when nothing changed
1649     my $changes;
1650     foreach my $opt (keys %{$conf->{pending}}) {
1651         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1652             $changes = 1;
1653             delete $conf->{pending}->{$opt};
1654         }
1655     }
1656
1657     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1658     my $pending_delete_hash = {};
1659     while (my ($opt, $force) = each %$current_delete_hash) {
1660         if (defined($conf->{$opt})) {
1661             $pending_delete_hash->{$opt} = $force;
1662         } else {
1663             $changes = 1;
1664         }
1665     }
1666
1667     if (%$pending_delete_hash) {
1668         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1669     } else {
1670         delete $conf->{pending}->{delete};
1671     }
1672
1673     return $changes;
1674 }
1675
1676 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1677 my $smbios1_fmt = {
1678     uuid => {
1679         type => 'string',
1680         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1681         format_description => 'UUID',
1682         optional => 1,
1683     },
1684     version => {
1685         type => 'string',
1686         pattern => '\S+',
1687         format_description => 'str',
1688         optional => 1,
1689     },
1690     serial => {
1691         type => 'string',
1692         pattern => '\S+',
1693         format_description => 'str',
1694         optional => 1,
1695     },
1696     manufacturer => {
1697         type => 'string',
1698         pattern => '\S+',
1699         format_description => 'name',
1700         optional => 1,
1701     },
1702     product => {
1703         type => 'string',
1704         pattern => '\S+',
1705         format_description => 'name',
1706         optional => 1,
1707     },
1708     sku => {
1709         type => 'string',
1710         pattern => '\S+',
1711         format_description => 'str',
1712         optional => 1,
1713     },
1714     family => {
1715         type => 'string',
1716         pattern => '\S+',
1717         format_description => 'str',
1718         optional => 1,
1719     },
1720 };
1721
1722 sub parse_smbios1 {
1723     my ($data) = @_;
1724
1725     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1726     warn $@ if $@;
1727     return $res;
1728 }
1729
1730 sub print_smbios1 {
1731     my ($smbios1) = @_;
1732     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1733 }
1734
1735 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1736
1737 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1738 sub verify_bootdisk {
1739     my ($value, $noerr) = @_;
1740
1741     return $value if is_valid_drivename($value);
1742
1743     return undef if $noerr;
1744
1745     die "invalid boot disk '$value'\n";
1746 }
1747
1748 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1749 sub verify_numa {
1750     my ($value, $noerr) = @_;
1751
1752     return $value if parse_numa($value);
1753
1754     return undef if $noerr;
1755
1756     die "unable to parse numa options\n";
1757 }
1758
1759 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1760 sub verify_net {
1761     my ($value, $noerr) = @_;
1762
1763     return $value if parse_net($value);
1764
1765     return undef if $noerr;
1766
1767     die "unable to parse network options\n";
1768 }
1769
1770 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1771 sub verify_hostpci {
1772     my ($value, $noerr) = @_;
1773
1774     return $value if parse_hostpci($value);
1775
1776     return undef if $noerr;
1777
1778     die "unable to parse pci id\n";
1779 }
1780
1781 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1782 sub verify_watchdog {
1783     my ($value, $noerr) = @_;
1784
1785     return $value if parse_watchdog($value);
1786
1787     return undef if $noerr;
1788
1789     die "unable to parse watchdog options\n";
1790 }
1791
1792 sub parse_watchdog {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my $res = {};
1798
1799     foreach my $p (split(/,/, $value)) {
1800         next if $p =~ m/^\s*$/;
1801
1802         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1803             $res->{model} = $2;
1804         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1805             $res->{action} = $2;
1806         } else {
1807             return undef;
1808         }
1809     }
1810
1811     return $res;
1812 }
1813
1814 sub parse_usb_device {
1815     my ($value) = @_;
1816
1817     return undef if !$value;
1818
1819     my $res = {};
1820     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1821         $res->{vendorid} = $2;
1822         $res->{productid} = $4;
1823     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1824         $res->{hostbus} = $1;
1825         $res->{hostport} = $2;
1826     } elsif ($value =~ m/^spice$/i) {
1827         $res->{spice} = 1;
1828     } else {
1829         return undef;
1830     }
1831
1832     return $res;
1833 }
1834
1835 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1836 sub verify_usb_device {
1837     my ($value, $noerr) = @_;
1838
1839     return $value if parse_usb_device($value);
1840
1841     return undef if $noerr;
1842
1843     die "unable to parse usb device\n";
1844 }
1845
1846 # add JSON properties for create and set function
1847 sub json_config_properties {
1848     my $prop = shift;
1849
1850     foreach my $opt (keys %$confdesc) {
1851         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1852         $prop->{$opt} = $confdesc->{$opt};
1853     }
1854
1855     return $prop;
1856 }
1857
1858 sub check_type {
1859     my ($key, $value) = @_;
1860
1861     die "unknown setting '$key'\n" if !$confdesc->{$key};
1862
1863     my $type = $confdesc->{$key}->{type};
1864
1865     if (!defined($value)) {
1866         die "got undefined value\n";
1867     }
1868
1869     if ($value =~ m/[\n\r]/) {
1870         die "property contains a line feed\n";
1871     }
1872
1873     if ($type eq 'boolean') {
1874         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1875         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1876         die "type check ('boolean') failed - got '$value'\n";
1877     } elsif ($type eq 'integer') {
1878         return int($1) if $value =~ m/^(\d+)$/;
1879         die "type check ('integer') failed - got '$value'\n";
1880     } elsif ($type eq 'number') {
1881         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1882         die "type check ('number') failed - got '$value'\n";
1883     } elsif ($type eq 'string') {
1884         if (my $fmt = $confdesc->{$key}->{format}) {
1885             if ($fmt eq 'pve-qm-drive') {
1886                 # special case - we need to pass $key to parse_drive()
1887                 my $drive = parse_drive($key, $value);
1888                 return $value if $drive;
1889                 die "unable to parse drive options\n";
1890             }
1891             PVE::JSONSchema::check_format($fmt, $value);
1892             return $value;
1893         }
1894         $value =~ s/^\"(.*)\"$/$1/;
1895         return $value;
1896     } else {
1897         die "internal error"
1898     }
1899 }
1900
1901 sub check_iommu_support{
1902     #fixme : need to check IOMMU support
1903     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1904
1905     my $iommu=1;
1906     return $iommu;
1907
1908 }
1909
1910 sub touch_config {
1911     my ($vmid) = @_;
1912
1913     my $conf = PVE::QemuConfig->config_file($vmid);
1914     utime undef, undef, $conf;
1915 }
1916
1917 sub destroy_vm {
1918     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1919
1920     my $conffile = PVE::QemuConfig->config_file($vmid);
1921
1922     my $conf = PVE::QemuConfig->load_config($vmid);
1923
1924     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1925
1926     # only remove disks owned by this VM
1927     foreach_drive($conf, sub {
1928         my ($ds, $drive) = @_;
1929
1930         return if drive_is_cdrom($drive);
1931
1932         my $volid = $drive->{file};
1933
1934         return if !$volid || $volid =~ m|^/|;
1935
1936         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1937         return if !$path || !$owner || ($owner != $vmid);
1938
1939         PVE::Storage::vdisk_free($storecfg, $volid);
1940     });
1941
1942     if ($keep_empty_config) {
1943         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1944     } else {
1945         unlink $conffile;
1946     }
1947
1948     # also remove unused disk
1949     eval {
1950         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1951
1952         eval {
1953             PVE::Storage::foreach_volid($dl, sub {
1954                 my ($volid, $sid, $volname, $d) = @_;
1955                 PVE::Storage::vdisk_free($storecfg, $volid);
1956             });
1957         };
1958         warn $@ if $@;
1959
1960     };
1961     warn $@ if $@;
1962 }
1963
1964 sub parse_vm_config {
1965     my ($filename, $raw) = @_;
1966
1967     return undef if !defined($raw);
1968
1969     my $res = {
1970         digest => Digest::SHA::sha1_hex($raw),
1971         snapshots => {},
1972         pending => {},
1973     };
1974
1975     $filename =~ m|/qemu-server/(\d+)\.conf$|
1976         || die "got strange filename '$filename'";
1977
1978     my $vmid = $1;
1979
1980     my $conf = $res;
1981     my $descr;
1982     my $section = '';
1983
1984     my @lines = split(/\n/, $raw);
1985     foreach my $line (@lines) {
1986         next if $line =~ m/^\s*$/;
1987
1988         if ($line =~ m/^\[PENDING\]\s*$/i) {
1989             $section = 'pending';
1990             if (defined($descr)) {
1991                 $descr =~ s/\s+$//;
1992                 $conf->{description} = $descr;
1993             }
1994             $descr = undef;
1995             $conf = $res->{$section} = {};
1996             next;
1997
1998         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1999             $section = $1;
2000             if (defined($descr)) {
2001                 $descr =~ s/\s+$//;
2002                 $conf->{description} = $descr;
2003             }
2004             $descr = undef;
2005             $conf = $res->{snapshots}->{$section} = {};
2006             next;
2007         }
2008
2009         if ($line =~ m/^\#(.*)\s*$/) {
2010             $descr = '' if !defined($descr);
2011             $descr .= PVE::Tools::decode_text($1) . "\n";
2012             next;
2013         }
2014
2015         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2016             $descr = '' if !defined($descr);
2017             $descr .= PVE::Tools::decode_text($2);
2018         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2019             $conf->{snapstate} = $1;
2020         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2021             my $key = $1;
2022             my $value = $2;
2023             $conf->{$key} = $value;
2024         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2025             my $value = $1;
2026             if ($section eq 'pending') {
2027                 $conf->{delete} = $value; # we parse this later
2028             } else {
2029                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2030             }
2031         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2032             my $key = $1;
2033             my $value = $2;
2034             eval { $value = check_type($key, $value); };
2035             if ($@) {
2036                 warn "vm $vmid - unable to parse value of '$key' - $@";
2037             } else {
2038                 my $fmt = $confdesc->{$key}->{format};
2039                 if ($fmt && $fmt eq 'pve-qm-drive') {
2040                     my $v = parse_drive($key, $value);
2041                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2042                         $v->{file} = $volid;
2043                         $value = print_drive($vmid, $v);
2044                     } else {
2045                         warn "vm $vmid - unable to parse value of '$key'\n";
2046                         next;
2047                     }
2048                 }
2049
2050                 if ($key eq 'cdrom') {
2051                     $conf->{ide2} = $value;
2052                 } else {
2053                     $conf->{$key} = $value;
2054                 }
2055             }
2056         }
2057     }
2058
2059     if (defined($descr)) {
2060         $descr =~ s/\s+$//;
2061         $conf->{description} = $descr;
2062     }
2063     delete $res->{snapstate}; # just to be sure
2064
2065     return $res;
2066 }
2067
2068 sub write_vm_config {
2069     my ($filename, $conf) = @_;
2070
2071     delete $conf->{snapstate}; # just to be sure
2072
2073     if ($conf->{cdrom}) {
2074         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2075         $conf->{ide2} = $conf->{cdrom};
2076         delete $conf->{cdrom};
2077     }
2078
2079     # we do not use 'smp' any longer
2080     if ($conf->{sockets}) {
2081         delete $conf->{smp};
2082     } elsif ($conf->{smp}) {
2083         $conf->{sockets} = $conf->{smp};
2084         delete $conf->{cores};
2085         delete $conf->{smp};
2086     }
2087
2088     my $used_volids = {};
2089
2090     my $cleanup_config = sub {
2091         my ($cref, $pending, $snapname) = @_;
2092
2093         foreach my $key (keys %$cref) {
2094             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2095                 $key eq 'snapstate' || $key eq 'pending';
2096             my $value = $cref->{$key};
2097             if ($key eq 'delete') {
2098                 die "propertry 'delete' is only allowed in [PENDING]\n"
2099                     if !$pending;
2100                 # fixme: check syntax?
2101                 next;
2102             }
2103             eval { $value = check_type($key, $value); };
2104             die "unable to parse value of '$key' - $@" if $@;
2105
2106             $cref->{$key} = $value;
2107
2108             if (!$snapname && is_valid_drivename($key)) {
2109                 my $drive = parse_drive($key, $value);
2110                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2111             }
2112         }
2113     };
2114
2115     &$cleanup_config($conf);
2116
2117     &$cleanup_config($conf->{pending}, 1);
2118
2119     foreach my $snapname (keys %{$conf->{snapshots}}) {
2120         die "internal error" if $snapname eq 'pending';
2121         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2122     }
2123
2124     # remove 'unusedX' settings if we re-add a volume
2125     foreach my $key (keys %$conf) {
2126         my $value = $conf->{$key};
2127         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2128             delete $conf->{$key};
2129         }
2130     }
2131
2132     my $generate_raw_config = sub {
2133         my ($conf, $pending) = @_;
2134
2135         my $raw = '';
2136
2137         # add description as comment to top of file
2138         if (defined(my $descr = $conf->{description})) {
2139             if ($descr) {
2140                 foreach my $cl (split(/\n/, $descr)) {
2141                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2142                 }
2143             } else {
2144                 $raw .= "#\n" if $pending;
2145             }
2146         }
2147
2148         foreach my $key (sort keys %$conf) {
2149             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2150             $raw .= "$key: $conf->{$key}\n";
2151         }
2152         return $raw;
2153     };
2154
2155     my $raw = &$generate_raw_config($conf);
2156
2157     if (scalar(keys %{$conf->{pending}})){
2158         $raw .= "\n[PENDING]\n";
2159         $raw .= &$generate_raw_config($conf->{pending}, 1);
2160     }
2161
2162     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2163         $raw .= "\n[$snapname]\n";
2164         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2165     }
2166
2167     return $raw;
2168 }
2169
2170 sub load_defaults {
2171
2172     my $res = {};
2173
2174     # we use static defaults from our JSON schema configuration
2175     foreach my $key (keys %$confdesc) {
2176         if (defined(my $default = $confdesc->{$key}->{default})) {
2177             $res->{$key} = $default;
2178         }
2179     }
2180
2181     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2182     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2183
2184     return $res;
2185 }
2186
2187 sub config_list {
2188     my $vmlist = PVE::Cluster::get_vmlist();
2189     my $res = {};
2190     return $res if !$vmlist || !$vmlist->{ids};
2191     my $ids = $vmlist->{ids};
2192
2193     foreach my $vmid (keys %$ids) {
2194         my $d = $ids->{$vmid};
2195         next if !$d->{node} || $d->{node} ne $nodename;
2196         next if !$d->{type} || $d->{type} ne 'qemu';
2197         $res->{$vmid}->{exists} = 1;
2198     }
2199     return $res;
2200 }
2201
2202 # test if VM uses local resources (to prevent migration)
2203 sub check_local_resources {
2204     my ($conf, $noerr) = @_;
2205
2206     my $loc_res = 0;
2207
2208     $loc_res = 1 if $conf->{hostusb}; # old syntax
2209     $loc_res = 1 if $conf->{hostpci}; # old syntax
2210
2211     foreach my $k (keys %$conf) {
2212         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2213         # sockets are safe: they will recreated be on the target side post-migrate
2214         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2215         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2216     }
2217
2218     die "VM uses local resources\n" if $loc_res && !$noerr;
2219
2220     return $loc_res;
2221 }
2222
2223 # check if used storages are available on all nodes (use by migrate)
2224 sub check_storage_availability {
2225     my ($storecfg, $conf, $node) = @_;
2226
2227     foreach_drive($conf, sub {
2228         my ($ds, $drive) = @_;
2229
2230         my $volid = $drive->{file};
2231         return if !$volid;
2232
2233         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2234         return if !$sid;
2235
2236         # check if storage is available on both nodes
2237         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2238         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2239    });
2240 }
2241
2242 # list nodes where all VM images are available (used by has_feature API)
2243 sub shared_nodes {
2244     my ($conf, $storecfg) = @_;
2245
2246     my $nodelist = PVE::Cluster::get_nodelist();
2247     my $nodehash = { map { $_ => 1 } @$nodelist };
2248     my $nodename = PVE::INotify::nodename();
2249
2250     foreach_drive($conf, sub {
2251         my ($ds, $drive) = @_;
2252
2253         my $volid = $drive->{file};
2254         return if !$volid;
2255
2256         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2257         if ($storeid) {
2258             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2259             if ($scfg->{disable}) {
2260                 $nodehash = {};
2261             } elsif (my $avail = $scfg->{nodes}) {
2262                 foreach my $node (keys %$nodehash) {
2263                     delete $nodehash->{$node} if !$avail->{$node};
2264                 }
2265             } elsif (!$scfg->{shared}) {
2266                 foreach my $node (keys %$nodehash) {
2267                     delete $nodehash->{$node} if $node ne $nodename
2268                 }
2269             }
2270         }
2271     });
2272
2273     return $nodehash
2274 }
2275
2276 sub check_cmdline {
2277     my ($pidfile, $pid) = @_;
2278
2279     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2280     if (defined($fh)) {
2281         my $line = <$fh>;
2282         $fh->close;
2283         return undef if !$line;
2284         my @param = split(/\0/, $line);
2285
2286         my $cmd = $param[0];
2287         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2288
2289         for (my $i = 0; $i < scalar (@param); $i++) {
2290             my $p = $param[$i];
2291             next if !$p;
2292             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2293                 my $p = $param[$i+1];
2294                 return 1 if $p && ($p eq $pidfile);
2295                 return undef;
2296             }
2297         }
2298     }
2299     return undef;
2300 }
2301
2302 sub check_running {
2303     my ($vmid, $nocheck, $node) = @_;
2304
2305     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2306
2307     die "unable to find configuration file for VM $vmid - no such machine\n"
2308         if !$nocheck && ! -f $filename;
2309
2310     my $pidfile = pidfile_name($vmid);
2311
2312     if (my $fd = IO::File->new("<$pidfile")) {
2313         my $st = stat($fd);
2314         my $line = <$fd>;
2315         close($fd);
2316
2317         my $mtime = $st->mtime;
2318         if ($mtime > time()) {
2319             warn "file '$filename' modified in future\n";
2320         }
2321
2322         if ($line =~ m/^(\d+)$/) {
2323             my $pid = $1;
2324             if (check_cmdline($pidfile, $pid)) {
2325                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2326                     return $pid;
2327                 }
2328             }
2329         }
2330     }
2331
2332     return undef;
2333 }
2334
2335 sub vzlist {
2336
2337     my $vzlist = config_list();
2338
2339     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2340
2341     while (defined(my $de = $fd->read)) {
2342         next if $de !~ m/^(\d+)\.pid$/;
2343         my $vmid = $1;
2344         next if !defined($vzlist->{$vmid});
2345         if (my $pid = check_running($vmid)) {
2346             $vzlist->{$vmid}->{pid} = $pid;
2347         }
2348     }
2349
2350     return $vzlist;
2351 }
2352
2353 sub disksize {
2354     my ($storecfg, $conf) = @_;
2355
2356     my $bootdisk = $conf->{bootdisk};
2357     return undef if !$bootdisk;
2358     return undef if !is_valid_drivename($bootdisk);
2359
2360     return undef if !$conf->{$bootdisk};
2361
2362     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2363     return undef if !defined($drive);
2364
2365     return undef if drive_is_cdrom($drive);
2366
2367     my $volid = $drive->{file};
2368     return undef if !$volid;
2369
2370     return $drive->{size};
2371 }
2372
2373 my $last_proc_pid_stat;
2374
2375 # get VM status information
2376 # This must be fast and should not block ($full == false)
2377 # We only query KVM using QMP if $full == true (this can be slow)
2378 sub vmstatus {
2379     my ($opt_vmid, $full) = @_;
2380
2381     my $res = {};
2382
2383     my $storecfg = PVE::Storage::config();
2384
2385     my $list = vzlist();
2386     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2387
2388     my $cpucount = $cpuinfo->{cpus} || 1;
2389
2390     foreach my $vmid (keys %$list) {
2391         next if $opt_vmid && ($vmid ne $opt_vmid);
2392
2393         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2394         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2395
2396         my $d = {};
2397         $d->{pid} = $list->{$vmid}->{pid};
2398
2399         # fixme: better status?
2400         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2401
2402         my $size = disksize($storecfg, $conf);
2403         if (defined($size)) {
2404             $d->{disk} = 0; # no info available
2405             $d->{maxdisk} = $size;
2406         } else {
2407             $d->{disk} = 0;
2408             $d->{maxdisk} = 0;
2409         }
2410
2411         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2412         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2413         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2414
2415         $d->{name} = $conf->{name} || "VM $vmid";
2416         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2417
2418         if ($conf->{balloon}) {
2419             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2420             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2421         }
2422
2423         $d->{uptime} = 0;
2424         $d->{cpu} = 0;
2425         $d->{mem} = 0;
2426
2427         $d->{netout} = 0;
2428         $d->{netin} = 0;
2429
2430         $d->{diskread} = 0;
2431         $d->{diskwrite} = 0;
2432
2433         $d->{template} = PVE::QemuConfig->is_template($conf);
2434
2435         $res->{$vmid} = $d;
2436     }
2437
2438     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2439     foreach my $dev (keys %$netdev) {
2440         next if $dev !~ m/^tap([1-9]\d*)i/;
2441         my $vmid = $1;
2442         my $d = $res->{$vmid};
2443         next if !$d;
2444
2445         $d->{netout} += $netdev->{$dev}->{receive};
2446         $d->{netin} += $netdev->{$dev}->{transmit};
2447
2448         if ($full) {
2449             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2450             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2451         }
2452
2453     }
2454
2455     my $ctime = gettimeofday;
2456
2457     foreach my $vmid (keys %$list) {
2458
2459         my $d = $res->{$vmid};
2460         my $pid = $d->{pid};
2461         next if !$pid;
2462
2463         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2464         next if !$pstat; # not running
2465
2466         my $used = $pstat->{utime} + $pstat->{stime};
2467
2468         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2469
2470         if ($pstat->{vsize}) {
2471             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2472         }
2473
2474         my $old = $last_proc_pid_stat->{$pid};
2475         if (!$old) {
2476             $last_proc_pid_stat->{$pid} = {
2477                 time => $ctime,
2478                 used => $used,
2479                 cpu => 0,
2480             };
2481             next;
2482         }
2483
2484         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2485
2486         if ($dtime > 1000) {
2487             my $dutime = $used -  $old->{used};
2488
2489             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2490             $last_proc_pid_stat->{$pid} = {
2491                 time => $ctime,
2492                 used => $used,
2493                 cpu => $d->{cpu},
2494             };
2495         } else {
2496             $d->{cpu} = $old->{cpu};
2497         }
2498     }
2499
2500     return $res if !$full;
2501
2502     my $qmpclient = PVE::QMPClient->new();
2503
2504     my $ballooncb = sub {
2505         my ($vmid, $resp) = @_;
2506
2507         my $info = $resp->{'return'};
2508         return if !$info->{max_mem};
2509
2510         my $d = $res->{$vmid};
2511
2512         # use memory assigned to VM
2513         $d->{maxmem} = $info->{max_mem};
2514         $d->{balloon} = $info->{actual};
2515
2516         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2517             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2518             $d->{freemem} = $info->{free_mem};
2519         }
2520
2521         $d->{ballooninfo} = $info;
2522     };
2523
2524     my $blockstatscb = sub {
2525         my ($vmid, $resp) = @_;
2526         my $data = $resp->{'return'} || [];
2527         my $totalrdbytes = 0;
2528         my $totalwrbytes = 0;
2529
2530         for my $blockstat (@$data) {
2531             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2532             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2533
2534             $blockstat->{device} =~ s/drive-//;
2535             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2536         }
2537         $res->{$vmid}->{diskread} = $totalrdbytes;
2538         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2539     };
2540
2541     my $statuscb = sub {
2542         my ($vmid, $resp) = @_;
2543
2544         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2545         # this fails if ballon driver is not loaded, so this must be
2546         # the last commnand (following command are aborted if this fails).
2547         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2548
2549         my $status = 'unknown';
2550         if (!defined($status = $resp->{'return'}->{status})) {
2551             warn "unable to get VM status\n";
2552             return;
2553         }
2554
2555         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2556     };
2557
2558     foreach my $vmid (keys %$list) {
2559         next if $opt_vmid && ($vmid ne $opt_vmid);
2560         next if !$res->{$vmid}->{pid}; # not running
2561         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2562     }
2563
2564     $qmpclient->queue_execute(undef, 1);
2565
2566     foreach my $vmid (keys %$list) {
2567         next if $opt_vmid && ($vmid ne $opt_vmid);
2568         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2569     }
2570
2571     return $res;
2572 }
2573
2574 sub foreach_dimm {
2575     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2576
2577     my $dimm_id = 0;
2578     my $current_size = 1024;
2579     my $dimm_size = 512;
2580     return if $current_size == $memory;
2581
2582     for (my $j = 0; $j < 8; $j++) {
2583         for (my $i = 0; $i < 32; $i++) {
2584             my $name = "dimm${dimm_id}";
2585             $dimm_id++;
2586             my $numanode = $i % $sockets;
2587             $current_size += $dimm_size;
2588             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2589             return  $current_size if $current_size >= $memory;
2590         }
2591         $dimm_size *= 2;
2592     }
2593 }
2594
2595 sub foreach_reverse_dimm {
2596     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2597
2598     my $dimm_id = 253;
2599     my $current_size = 4177920;
2600     my $dimm_size = 65536;
2601     return if $current_size == $memory;
2602
2603     for (my $j = 0; $j < 8; $j++) {
2604         for (my $i = 0; $i < 32; $i++) {
2605             my $name = "dimm${dimm_id}";
2606             $dimm_id--;
2607             my $numanode = $i % $sockets;
2608             $current_size -= $dimm_size;
2609             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2610             return  $current_size if $current_size <= $memory;
2611         }
2612         $dimm_size /= 2;
2613     }
2614 }
2615
2616 sub foreach_drive {
2617     my ($conf, $func) = @_;
2618
2619     foreach my $ds (valid_drive_names()) {
2620         next if !defined($conf->{$ds});
2621
2622         my $drive = parse_drive($ds, $conf->{$ds});
2623         next if !$drive;
2624
2625         &$func($ds, $drive);
2626     }
2627 }
2628
2629 sub foreach_volid {
2630     my ($conf, $func) = @_;
2631
2632     my $volhash = {};
2633
2634     my $test_volid = sub {
2635         my ($volid, $is_cdrom) = @_;
2636
2637         return if !$volid;
2638
2639         $volhash->{$volid} = $is_cdrom || 0;
2640     };
2641
2642     foreach_drive($conf, sub {
2643         my ($ds, $drive) = @_;
2644         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2645     });
2646
2647     foreach my $snapname (keys %{$conf->{snapshots}}) {
2648         my $snap = $conf->{snapshots}->{$snapname};
2649         &$test_volid($snap->{vmstate}, 0);
2650         foreach_drive($snap, sub {
2651             my ($ds, $drive) = @_;
2652             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2653         });
2654     }
2655
2656     foreach my $volid (keys %$volhash) {
2657         &$func($volid, $volhash->{$volid});
2658     }
2659 }
2660
2661 sub vga_conf_has_spice {
2662     my ($vga) = @_;
2663
2664     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2665
2666     return $1 || 1;
2667 }
2668
2669 sub config_to_command {
2670     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2671
2672     my $cmd = [];
2673     my $globalFlags = [];
2674     my $machineFlags = [];
2675     my $rtcFlags = [];
2676     my $cpuFlags = [];
2677     my $devices = [];
2678     my $pciaddr = '';
2679     my $bridges = {};
2680     my $kvmver = kvm_user_version();
2681     my $vernum = 0; # unknown
2682     my $ostype = $conf->{ostype};
2683     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2684         $vernum = $1*1000000+$2*1000;
2685     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2686         $vernum = $1*1000000+$2*1000+$3;
2687     }
2688
2689     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2690
2691     my $have_ovz = -f '/proc/vz/vestat';
2692
2693     my $q35 = machine_type_is_q35($conf);
2694     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2695     my $machine_type = $forcemachine || $conf->{machine};
2696     my $use_old_bios_files = undef;
2697     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2698
2699     my $cpuunits = defined($conf->{cpuunits}) ?
2700             $conf->{cpuunits} : $defaults->{cpuunits};
2701
2702     push @$cmd, '/usr/bin/systemd-run';
2703     push @$cmd, '--scope';
2704     push @$cmd, '--slice', "qemu";
2705     push @$cmd, '--unit', $vmid;
2706     # set KillMode=none, so that systemd don't kill those scopes
2707     # at shutdown (pve-manager service should stop the VMs instead)
2708     push @$cmd, '-p', "KillMode=none";
2709     push @$cmd, '-p', "CPUShares=$cpuunits";
2710     if ($conf->{cpulimit}) {
2711         my $cpulimit = int($conf->{cpulimit} * 100);
2712         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2713     }
2714
2715     push @$cmd, '/usr/bin/kvm';
2716
2717     push @$cmd, '-id', $vmid;
2718
2719     my $use_virtio = 0;
2720
2721     my $qmpsocket = qmp_socket($vmid);
2722     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2723     push @$cmd, '-mon', "chardev=qmp,mode=control";
2724
2725
2726     push @$cmd, '-pidfile' , pidfile_name($vmid);
2727
2728     push @$cmd, '-daemonize';
2729
2730     if ($conf->{smbios1}) {
2731         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2732     }
2733
2734     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2735         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2736         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2737         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2738         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2739         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2740         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2741     }
2742
2743     if ($q35) {
2744         # the q35 chipset support native usb2, so we enable usb controller
2745         # by default for this machine type
2746         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2747     } else {
2748         $pciaddr = print_pci_addr("piix3", $bridges);
2749         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2750
2751         my $use_usb2 = 0;
2752         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2753             next if !$conf->{"usb$i"};
2754             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2755             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2756             $use_usb2 = 1;
2757         }
2758         # include usb device config
2759         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2760     }
2761
2762     # add usb3 controller if needed
2763
2764     my $use_usb3 = 0;
2765     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2766         next if !$conf->{"usb$i"};
2767         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2768         next if !$d || !$d->{usb3};
2769         $use_usb3 = 1;
2770     }
2771
2772     $pciaddr = print_pci_addr("xhci", $bridges);
2773     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2774
2775     my $vga = $conf->{vga};
2776
2777     my $qxlnum = vga_conf_has_spice($vga);
2778     $vga = 'qxl' if $qxlnum;
2779
2780     if (!$vga) {
2781         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2782                                 $conf->{ostype} eq 'win7' ||
2783                                 $conf->{ostype} eq 'w2k8')) {
2784             $vga = 'std';
2785         } else {
2786             $vga = 'cirrus';
2787         }
2788     }
2789
2790     # enable absolute mouse coordinates (needed by vnc)
2791     my $tablet;
2792     if (defined($conf->{tablet})) {
2793         $tablet = $conf->{tablet};
2794     } else {
2795         $tablet = $defaults->{tablet};
2796         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2797         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2798     }
2799
2800     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2801
2802     my $kvm_off = 0;
2803     # host pci devices
2804     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2805         my $d = parse_hostpci($conf->{"hostpci$i"});
2806         next if !$d;
2807
2808         my $pcie = $d->{pcie};
2809         if($pcie){
2810             die "q35 machine model is not enabled" if !$q35;
2811             $pciaddr = print_pcie_addr("hostpci$i");
2812         }else{
2813             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2814         }
2815
2816         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2817         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2818         if ($xvga && $xvga ne '') {
2819             $kvm_off = 1;
2820             $vga = 'none';
2821             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2822                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2823             }
2824             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2825                 $xvga = "";
2826             }
2827         }
2828         my $pcidevices = $d->{pciid};
2829         my $multifunction = 1 if @$pcidevices > 1;
2830
2831         my $j=0;
2832         foreach my $pcidevice (@$pcidevices) {
2833
2834             my $id = "hostpci$i";
2835             $id .= ".$j" if $multifunction;
2836             my $addr = $pciaddr;
2837             $addr .= ".$j" if $multifunction;
2838             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2839
2840             if($j == 0){
2841                 $devicestr .= "$rombar$xvga";
2842                 $devicestr .= ",multifunction=on" if $multifunction;
2843             }
2844
2845             push @$devices, '-device', $devicestr;
2846             $j++;
2847         }
2848     }
2849
2850     # usb devices
2851     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2852         next if !$conf->{"usb$i"};
2853         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2854         next if !$d;
2855
2856         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2857         my $usbbus = '';
2858         if (defined($d->{usb3}) && $d->{usb3}) {
2859             $usbbus = ',bus=xhci.0';
2860         }
2861
2862         if (defined($d->{host})) {
2863             $d = parse_usb_device($d->{host});
2864             if (defined($d->{vendorid}) && defined($d->{productid})) {
2865                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2866             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2867                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2868             } elsif (defined($d->{spice}) && $d->{spice}) {
2869                 # usb redir support for spice, currently no usb3
2870                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2871                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2872             }
2873         }
2874     }
2875
2876     # serial devices
2877     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2878         if (my $path = $conf->{"serial$i"}) {
2879             if ($path eq 'socket') {
2880                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2881                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2882                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2883             } else {
2884                 die "no such serial device\n" if ! -c $path;
2885                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2886                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2887             }
2888         }
2889     }
2890
2891     # parallel devices
2892     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2893         if (my $path = $conf->{"parallel$i"}) {
2894             die "no such parallel device\n" if ! -c $path;
2895             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2896             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2897             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2898         }
2899     }
2900
2901     my $vmname = $conf->{name} || "vm$vmid";
2902
2903     push @$cmd, '-name', $vmname;
2904
2905     my $sockets = 1;
2906     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2907     $sockets = $conf->{sockets} if  $conf->{sockets};
2908
2909     my $cores = $conf->{cores} || 1;
2910
2911     my $maxcpus = $sockets * $cores;
2912
2913     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2914
2915     my $allowed_vcpus = $cpuinfo->{cpus};
2916
2917     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2918         if ($allowed_vcpus < $maxcpus);
2919
2920     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2921
2922     push @$cmd, '-nodefaults';
2923
2924     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2925
2926     my $bootindex_hash = {};
2927     my $i = 1;
2928     foreach my $o (split(//, $bootorder)) {
2929         $bootindex_hash->{$o} = $i*100;
2930         $i++;
2931     }
2932
2933     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2934
2935     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2936
2937     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2938
2939     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2940
2941     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2942         my $socket = vnc_socket($vmid);
2943         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2944     } else {
2945         push @$cmd, '-nographic';
2946     }
2947
2948     # time drift fix
2949     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2950
2951     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2952     my $useLocaltime = $conf->{localtime};
2953
2954     if ($ostype) {
2955         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2956
2957         if ($ostype =~ m/^w/) { # windows
2958             $useLocaltime = 1 if !defined($conf->{localtime});
2959
2960             # use time drift fix when acpi is enabled
2961             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2962                 $tdf = 1 if !defined($conf->{tdf});
2963             }
2964         }
2965
2966         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2967             $ostype eq 'wvista') {
2968             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2969             push @$cmd, '-no-hpet';
2970             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2971                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2972                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2973                 push @$cpuFlags , 'hv_time' if !$nokvm;
2974
2975             } else {
2976                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2977             }
2978         }
2979
2980         if ($ostype eq 'win7' || $ostype eq 'win8') {
2981             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2982         }
2983     }
2984
2985     push @$rtcFlags, 'driftfix=slew' if $tdf;
2986
2987     if ($nokvm) {
2988         push @$machineFlags, 'accel=tcg';
2989     } else {
2990         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2991     }
2992
2993     if ($machine_type) {
2994         push @$machineFlags, "type=${machine_type}";
2995     }
2996
2997     if ($conf->{startdate}) {
2998         push @$rtcFlags, "base=$conf->{startdate}";
2999     } elsif ($useLocaltime) {
3000         push @$rtcFlags, 'base=localtime';
3001     }
3002
3003     my $cpu = $nokvm ? "qemu64" : "kvm64";
3004     if (my $cputype = $conf->{cpu}) {
3005         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3006             or die "Cannot parse cpu description: $cputype\n";
3007         $cpu = $cpuconf->{cputype};
3008         $kvm_off = 1 if $cpuconf->{hidden};
3009     }
3010
3011     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3012
3013     push @$cpuFlags , '-x2apic'
3014         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3015
3016     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3017
3018     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3019
3020     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3021
3022         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3023         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3024     }
3025
3026     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3027
3028     push @$cpuFlags, 'kvm=off' if $kvm_off;
3029
3030     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3031
3032     push @$cmd, '-cpu', $cpu;
3033
3034     my $memory = $conf->{memory} || $defaults->{memory};
3035     my $static_memory = 0;
3036     my $dimm_memory = 0;
3037
3038     if ($hotplug_features->{memory}) {
3039         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3040         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3041         $static_memory = $STATICMEM;
3042         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3043         $dimm_memory = $memory - $static_memory;
3044         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3045
3046     } else {
3047
3048         $static_memory = $memory;
3049         push @$cmd, '-m', $static_memory;
3050     }
3051
3052     if ($conf->{numa}) {
3053
3054         my $numa_totalmemory = undef;
3055         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3056             next if !$conf->{"numa$i"};
3057             my $numa = parse_numa($conf->{"numa$i"});
3058             next if !$numa;
3059             # memory
3060             die "missing numa node$i memory value\n" if !$numa->{memory};
3061             my $numa_memory = $numa->{memory};
3062             $numa_totalmemory += $numa_memory;
3063             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3064
3065             # cpus
3066             my $cpus_start = $numa->{cpus}->{start};
3067             die "missing numa node$i cpus\n" if !defined($cpus_start);
3068             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3069             my $cpus = $cpus_start;
3070             if (defined($cpus_end)) {
3071                 $cpus .= "-$cpus_end";
3072                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3073             }
3074
3075             # hostnodes
3076             my $hostnodes_start = $numa->{hostnodes}->{start};
3077             if (defined($hostnodes_start)) {
3078                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3079                 my $hostnodes = $hostnodes_start;
3080                 if (defined($hostnodes_end)) {
3081                     $hostnodes .= "-$hostnodes_end";
3082                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3083                 }
3084
3085                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3086                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3087                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3088                 }
3089
3090                 # policy
3091                 my $policy = $numa->{policy};
3092                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3093                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3094             }
3095
3096             push @$cmd, '-object', $numa_object;
3097             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3098         }
3099
3100         die "total memory for NUMA nodes must be equal to vm static memory\n"
3101             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3102
3103         #if no custom tology, we split memory and cores across numa nodes
3104         if(!$numa_totalmemory) {
3105
3106             my $numa_memory = ($static_memory / $sockets) . "M";
3107
3108             for (my $i = 0; $i < $sockets; $i++)  {
3109
3110                 my $cpustart = ($cores * $i);
3111                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3112                 my $cpus = $cpustart;
3113                 $cpus .= "-$cpuend" if $cpuend;
3114
3115                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3116                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3117             }
3118         }
3119     }
3120
3121     if ($hotplug_features->{memory}) {
3122         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3123             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3124             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3125             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3126
3127             #if dimm_memory is not aligned to dimm map
3128             if($current_size > $memory) {
3129                  $conf->{memory} = $current_size;
3130                  PVE::QemuConfig->write_config($vmid, $conf);
3131             }
3132         });
3133     }
3134
3135     push @$cmd, '-S' if $conf->{freeze};
3136
3137     # set keyboard layout
3138     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3139     push @$cmd, '-k', $kb if $kb;
3140
3141     # enable sound
3142     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3143     #push @$cmd, '-soundhw', 'es1370';
3144     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3145
3146     if($conf->{agent}) {
3147         my $qgasocket = qmp_socket($vmid, 1);
3148         my $pciaddr = print_pci_addr("qga0", $bridges);
3149         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3150         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3151         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3152     }
3153
3154     my $spice_port;
3155
3156     if ($qxlnum) {
3157         if ($qxlnum > 1) {
3158             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3159                 for(my $i = 1; $i < $qxlnum; $i++){
3160                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3161                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3162                 }
3163             } else {
3164                 # assume other OS works like Linux
3165                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3166                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3167             }
3168         }
3169
3170         my $pciaddr = print_pci_addr("spice", $bridges);
3171
3172         my $nodename = PVE::INotify::nodename();
3173         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3174         $spice_port = PVE::Tools::next_spice_port($pfamily);
3175
3176         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3177
3178         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3179         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3180         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3181     }
3182
3183     # enable balloon by default, unless explicitly disabled
3184     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3185         $pciaddr = print_pci_addr("balloon0", $bridges);
3186         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3187     }
3188
3189     if ($conf->{watchdog}) {
3190         my $wdopts = parse_watchdog($conf->{watchdog});
3191         $pciaddr = print_pci_addr("watchdog", $bridges);
3192         my $watchdog = $wdopts->{model} || 'i6300esb';
3193         push @$devices, '-device', "$watchdog$pciaddr";
3194         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3195     }
3196
3197     my $vollist = [];
3198     my $scsicontroller = {};
3199     my $ahcicontroller = {};
3200     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3201
3202     # Add iscsi initiator name if available
3203     if (my $initiator = get_initiator_name()) {
3204         push @$devices, '-iscsi', "initiator-name=$initiator";
3205     }
3206
3207     foreach_drive($conf, sub {
3208         my ($ds, $drive) = @_;
3209
3210         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3211             push @$vollist, $drive->{file};
3212         }
3213
3214         $use_virtio = 1 if $ds =~ m/^virtio/;
3215
3216         if (drive_is_cdrom ($drive)) {
3217             if ($bootindex_hash->{d}) {
3218                 $drive->{bootindex} = $bootindex_hash->{d};
3219                 $bootindex_hash->{d} += 1;
3220             }
3221         } else {
3222             if ($bootindex_hash->{c}) {
3223                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3224                 $bootindex_hash->{c} += 1;
3225             }
3226         }
3227
3228         if($drive->{interface} eq 'virtio'){
3229            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3230         }
3231
3232         if ($drive->{interface} eq 'scsi') {
3233
3234             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3235
3236             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3237             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3238
3239             my $iothread = '';
3240             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3241                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3242                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3243             }
3244
3245             my $queues = '';
3246             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3247                 $queues = ",num_queues=$drive->{queues}";
3248             }
3249
3250             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3251             $scsicontroller->{$controller}=1;
3252         }
3253
3254         if ($drive->{interface} eq 'sata') {
3255            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3256            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3257            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3258            $ahcicontroller->{$controller}=1;
3259         }
3260
3261         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3262         push @$devices, '-drive',$drive_cmd;
3263         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3264     });
3265
3266     for (my $i = 0; $i < $MAX_NETS; $i++) {
3267          next if !$conf->{"net$i"};
3268          my $d = parse_net($conf->{"net$i"});
3269          next if !$d;
3270
3271          $use_virtio = 1 if $d->{model} eq 'virtio';
3272
3273          if ($bootindex_hash->{n}) {
3274             $d->{bootindex} = $bootindex_hash->{n};
3275             $bootindex_hash->{n} += 1;
3276          }
3277
3278          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3279          push @$devices, '-netdev', $netdevfull;
3280
3281          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3282          push @$devices, '-device', $netdevicefull;
3283     }
3284
3285     if (!$q35) {
3286         # add pci bridges
3287         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3288            $bridges->{1} = 1;
3289            $bridges->{2} = 1;
3290         }
3291
3292         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3293
3294         while (my ($k, $v) = each %$bridges) {
3295             $pciaddr = print_pci_addr("pci.$k");
3296             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3297         }
3298     }
3299
3300     # add custom args
3301     if ($conf->{args}) {
3302         my $aa = PVE::Tools::split_args($conf->{args});
3303         push @$cmd, @$aa;
3304     }
3305
3306     push @$cmd, @$devices;
3307     push @$cmd, '-rtc', join(',', @$rtcFlags)
3308         if scalar(@$rtcFlags);
3309     push @$cmd, '-machine', join(',', @$machineFlags)
3310         if scalar(@$machineFlags);
3311     push @$cmd, '-global', join(',', @$globalFlags)
3312         if scalar(@$globalFlags);
3313
3314     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3315 }
3316
3317 sub vnc_socket {
3318     my ($vmid) = @_;
3319     return "${var_run_tmpdir}/$vmid.vnc";
3320 }
3321
3322 sub spice_port {
3323     my ($vmid) = @_;
3324
3325     my $res = vm_mon_cmd($vmid, 'query-spice');
3326
3327     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3328 }
3329
3330 sub qmp_socket {
3331     my ($vmid, $qga) = @_;
3332     my $sockettype = $qga ? 'qga' : 'qmp';
3333     return "${var_run_tmpdir}/$vmid.$sockettype";
3334 }
3335
3336 sub pidfile_name {
3337     my ($vmid) = @_;
3338     return "${var_run_tmpdir}/$vmid.pid";
3339 }
3340
3341 sub vm_devices_list {
3342     my ($vmid) = @_;
3343
3344     my $res = vm_mon_cmd($vmid, 'query-pci');
3345     my $devices = {};
3346     foreach my $pcibus (@$res) {
3347         foreach my $device (@{$pcibus->{devices}}) {
3348             next if !$device->{'qdev_id'};
3349             if ($device->{'pci_bridge'}) {
3350                 $devices->{$device->{'qdev_id'}} = 1;
3351                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3352                     next if !$bridge_device->{'qdev_id'};
3353                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3354                     $devices->{$device->{'qdev_id'}}++;
3355                 }
3356             } else {
3357                 $devices->{$device->{'qdev_id'}} = 1;
3358             }
3359         }
3360     }
3361
3362     my $resblock = vm_mon_cmd($vmid, 'query-block');
3363     foreach my $block (@$resblock) {
3364         if($block->{device} =~ m/^drive-(\S+)/){
3365                 $devices->{$1} = 1;
3366         }
3367     }
3368
3369     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3370     foreach my $mice (@$resmice) {
3371         if ($mice->{name} eq 'QEMU HID Tablet') {
3372             $devices->{tablet} = 1;
3373             last;
3374         }
3375     }
3376
3377     return $devices;
3378 }
3379
3380 sub vm_deviceplug {
3381     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3382
3383     my $q35 = machine_type_is_q35($conf);
3384
3385     my $devices_list = vm_devices_list($vmid);
3386     return 1 if defined($devices_list->{$deviceid});
3387
3388     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3389
3390     if ($deviceid eq 'tablet') {
3391
3392         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3393
3394     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3395
3396         qemu_iothread_add($vmid, $deviceid, $device);
3397
3398         qemu_driveadd($storecfg, $vmid, $device);
3399         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3400
3401         qemu_deviceadd($vmid, $devicefull);
3402         eval { qemu_deviceaddverify($vmid, $deviceid); };
3403         if (my $err = $@) {
3404             eval { qemu_drivedel($vmid, $deviceid); };
3405             warn $@ if $@;
3406             die $err;
3407         }
3408
3409     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3410
3411
3412         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3413         my $pciaddr = print_pci_addr($deviceid);
3414         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3415
3416         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3417
3418         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3419             qemu_iothread_add($vmid, $deviceid, $device);
3420             $devicefull .= ",iothread=iothread-$deviceid";
3421         }
3422
3423         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3424             $devicefull .= ",num_queues=$device->{queues}";
3425         }
3426
3427         qemu_deviceadd($vmid, $devicefull);
3428         qemu_deviceaddverify($vmid, $deviceid);
3429
3430     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3431
3432         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3433         qemu_driveadd($storecfg, $vmid, $device);
3434
3435         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3436         eval { qemu_deviceadd($vmid, $devicefull); };
3437         if (my $err = $@) {
3438             eval { qemu_drivedel($vmid, $deviceid); };
3439             warn $@ if $@;
3440             die $err;
3441         }
3442
3443     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3444
3445         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3446
3447         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3448         my $use_old_bios_files = undef;
3449         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3450
3451         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3452         qemu_deviceadd($vmid, $netdevicefull);
3453         eval { qemu_deviceaddverify($vmid, $deviceid); };
3454         if (my $err = $@) {
3455             eval { qemu_netdevdel($vmid, $deviceid); };
3456             warn $@ if $@;
3457             die $err;
3458         }
3459
3460     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3461
3462         my $bridgeid = $2;
3463         my $pciaddr = print_pci_addr($deviceid);
3464         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3465
3466         qemu_deviceadd($vmid, $devicefull);
3467         qemu_deviceaddverify($vmid, $deviceid);
3468
3469     } else {
3470         die "can't hotplug device '$deviceid'\n";
3471     }
3472
3473     return 1;
3474 }
3475
3476 # fixme: this should raise exceptions on error!
3477 sub vm_deviceunplug {
3478     my ($vmid, $conf, $deviceid) = @_;
3479
3480     my $devices_list = vm_devices_list($vmid);
3481     return 1 if !defined($devices_list->{$deviceid});
3482
3483     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3484
3485     if ($deviceid eq 'tablet') {
3486
3487         qemu_devicedel($vmid, $deviceid);
3488
3489     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3490
3491         qemu_devicedel($vmid, $deviceid);
3492         qemu_devicedelverify($vmid, $deviceid);
3493         qemu_drivedel($vmid, $deviceid);
3494         qemu_iothread_del($conf, $vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3497
3498         qemu_devicedel($vmid, $deviceid);
3499         qemu_devicedelverify($vmid, $deviceid);
3500         qemu_iothread_del($conf, $vmid, $deviceid);
3501
3502     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3503
3504         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3505         my $device = parse_drive($deviceid, $conf->{$deviceid});
3506         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3507
3508         qemu_devicedel($vmid, $deviceid);
3509         qemu_drivedel($vmid, $deviceid);
3510         qemu_deletescsihw($conf, $vmid, $deviceid);
3511
3512     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3513
3514         qemu_devicedel($vmid, $deviceid);
3515         qemu_devicedelverify($vmid, $deviceid);
3516         qemu_netdevdel($vmid, $deviceid);
3517
3518     } else {
3519         die "can't unplug device '$deviceid'\n";
3520     }
3521
3522     return 1;
3523 }
3524
3525 sub qemu_deviceadd {
3526     my ($vmid, $devicefull) = @_;
3527
3528     $devicefull = "driver=".$devicefull;
3529     my %options =  split(/[=,]/, $devicefull);
3530
3531     vm_mon_cmd($vmid, "device_add" , %options);
3532 }
3533
3534 sub qemu_devicedel {
3535     my ($vmid, $deviceid) = @_;
3536
3537     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3538 }
3539
3540 sub qemu_iothread_add {
3541     my($vmid, $deviceid, $device) = @_;
3542
3543     if ($device->{iothread}) {
3544         my $iothreads = vm_iothreads_list($vmid);
3545         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3546     }
3547 }
3548
3549 sub qemu_iothread_del {
3550     my($conf, $vmid, $deviceid) = @_;
3551
3552     my $device = parse_drive($deviceid, $conf->{$deviceid});
3553     if ($device->{iothread}) {
3554         my $iothreads = vm_iothreads_list($vmid);
3555         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3556     }
3557 }
3558
3559 sub qemu_objectadd {
3560     my($vmid, $objectid, $qomtype) = @_;
3561
3562     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3563
3564     return 1;
3565 }
3566
3567 sub qemu_objectdel {
3568     my($vmid, $objectid) = @_;
3569
3570     vm_mon_cmd($vmid, "object-del", id => $objectid);
3571
3572     return 1;
3573 }
3574
3575 sub qemu_driveadd {
3576     my ($storecfg, $vmid, $device) = @_;
3577
3578     my $drive = print_drive_full($storecfg, $vmid, $device);
3579     $drive =~ s/\\/\\\\/g;
3580     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3581
3582     # If the command succeeds qemu prints: "OK"
3583     return 1 if $ret =~ m/OK/s;
3584
3585     die "adding drive failed: $ret\n";
3586 }
3587
3588 sub qemu_drivedel {
3589     my($vmid, $deviceid) = @_;
3590
3591     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3592     $ret =~ s/^\s+//;
3593
3594     return 1 if $ret eq "";
3595
3596     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3597     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3598
3599     die "deleting drive $deviceid failed : $ret\n";
3600 }
3601
3602 sub qemu_deviceaddverify {
3603     my ($vmid, $deviceid) = @_;
3604
3605     for (my $i = 0; $i <= 5; $i++) {
3606          my $devices_list = vm_devices_list($vmid);
3607          return 1 if defined($devices_list->{$deviceid});
3608          sleep 1;
3609     }
3610
3611     die "error on hotplug device '$deviceid'\n";
3612 }
3613
3614
3615 sub qemu_devicedelverify {
3616     my ($vmid, $deviceid) = @_;
3617
3618     # need to verify that the device is correctly removed as device_del
3619     # is async and empty return is not reliable
3620
3621     for (my $i = 0; $i <= 5; $i++) {
3622          my $devices_list = vm_devices_list($vmid);
3623          return 1 if !defined($devices_list->{$deviceid});
3624          sleep 1;
3625     }
3626
3627     die "error on hot-unplugging device '$deviceid'\n";
3628 }
3629
3630 sub qemu_findorcreatescsihw {
3631     my ($storecfg, $conf, $vmid, $device) = @_;
3632
3633     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3634
3635     my $scsihwid="$controller_prefix$controller";
3636     my $devices_list = vm_devices_list($vmid);
3637
3638     if(!defined($devices_list->{$scsihwid})) {
3639         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3640     }
3641
3642     return 1;
3643 }
3644
3645 sub qemu_deletescsihw {
3646     my ($conf, $vmid, $opt) = @_;
3647
3648     my $device = parse_drive($opt, $conf->{$opt});
3649
3650     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3651         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3652         return 1;
3653     }
3654
3655     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3656
3657     my $devices_list = vm_devices_list($vmid);
3658     foreach my $opt (keys %{$devices_list}) {
3659         if (PVE::QemuServer::is_valid_drivename($opt)) {
3660             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3661             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3662                 return 1;
3663             }
3664         }
3665     }
3666
3667     my $scsihwid="scsihw$controller";
3668
3669     vm_deviceunplug($vmid, $conf, $scsihwid);
3670
3671     return 1;
3672 }
3673
3674 sub qemu_add_pci_bridge {
3675     my ($storecfg, $conf, $vmid, $device) = @_;
3676
3677     my $bridges = {};
3678
3679     my $bridgeid;
3680
3681     print_pci_addr($device, $bridges);
3682
3683     while (my ($k, $v) = each %$bridges) {
3684         $bridgeid = $k;
3685     }
3686     return 1 if !defined($bridgeid) || $bridgeid < 1;
3687
3688     my $bridge = "pci.$bridgeid";
3689     my $devices_list = vm_devices_list($vmid);
3690
3691     if (!defined($devices_list->{$bridge})) {
3692         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3693     }
3694
3695     return 1;
3696 }
3697
3698 sub qemu_set_link_status {
3699     my ($vmid, $device, $up) = @_;
3700
3701     vm_mon_cmd($vmid, "set_link", name => $device,
3702                up => $up ? JSON::true : JSON::false);
3703 }
3704
3705 sub qemu_netdevadd {
3706     my ($vmid, $conf, $device, $deviceid) = @_;
3707
3708     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3709     my %options =  split(/[=,]/, $netdev);
3710
3711     vm_mon_cmd($vmid, "netdev_add",  %options);
3712     return 1;
3713 }
3714
3715 sub qemu_netdevdel {
3716     my ($vmid, $deviceid) = @_;
3717
3718     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3719 }
3720
3721 sub qemu_cpu_hotplug {
3722     my ($vmid, $conf, $vcpus) = @_;
3723
3724     my $sockets = 1;
3725     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3726     $sockets = $conf->{sockets} if  $conf->{sockets};
3727     my $cores = $conf->{cores} || 1;
3728     my $maxcpus = $sockets * $cores;
3729
3730     $vcpus = $maxcpus if !$vcpus;
3731
3732     die "you can't add more vcpus than maxcpus\n"
3733         if $vcpus > $maxcpus;
3734
3735     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3736     die "online cpu unplug is not yet possible\n"
3737         if $vcpus < $currentvcpus;
3738
3739     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3740     die "vcpus in running vm is different than configuration\n"
3741         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3742
3743     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3744         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3745     }
3746 }
3747
3748 sub qemu_memory_hotplug {
3749     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3750
3751     return $value if !check_running($vmid);
3752
3753     my $memory = $conf->{memory} || $defaults->{memory};
3754     $value = $defaults->{memory} if !$value;
3755     return $value if $value == $memory;
3756
3757     my $static_memory = $STATICMEM;
3758     my $dimm_memory = $memory - $static_memory;
3759
3760     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3761     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3762
3763
3764     my $sockets = 1;
3765     $sockets = $conf->{sockets} if $conf->{sockets};
3766
3767     if($value > $memory) {
3768
3769         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3770             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3771
3772                 return if $current_size <= $conf->{memory};
3773
3774                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3775                 if (my $err = $@) {
3776                     eval { qemu_objectdel($vmid, "mem-$name"); };
3777                     die $err;
3778                 }
3779
3780                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3781                 if (my $err = $@) {
3782                     eval { qemu_objectdel($vmid, "mem-$name"); };
3783                     die $err;
3784                 }
3785                 #update conf after each succesful module hotplug
3786                 $conf->{memory} = $current_size;
3787                 PVE::QemuConfig->write_config($vmid, $conf);
3788         });
3789
3790     } else {
3791
3792         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3793             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3794
3795                 return if $current_size >= $conf->{memory};
3796                 print "try to unplug memory dimm $name\n";
3797
3798                 my $retry = 0;
3799                 while (1) {
3800                     eval { qemu_devicedel($vmid, $name) };
3801                     sleep 3;
3802                     my $dimm_list = qemu_dimm_list($vmid);
3803                     last if !$dimm_list->{$name};
3804                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3805                     $retry++;
3806                 }
3807
3808                 #update conf after each succesful module unplug
3809                 $conf->{memory} = $current_size;
3810
3811                 eval { qemu_objectdel($vmid, "mem-$name"); };
3812                 PVE::QemuConfig->write_config($vmid, $conf);
3813         });
3814     }
3815 }
3816
3817 sub qemu_dimm_list {
3818     my ($vmid) = @_;
3819
3820     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3821     my $dimms = {};
3822
3823     foreach my $dimm (@$dimmarray) {
3824
3825         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3826         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3827         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3828         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3829         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3830     }
3831     return $dimms;
3832 }
3833
3834 sub qemu_block_set_io_throttle {
3835     my ($vmid, $deviceid,
3836         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3837         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3838
3839     return if !check_running($vmid) ;
3840
3841     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3842         bps => int($bps),
3843         bps_rd => int($bps_rd),
3844         bps_wr => int($bps_wr),
3845         iops => int($iops),
3846         iops_rd => int($iops_rd),
3847         iops_wr => int($iops_wr),
3848         bps_max => int($bps_max),
3849         bps_rd_max => int($bps_rd_max),
3850         bps_wr_max => int($bps_wr_max),
3851         iops_max => int($iops_max),
3852         iops_rd_max => int($iops_rd_max),
3853         iops_wr_max => int($iops_wr_max)
3854     );
3855
3856 }
3857
3858 # old code, only used to shutdown old VM after update
3859 sub __read_avail {
3860     my ($fh, $timeout) = @_;
3861
3862     my $sel = new IO::Select;
3863     $sel->add($fh);
3864
3865     my $res = '';
3866     my $buf;
3867
3868     my @ready;
3869     while (scalar (@ready = $sel->can_read($timeout))) {
3870         my $count;
3871         if ($count = $fh->sysread($buf, 8192)) {
3872             if ($buf =~ /^(.*)\(qemu\) $/s) {
3873                 $res .= $1;
3874                 last;
3875             } else {
3876                 $res .= $buf;
3877             }
3878         } else {
3879             if (!defined($count)) {
3880                 die "$!\n";
3881             }
3882             last;
3883         }
3884     }
3885
3886     die "monitor read timeout\n" if !scalar(@ready);
3887
3888     return $res;
3889 }
3890
3891 # old code, only used to shutdown old VM after update
3892 sub vm_monitor_command {
3893     my ($vmid, $cmdstr, $nocheck) = @_;
3894
3895     my $res;
3896
3897     eval {
3898         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3899
3900         my $sname = "${var_run_tmpdir}/$vmid.mon";
3901
3902         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3903             die "unable to connect to VM $vmid socket - $!\n";
3904
3905         my $timeout = 3;
3906
3907         # hack: migrate sometime blocks the monitor (when migrate_downtime
3908         # is set)
3909         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3910             $timeout = 60*60; # 1 hour
3911         }
3912
3913         # read banner;
3914         my $data = __read_avail($sock, $timeout);
3915
3916         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3917             die "got unexpected qemu monitor banner\n";
3918         }
3919
3920         my $sel = new IO::Select;
3921         $sel->add($sock);
3922
3923         if (!scalar(my @ready = $sel->can_write($timeout))) {
3924             die "monitor write error - timeout";
3925         }
3926
3927         my $fullcmd = "$cmdstr\r";
3928
3929         # syslog('info', "VM $vmid monitor command: $cmdstr");
3930
3931         my $b;
3932         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3933             die "monitor write error - $!";
3934         }
3935
3936         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3937
3938         $timeout = 20;
3939
3940         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3941             $timeout = 60*60; # 1 hour
3942         } elsif ($cmdstr =~ m/^(eject|change)/) {
3943             $timeout = 60; # note: cdrom mount command is slow
3944         }
3945         if ($res = __read_avail($sock, $timeout)) {
3946
3947             my @lines = split("\r?\n", $res);
3948
3949             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3950
3951             $res = join("\n", @lines);
3952             $res .= "\n";
3953         }
3954     };
3955
3956     my $err = $@;
3957
3958     if ($err) {
3959         syslog("err", "VM $vmid monitor command failed - $err");
3960         die $err;
3961     }
3962
3963     return $res;
3964 }
3965
3966 sub qemu_block_resize {
3967     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3968
3969     my $running = check_running($vmid);
3970
3971     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3972
3973     return if !$running;
3974
3975     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3976
3977 }
3978
3979 sub qemu_volume_snapshot {
3980     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3981
3982     my $running = check_running($vmid);
3983
3984     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3985         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3986     } else {
3987         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3988     }
3989 }
3990
3991 sub qemu_volume_snapshot_delete {
3992     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3993
3994     my $running = check_running($vmid);
3995
3996     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3997
3998     return if !$running;
3999
4000     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4001 }
4002
4003 sub set_migration_caps {
4004     my ($vmid) = @_;
4005
4006     my $cap_ref = [];
4007
4008     my $enabled_cap = {
4009         "auto-converge" => 1,
4010         "xbzrle" => 1,
4011         "x-rdma-pin-all" => 0,
4012         "zero-blocks" => 0,
4013         "compress" => 0
4014     };
4015
4016     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4017
4018     for my $supported_capability (@$supported_capabilities) {
4019         push @$cap_ref, {
4020             capability => $supported_capability->{capability},
4021             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4022         };
4023     }
4024
4025     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4026 }
4027
4028 my $fast_plug_option = {
4029     'lock' => 1,
4030     'name' => 1,
4031     'onboot' => 1,
4032     'shares' => 1,
4033     'startup' => 1,
4034     'description' => 1,
4035 };
4036
4037 # hotplug changes in [PENDING]
4038 # $selection hash can be used to only apply specified options, for
4039 # example: { cores => 1 } (only apply changed 'cores')
4040 # $errors ref is used to return error messages
4041 sub vmconfig_hotplug_pending {
4042     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4043
4044     my $defaults = load_defaults();
4045
4046     # commit values which do not have any impact on running VM first
4047     # Note: those option cannot raise errors, we we do not care about
4048     # $selection and always apply them.
4049
4050     my $add_error = sub {
4051         my ($opt, $msg) = @_;
4052         $errors->{$opt} = "hotplug problem - $msg";
4053     };
4054
4055     my $changes = 0;
4056     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4057         if ($fast_plug_option->{$opt}) {
4058             $conf->{$opt} = $conf->{pending}->{$opt};
4059             delete $conf->{pending}->{$opt};
4060             $changes = 1;
4061         }
4062     }
4063
4064     if ($changes) {
4065         PVE::QemuConfig->write_config($vmid, $conf);
4066         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4067     }
4068
4069     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4070
4071     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4072     while (my ($opt, $force) = each %$pending_delete_hash) {
4073         next if $selection && !$selection->{$opt};
4074         eval {
4075             if ($opt eq 'hotplug') {
4076                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4077             } elsif ($opt eq 'tablet') {
4078                 die "skip\n" if !$hotplug_features->{usb};
4079                 if ($defaults->{tablet}) {
4080                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4081                 } else {
4082                     vm_deviceunplug($vmid, $conf, $opt);
4083                 }
4084             } elsif ($opt eq 'vcpus') {
4085                 die "skip\n" if !$hotplug_features->{cpu};
4086                 qemu_cpu_hotplug($vmid, $conf, undef);
4087             } elsif ($opt eq 'balloon') {
4088                 # enable balloon device is not hotpluggable
4089                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4090             } elsif ($fast_plug_option->{$opt}) {
4091                 # do nothing
4092             } elsif ($opt =~ m/^net(\d+)$/) {
4093                 die "skip\n" if !$hotplug_features->{network};
4094                 vm_deviceunplug($vmid, $conf, $opt);
4095             } elsif (is_valid_drivename($opt)) {
4096                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4097                 vm_deviceunplug($vmid, $conf, $opt);
4098                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4099             } elsif ($opt =~ m/^memory$/) {
4100                 die "skip\n" if !$hotplug_features->{memory};
4101                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4102             } elsif ($opt eq 'cpuunits') {
4103                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4104             } elsif ($opt eq 'cpulimit') {
4105                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4106             } else {
4107                 die "skip\n";
4108             }
4109         };
4110         if (my $err = $@) {
4111             &$add_error($opt, $err) if $err ne "skip\n";
4112         } else {
4113             # save new config if hotplug was successful
4114             delete $conf->{$opt};
4115             vmconfig_undelete_pending_option($conf, $opt);
4116             PVE::QemuConfig->write_config($vmid, $conf);
4117             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4118         }
4119     }
4120
4121     foreach my $opt (keys %{$conf->{pending}}) {
4122         next if $selection && !$selection->{$opt};
4123         my $value = $conf->{pending}->{$opt};
4124         eval {
4125             if ($opt eq 'hotplug') {
4126                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4127             } elsif ($opt eq 'tablet') {
4128                 die "skip\n" if !$hotplug_features->{usb};
4129                 if ($value == 1) {
4130                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4131                 } elsif ($value == 0) {
4132                     vm_deviceunplug($vmid, $conf, $opt);
4133                 }
4134             } elsif ($opt eq 'vcpus') {
4135                 die "skip\n" if !$hotplug_features->{cpu};
4136                 qemu_cpu_hotplug($vmid, $conf, $value);
4137             } elsif ($opt eq 'balloon') {
4138                 # enable/disable balloning device is not hotpluggable
4139                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4140                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4141                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4142
4143                 # allow manual ballooning if shares is set to zero
4144                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4145                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4146                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4147                 }
4148             } elsif ($opt =~ m/^net(\d+)$/) {
4149                 # some changes can be done without hotplug
4150                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4151                                     $vmid, $opt, $value);
4152             } elsif (is_valid_drivename($opt)) {
4153                 # some changes can be done without hotplug
4154                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4155                                      $vmid, $opt, $value, 1);
4156             } elsif ($opt =~ m/^memory$/) { #dimms
4157                 die "skip\n" if !$hotplug_features->{memory};
4158                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4159             } elsif ($opt eq 'cpuunits') {
4160                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4161             } elsif ($opt eq 'cpulimit') {
4162                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4163                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4164             } else {
4165                 die "skip\n";  # skip non-hot-pluggable options
4166             }
4167         };
4168         if (my $err = $@) {
4169             &$add_error($opt, $err) if $err ne "skip\n";
4170         } else {
4171             # save new config if hotplug was successful
4172             $conf->{$opt} = $value;
4173             delete $conf->{pending}->{$opt};
4174             PVE::QemuConfig->write_config($vmid, $conf);
4175             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4176         }
4177     }
4178 }
4179
4180 sub try_deallocate_drive {
4181     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4182
4183     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4184         my $volid = $drive->{file};
4185         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4186             my $sid = PVE::Storage::parse_volume_id($volid);
4187             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4188
4189             # check if the disk is really unused
4190             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4191                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4192             PVE::Storage::vdisk_free($storecfg, $volid);
4193             return 1;
4194         } else {
4195             # If vm is not owner of this disk remove from config
4196             return 1;
4197         }
4198     }
4199
4200     return undef;
4201 }
4202
4203 sub vmconfig_delete_or_detach_drive {
4204     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4205
4206     my $drive = parse_drive($opt, $conf->{$opt});
4207
4208     my $rpcenv = PVE::RPCEnvironment::get();
4209     my $authuser = $rpcenv->get_user();
4210
4211     if ($force) {
4212         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4213         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4214     } else {
4215         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4216     }
4217 }
4218
4219 sub vmconfig_apply_pending {
4220     my ($vmid, $conf, $storecfg) = @_;
4221
4222     # cold plug
4223
4224     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4225     while (my ($opt, $force) = each %$pending_delete_hash) {
4226         die "internal error" if $opt =~ m/^unused/;
4227         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4228         if (!defined($conf->{$opt})) {
4229             vmconfig_undelete_pending_option($conf, $opt);
4230             PVE::QemuConfig->write_config($vmid, $conf);
4231         } elsif (is_valid_drivename($opt)) {
4232             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4233             vmconfig_undelete_pending_option($conf, $opt);
4234             delete $conf->{$opt};
4235             PVE::QemuConfig->write_config($vmid, $conf);
4236         } else {
4237             vmconfig_undelete_pending_option($conf, $opt);
4238             delete $conf->{$opt};
4239             PVE::QemuConfig->write_config($vmid, $conf);
4240         }
4241     }
4242
4243     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4244
4245     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4246         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4247
4248         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4249             # skip if nothing changed
4250         } elsif (is_valid_drivename($opt)) {
4251             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4252                 if defined($conf->{$opt});
4253             $conf->{$opt} = $conf->{pending}->{$opt};
4254         } else {
4255             $conf->{$opt} = $conf->{pending}->{$opt};
4256         }
4257
4258         delete $conf->{pending}->{$opt};
4259         PVE::QemuConfig->write_config($vmid, $conf);
4260     }
4261 }
4262
4263 my $safe_num_ne = sub {
4264     my ($a, $b) = @_;
4265
4266     return 0 if !defined($a) && !defined($b);
4267     return 1 if !defined($a);
4268     return 1 if !defined($b);
4269
4270     return $a != $b;
4271 };
4272
4273 my $safe_string_ne = sub {
4274     my ($a, $b) = @_;
4275
4276     return 0 if !defined($a) && !defined($b);
4277     return 1 if !defined($a);
4278     return 1 if !defined($b);
4279
4280     return $a ne $b;
4281 };
4282
4283 sub vmconfig_update_net {
4284     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4285
4286     my $newnet = parse_net($value);
4287
4288     if ($conf->{$opt}) {
4289         my $oldnet = parse_net($conf->{$opt});
4290
4291         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4292             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4293             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4294             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4295
4296             # for non online change, we try to hot-unplug
4297             die "skip\n" if !$hotplug;
4298             vm_deviceunplug($vmid, $conf, $opt);
4299         } else {
4300
4301             die "internal error" if $opt !~ m/net(\d+)/;
4302             my $iface = "tap${vmid}i$1";
4303
4304             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4305                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4306                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4307                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4308                 PVE::Network::tap_unplug($iface);
4309                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4310             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4311                 # Rate can be applied on its own but any change above needs to
4312                 # include the rate in tap_plug since OVS resets everything.
4313                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4314             }
4315
4316             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4317                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4318             }
4319
4320             return 1;
4321         }
4322     }
4323
4324     if ($hotplug) {
4325         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4326     } else {
4327         die "skip\n";
4328     }
4329 }
4330
4331 sub vmconfig_update_disk {
4332     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4333
4334     # fixme: do we need force?
4335
4336     my $drive = parse_drive($opt, $value);
4337
4338     if ($conf->{$opt}) {
4339
4340         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4341
4342             my $media = $drive->{media} || 'disk';
4343             my $oldmedia = $old_drive->{media} || 'disk';
4344             die "unable to change media type\n" if $media ne $oldmedia;
4345
4346             if (!drive_is_cdrom($old_drive)) {
4347
4348                 if ($drive->{file} ne $old_drive->{file}) {
4349
4350                     die "skip\n" if !$hotplug;
4351
4352                     # unplug and register as unused
4353                     vm_deviceunplug($vmid, $conf, $opt);
4354                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4355
4356                 } else {
4357                     # update existing disk
4358
4359                     # skip non hotpluggable value
4360                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4361                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4362                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4363                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4364                         die "skip\n";
4365                     }
4366
4367                     # apply throttle
4368                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4369                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4370                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4371                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4372                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4373                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4374                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4375                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4376                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4377                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4378                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4379                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4380
4381                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4382                                                    ($drive->{mbps} || 0)*1024*1024,
4383                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4384                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4385                                                    $drive->{iops} || 0,
4386                                                    $drive->{iops_rd} || 0,
4387                                                    $drive->{iops_wr} || 0,
4388                                                    ($drive->{mbps_max} || 0)*1024*1024,
4389                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4390                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4391                                                    $drive->{iops_max} || 0,
4392                                                    $drive->{iops_rd_max} || 0,
4393                                                    $drive->{iops_wr_max} || 0);
4394
4395                     }
4396
4397                     return 1;
4398                 }
4399
4400             } else { # cdrom
4401
4402                 if ($drive->{file} eq 'none') {
4403                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4404                 } else {
4405                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4406                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4407                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4408                 }
4409
4410                 return 1;
4411             }
4412         }
4413     }
4414
4415     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4416     # hotplug new disks
4417     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4418     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4419 }
4420
4421 sub vm_start {
4422     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4423         $forcemachine, $spice_ticket) = @_;
4424
4425     PVE::QemuConfig->lock_config($vmid, sub {
4426         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4427
4428         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4429
4430         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4431
4432         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4433
4434         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4435             vmconfig_apply_pending($vmid, $conf, $storecfg);
4436             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4437         }
4438
4439         my $defaults = load_defaults();
4440
4441         # set environment variable useful inside network script
4442         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4443
4444         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4445
4446         my $migrate_port = 0;
4447         my $migrate_uri;
4448         if ($statefile) {
4449             if ($statefile eq 'tcp') {
4450                 my $localip = "localhost";
4451                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4452                 my $nodename = PVE::INotify::nodename();
4453                 if ($datacenterconf->{migration_unsecure}) {
4454                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4455                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4456                 }
4457                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4458                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4459                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4460                 push @$cmd, '-incoming', $migrate_uri;
4461                 push @$cmd, '-S';
4462             } else {
4463                 push @$cmd, '-loadstate', $statefile;
4464             }
4465         } elsif ($paused) {
4466             push @$cmd, '-S';
4467         }
4468
4469         # host pci devices
4470         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4471           my $d = parse_hostpci($conf->{"hostpci$i"});
4472           next if !$d;
4473           my $pcidevices = $d->{pciid};
4474           foreach my $pcidevice (@$pcidevices) {
4475                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4476
4477                 my $info = pci_device_info("0000:$pciid");
4478                 die "IOMMU not present\n" if !check_iommu_support();
4479                 die "no pci device info for device '$pciid'\n" if !$info;
4480                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4481                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4482           }
4483         }
4484
4485         PVE::Storage::activate_volumes($storecfg, $vollist);
4486
4487         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4488                     umask => 0077); };
4489
4490         if (my $err = $@) {
4491             # deactivate volumes if start fails
4492             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4493             die "start failed: $err";
4494         }
4495
4496         print "migration listens on $migrate_uri\n" if $migrate_uri;
4497
4498         if ($statefile && $statefile ne 'tcp')  {
4499             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4500             warn $@ if $@;
4501         }
4502
4503         if ($migratedfrom) {
4504
4505             eval {
4506                 set_migration_caps($vmid);
4507             };
4508             warn $@ if $@;
4509
4510             if ($spice_port) {
4511                 print "spice listens on port $spice_port\n";
4512                 if ($spice_ticket) {
4513                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4514                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4515                 }
4516             }
4517
4518         } else {
4519
4520             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4521                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4522                     if $conf->{balloon};
4523             }
4524
4525             foreach my $opt (keys %$conf) {
4526                 next if $opt !~  m/^net\d+$/;
4527                 my $nicconf = parse_net($conf->{$opt});
4528                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4529             }
4530         }
4531
4532         vm_mon_cmd_nocheck($vmid, 'qom-set',
4533                     path => "machine/peripheral/balloon0",
4534                     property => "guest-stats-polling-interval",
4535                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4536
4537     });
4538 }
4539
4540 sub vm_mon_cmd {
4541     my ($vmid, $execute, %params) = @_;
4542
4543     my $cmd = { execute => $execute, arguments => \%params };
4544     vm_qmp_command($vmid, $cmd);
4545 }
4546
4547 sub vm_mon_cmd_nocheck {
4548     my ($vmid, $execute, %params) = @_;
4549
4550     my $cmd = { execute => $execute, arguments => \%params };
4551     vm_qmp_command($vmid, $cmd, 1);
4552 }
4553
4554 sub vm_qmp_command {
4555     my ($vmid, $cmd, $nocheck) = @_;
4556
4557     my $res;
4558
4559     my $timeout;
4560     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4561         $timeout = $cmd->{arguments}->{timeout};
4562         delete $cmd->{arguments}->{timeout};
4563     }
4564
4565     eval {
4566         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4567         my $sname = qmp_socket($vmid);
4568         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4569             my $qmpclient = PVE::QMPClient->new();
4570
4571             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4572         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4573             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4574                 if scalar(%{$cmd->{arguments}});
4575             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4576         } else {
4577             die "unable to open monitor socket\n";
4578         }
4579     };
4580     if (my $err = $@) {
4581         syslog("err", "VM $vmid qmp command failed - $err");
4582         die $err;
4583     }
4584
4585     return $res;
4586 }
4587
4588 sub vm_human_monitor_command {
4589     my ($vmid, $cmdline) = @_;
4590
4591     my $res;
4592
4593     my $cmd = {
4594         execute => 'human-monitor-command',
4595         arguments => { 'command-line' => $cmdline},
4596     };
4597
4598     return vm_qmp_command($vmid, $cmd);
4599 }
4600
4601 sub vm_commandline {
4602     my ($storecfg, $vmid) = @_;
4603
4604     my $conf = PVE::QemuConfig->load_config($vmid);
4605
4606     my $defaults = load_defaults();
4607
4608     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4609
4610     return join(' ', @$cmd);
4611 }
4612
4613 sub vm_reset {
4614     my ($vmid, $skiplock) = @_;
4615
4616     PVE::QemuConfig->lock_config($vmid, sub {
4617
4618         my $conf = PVE::QemuConfig->load_config($vmid);
4619
4620         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4621
4622         vm_mon_cmd($vmid, "system_reset");
4623     });
4624 }
4625
4626 sub get_vm_volumes {
4627     my ($conf) = @_;
4628
4629     my $vollist = [];
4630     foreach_volid($conf, sub {
4631         my ($volid, $is_cdrom) = @_;
4632
4633         return if $volid =~ m|^/|;
4634
4635         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4636         return if !$sid;
4637
4638         push @$vollist, $volid;
4639     });
4640
4641     return $vollist;
4642 }
4643
4644 sub vm_stop_cleanup {
4645     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4646
4647     eval {
4648
4649         if (!$keepActive) {
4650             my $vollist = get_vm_volumes($conf);
4651             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4652         }
4653
4654         foreach my $ext (qw(mon qmp pid vnc qga)) {
4655             unlink "/var/run/qemu-server/${vmid}.$ext";
4656         }
4657
4658         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4659     };
4660     warn $@ if $@; # avoid errors - just warn
4661 }
4662
4663 # Note: use $nockeck to skip tests if VM configuration file exists.
4664 # We need that when migration VMs to other nodes (files already moved)
4665 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4666 sub vm_stop {
4667     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4668
4669     $force = 1 if !defined($force) && !$shutdown;
4670
4671     if ($migratedfrom){
4672         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4673         kill 15, $pid if $pid;
4674         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4675         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4676         return;
4677     }
4678
4679     PVE::QemuConfig->lock_config($vmid, sub {
4680
4681         my $pid = check_running($vmid, $nocheck);
4682         return if !$pid;
4683
4684         my $conf;
4685         if (!$nocheck) {
4686             $conf = PVE::QemuConfig->load_config($vmid);
4687             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4688             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4689                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4690                 $timeout = $opts->{down} if $opts->{down};
4691             }
4692         }
4693
4694         $timeout = 60 if !defined($timeout);
4695
4696         eval {
4697             if ($shutdown) {
4698                 if (defined($conf) && $conf->{agent}) {
4699                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4700                 } else {
4701                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4702                 }
4703             } else {
4704                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4705             }
4706         };
4707         my $err = $@;
4708
4709         if (!$err) {
4710             my $count = 0;
4711             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4712                 $count++;
4713                 sleep 1;
4714             }
4715
4716             if ($count >= $timeout) {
4717                 if ($force) {
4718                     warn "VM still running - terminating now with SIGTERM\n";
4719                     kill 15, $pid;
4720                 } else {
4721                     die "VM quit/powerdown failed - got timeout\n";
4722                 }
4723             } else {
4724                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4725                 return;
4726             }
4727         } else {
4728             if ($force) {
4729                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4730                 kill 15, $pid;
4731             } else {
4732                 die "VM quit/powerdown failed\n";
4733             }
4734         }
4735
4736         # wait again
4737         $timeout = 10;
4738
4739         my $count = 0;
4740         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4741             $count++;
4742             sleep 1;
4743         }
4744
4745         if ($count >= $timeout) {
4746             warn "VM still running - terminating now with SIGKILL\n";
4747             kill 9, $pid;
4748             sleep 1;
4749         }
4750
4751         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4752    });
4753 }
4754
4755 sub vm_suspend {
4756     my ($vmid, $skiplock) = @_;
4757
4758     PVE::QemuConfig->lock_config($vmid, sub {
4759
4760         my $conf = PVE::QemuConfig->load_config($vmid);
4761
4762         PVE::QemuConfig->check_lock($conf)
4763             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4764
4765         vm_mon_cmd($vmid, "stop");
4766     });
4767 }
4768
4769 sub vm_resume {
4770     my ($vmid, $skiplock, $nocheck) = @_;
4771
4772     PVE::QemuConfig->lock_config($vmid, sub {
4773
4774         if (!$nocheck) {
4775
4776             my $conf = PVE::QemuConfig->load_config($vmid);
4777
4778             PVE::QemuConfig->check_lock($conf)
4779                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4780
4781             vm_mon_cmd($vmid, "cont");
4782
4783         } else {
4784             vm_mon_cmd_nocheck($vmid, "cont");
4785         }
4786     });
4787 }
4788
4789 sub vm_sendkey {
4790     my ($vmid, $skiplock, $key) = @_;
4791
4792     PVE::QemuConfig->lock_config($vmid, sub {
4793
4794         my $conf = PVE::QemuConfig->load_config($vmid);
4795
4796         # there is no qmp command, so we use the human monitor command
4797         vm_human_monitor_command($vmid, "sendkey $key");
4798     });
4799 }
4800
4801 sub vm_destroy {
4802     my ($storecfg, $vmid, $skiplock) = @_;
4803
4804     PVE::QemuConfig->lock_config($vmid, sub {
4805
4806         my $conf = PVE::QemuConfig->load_config($vmid);
4807
4808         if (!check_running($vmid)) {
4809             destroy_vm($storecfg, $vmid, undef, $skiplock);
4810         } else {
4811             die "VM $vmid is running - destroy failed\n";
4812         }
4813     });
4814 }
4815
4816 # pci helpers
4817
4818 sub file_write {
4819     my ($filename, $buf) = @_;
4820
4821     my $fh = IO::File->new($filename, "w");
4822     return undef if !$fh;
4823
4824     my $res = print $fh $buf;
4825
4826     $fh->close();
4827
4828     return $res;
4829 }
4830
4831 sub pci_device_info {
4832     my ($name) = @_;
4833
4834     my $res;
4835
4836     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4837     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4838
4839     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4840     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4841
4842     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4843     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4844
4845     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4846     return undef if !defined($product) || $product !~ s/^0x//;
4847
4848     $res = {
4849         name => $name,
4850         vendor => $vendor,
4851         product => $product,
4852         domain => $domain,
4853         bus => $bus,
4854         slot => $slot,
4855         func => $func,
4856         irq => $irq,
4857         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4858     };
4859
4860     return $res;
4861 }
4862
4863 sub pci_dev_reset {
4864     my ($dev) = @_;
4865
4866     my $name = $dev->{name};
4867
4868     my $fn = "$pcisysfs/devices/$name/reset";
4869
4870     return file_write($fn, "1");
4871 }
4872
4873 sub pci_dev_bind_to_vfio {
4874     my ($dev) = @_;
4875
4876     my $name = $dev->{name};
4877
4878     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4879
4880     if (!-d $vfio_basedir) {
4881         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4882     }
4883     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4884
4885     my $testdir = "$vfio_basedir/$name";
4886     return 1 if -d $testdir;
4887
4888     my $data = "$dev->{vendor} $dev->{product}";
4889     return undef if !file_write("$vfio_basedir/new_id", $data);
4890
4891     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4892     if (!file_write($fn, $name)) {
4893         return undef if -f $fn;
4894     }
4895
4896     $fn = "$vfio_basedir/bind";
4897     if (! -d $testdir) {
4898         return undef if !file_write($fn, $name);
4899     }
4900
4901     return -d $testdir;
4902 }
4903
4904 sub pci_dev_group_bind_to_vfio {
4905     my ($pciid) = @_;
4906
4907     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4908
4909     if (!-d $vfio_basedir) {
4910         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4911     }
4912     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4913
4914     # get IOMMU group devices
4915     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4916       my @devs = grep /^0000:/, readdir($D);
4917     closedir($D);
4918
4919     foreach my $pciid (@devs) {
4920         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4921
4922         # pci bridges, switches or root ports are not supported
4923         # they have a pci_bus subdirectory so skip them
4924         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4925
4926         my $info = pci_device_info($1);
4927         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4928     }
4929
4930     return 1;
4931 }
4932
4933 sub print_pci_addr {
4934     my ($id, $bridges) = @_;
4935
4936     my $res = '';
4937     my $devices = {
4938         piix3 => { bus => 0, addr => 1 },
4939         #addr2 : first videocard
4940         balloon0 => { bus => 0, addr => 3 },
4941         watchdog => { bus => 0, addr => 4 },
4942         scsihw0 => { bus => 0, addr => 5 },
4943         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4944         scsihw1 => { bus => 0, addr => 6 },
4945         ahci0 => { bus => 0, addr => 7 },
4946         qga0 => { bus => 0, addr => 8 },
4947         spice => { bus => 0, addr => 9 },
4948         virtio0 => { bus => 0, addr => 10 },
4949         virtio1 => { bus => 0, addr => 11 },
4950         virtio2 => { bus => 0, addr => 12 },
4951         virtio3 => { bus => 0, addr => 13 },
4952         virtio4 => { bus => 0, addr => 14 },
4953         virtio5 => { bus => 0, addr => 15 },
4954         hostpci0 => { bus => 0, addr => 16 },
4955         hostpci1 => { bus => 0, addr => 17 },
4956         net0 => { bus => 0, addr => 18 },
4957         net1 => { bus => 0, addr => 19 },
4958         net2 => { bus => 0, addr => 20 },
4959         net3 => { bus => 0, addr => 21 },
4960         net4 => { bus => 0, addr => 22 },
4961         net5 => { bus => 0, addr => 23 },
4962         vga1 => { bus => 0, addr => 24 },
4963         vga2 => { bus => 0, addr => 25 },
4964         vga3 => { bus => 0, addr => 26 },
4965         hostpci2 => { bus => 0, addr => 27 },
4966         hostpci3 => { bus => 0, addr => 28 },
4967         #addr29 : usb-host (pve-usb.cfg)
4968         'pci.1' => { bus => 0, addr => 30 },
4969         'pci.2' => { bus => 0, addr => 31 },
4970         'net6' => { bus => 1, addr => 1 },
4971         'net7' => { bus => 1, addr => 2 },
4972         'net8' => { bus => 1, addr => 3 },
4973         'net9' => { bus => 1, addr => 4 },
4974         'net10' => { bus => 1, addr => 5 },
4975         'net11' => { bus => 1, addr => 6 },
4976         'net12' => { bus => 1, addr => 7 },
4977         'net13' => { bus => 1, addr => 8 },
4978         'net14' => { bus => 1, addr => 9 },
4979         'net15' => { bus => 1, addr => 10 },
4980         'net16' => { bus => 1, addr => 11 },
4981         'net17' => { bus => 1, addr => 12 },
4982         'net18' => { bus => 1, addr => 13 },
4983         'net19' => { bus => 1, addr => 14 },
4984         'net20' => { bus => 1, addr => 15 },
4985         'net21' => { bus => 1, addr => 16 },
4986         'net22' => { bus => 1, addr => 17 },
4987         'net23' => { bus => 1, addr => 18 },
4988         'net24' => { bus => 1, addr => 19 },
4989         'net25' => { bus => 1, addr => 20 },
4990         'net26' => { bus => 1, addr => 21 },
4991         'net27' => { bus => 1, addr => 22 },
4992         'net28' => { bus => 1, addr => 23 },
4993         'net29' => { bus => 1, addr => 24 },
4994         'net30' => { bus => 1, addr => 25 },
4995         'net31' => { bus => 1, addr => 26 },
4996         'xhci' => { bus => 1, addr => 27 },
4997         'virtio6' => { bus => 2, addr => 1 },
4998         'virtio7' => { bus => 2, addr => 2 },
4999         'virtio8' => { bus => 2, addr => 3 },
5000         'virtio9' => { bus => 2, addr => 4 },
5001         'virtio10' => { bus => 2, addr => 5 },
5002         'virtio11' => { bus => 2, addr => 6 },
5003         'virtio12' => { bus => 2, addr => 7 },
5004         'virtio13' => { bus => 2, addr => 8 },
5005         'virtio14' => { bus => 2, addr => 9 },
5006         'virtio15' => { bus => 2, addr => 10 },
5007         'virtioscsi0' => { bus => 3, addr => 1 },
5008         'virtioscsi1' => { bus => 3, addr => 2 },
5009         'virtioscsi2' => { bus => 3, addr => 3 },
5010         'virtioscsi3' => { bus => 3, addr => 4 },
5011         'virtioscsi4' => { bus => 3, addr => 5 },
5012         'virtioscsi5' => { bus => 3, addr => 6 },
5013         'virtioscsi6' => { bus => 3, addr => 7 },
5014         'virtioscsi7' => { bus => 3, addr => 8 },
5015         'virtioscsi8' => { bus => 3, addr => 9 },
5016         'virtioscsi9' => { bus => 3, addr => 10 },
5017         'virtioscsi10' => { bus => 3, addr => 11 },
5018         'virtioscsi11' => { bus => 3, addr => 12 },
5019         'virtioscsi12' => { bus => 3, addr => 13 },
5020         'virtioscsi13' => { bus => 3, addr => 14 },
5021         'virtioscsi14' => { bus => 3, addr => 15 },
5022         'virtioscsi15' => { bus => 3, addr => 16 },
5023         'virtioscsi16' => { bus => 3, addr => 17 },
5024         'virtioscsi17' => { bus => 3, addr => 18 },
5025         'virtioscsi18' => { bus => 3, addr => 19 },
5026         'virtioscsi19' => { bus => 3, addr => 20 },
5027         'virtioscsi20' => { bus => 3, addr => 21 },
5028         'virtioscsi21' => { bus => 3, addr => 22 },
5029         'virtioscsi22' => { bus => 3, addr => 23 },
5030         'virtioscsi23' => { bus => 3, addr => 24 },
5031         'virtioscsi24' => { bus => 3, addr => 25 },
5032         'virtioscsi25' => { bus => 3, addr => 26 },
5033         'virtioscsi26' => { bus => 3, addr => 27 },
5034         'virtioscsi27' => { bus => 3, addr => 28 },
5035         'virtioscsi28' => { bus => 3, addr => 29 },
5036         'virtioscsi29' => { bus => 3, addr => 30 },
5037         'virtioscsi30' => { bus => 3, addr => 31 },
5038
5039     };
5040
5041     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5042            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5043            my $bus = $devices->{$id}->{bus};
5044            $res = ",bus=pci.$bus,addr=$addr";
5045            $bridges->{$bus} = 1 if $bridges;
5046     }
5047     return $res;
5048
5049 }
5050
5051 sub print_pcie_addr {
5052     my ($id) = @_;
5053
5054     my $res = '';
5055     my $devices = {
5056         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5057         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5058         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5059         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5060     };
5061
5062     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5063            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5064            my $bus = $devices->{$id}->{bus};
5065            $res = ",bus=$bus,addr=$addr";
5066     }
5067     return $res;
5068
5069 }
5070
5071 # vzdump restore implementaion
5072
5073 sub tar_archive_read_firstfile {
5074     my $archive = shift;
5075
5076     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5077
5078     # try to detect archive type first
5079     my $pid = open (TMP, "tar tf '$archive'|") ||
5080         die "unable to open file '$archive'\n";
5081     my $firstfile = <TMP>;
5082     kill 15, $pid;
5083     close TMP;
5084
5085     die "ERROR: archive contaions no data\n" if !$firstfile;
5086     chomp $firstfile;
5087
5088     return $firstfile;
5089 }
5090
5091 sub tar_restore_cleanup {
5092     my ($storecfg, $statfile) = @_;
5093
5094     print STDERR "starting cleanup\n";
5095
5096     if (my $fd = IO::File->new($statfile, "r")) {
5097         while (defined(my $line = <$fd>)) {
5098             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5099                 my $volid = $2;
5100                 eval {
5101                     if ($volid =~ m|^/|) {
5102                         unlink $volid || die 'unlink failed\n';
5103                     } else {
5104                         PVE::Storage::vdisk_free($storecfg, $volid);
5105                     }
5106                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5107                 };
5108                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5109             } else {
5110                 print STDERR "unable to parse line in statfile - $line";
5111             }
5112         }
5113         $fd->close();
5114     }
5115 }
5116
5117 sub restore_archive {
5118     my ($archive, $vmid, $user, $opts) = @_;
5119
5120     my $format = $opts->{format};
5121     my $comp;
5122
5123     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5124         $format = 'tar' if !$format;
5125         $comp = 'gzip';
5126     } elsif ($archive =~ m/\.tar$/) {
5127         $format = 'tar' if !$format;
5128     } elsif ($archive =~ m/.tar.lzo$/) {
5129         $format = 'tar' if !$format;
5130         $comp = 'lzop';
5131     } elsif ($archive =~ m/\.vma$/) {
5132         $format = 'vma' if !$format;
5133     } elsif ($archive =~ m/\.vma\.gz$/) {
5134         $format = 'vma' if !$format;
5135         $comp = 'gzip';
5136     } elsif ($archive =~ m/\.vma\.lzo$/) {
5137         $format = 'vma' if !$format;
5138         $comp = 'lzop';
5139     } else {
5140         $format = 'vma' if !$format; # default
5141     }
5142
5143     # try to detect archive format
5144     if ($format eq 'tar') {
5145         return restore_tar_archive($archive, $vmid, $user, $opts);
5146     } else {
5147         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5148     }
5149 }
5150
5151 sub restore_update_config_line {
5152     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5153
5154     return if $line =~ m/^\#qmdump\#/;
5155     return if $line =~ m/^\#vzdump\#/;
5156     return if $line =~ m/^lock:/;
5157     return if $line =~ m/^unused\d+:/;
5158     return if $line =~ m/^parent:/;
5159     return if $line =~ m/^template:/; # restored VM is never a template
5160
5161     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5162         # try to convert old 1.X settings
5163         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5164         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5165             my ($model, $macaddr) = split(/\=/, $devconfig);
5166             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5167             my $net = {
5168                 model => $model,
5169                 bridge => "vmbr$ind",
5170                 macaddr => $macaddr,
5171             };
5172             my $netstr = print_net($net);
5173
5174             print $outfd "net$cookie->{netcount}: $netstr\n";
5175             $cookie->{netcount}++;
5176         }
5177     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5178         my ($id, $netstr) = ($1, $2);
5179         my $net = parse_net($netstr);
5180         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5181         $netstr = print_net($net);
5182         print $outfd "$id: $netstr\n";
5183     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5184         my $virtdev = $1;
5185         my $value = $3;
5186         my $di = parse_drive($virtdev, $value);
5187         if (defined($di->{backup}) && !$di->{backup}) {
5188             print $outfd "#$line";
5189         } elsif ($map->{$virtdev}) {
5190             delete $di->{format}; # format can change on restore
5191             $di->{file} = $map->{$virtdev};
5192             $value = print_drive($vmid, $di);
5193             print $outfd "$virtdev: $value\n";
5194         } else {
5195             print $outfd $line;
5196         }
5197     } else {
5198         print $outfd $line;
5199     }
5200 }
5201
5202 sub scan_volids {
5203     my ($cfg, $vmid) = @_;
5204
5205     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5206
5207     my $volid_hash = {};
5208     foreach my $storeid (keys %$info) {
5209         foreach my $item (@{$info->{$storeid}}) {
5210             next if !($item->{volid} && $item->{size});
5211             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5212             $volid_hash->{$item->{volid}} = $item;
5213         }
5214     }
5215
5216     return $volid_hash;
5217 }
5218
5219 sub is_volume_in_use {
5220     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5221
5222     my $path = PVE::Storage::path($storecfg, $volid);
5223
5224     my $scan_config = sub {
5225         my ($cref, $snapname) = @_;
5226
5227         foreach my $key (keys %$cref) {
5228             my $value = $cref->{$key};
5229             if (is_valid_drivename($key)) {
5230                 next if $skip_drive && $key eq $skip_drive;
5231                 my $drive = parse_drive($key, $value);
5232                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5233                 return 1 if $volid eq $drive->{file};
5234                 if ($drive->{file} =~ m!^/!) {
5235                     return 1 if $drive->{file} eq $path;
5236                 } else {
5237                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5238                     next if !$storeid;
5239                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5240                     next if !$scfg;
5241                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5242                 }
5243             }
5244         }
5245
5246         return 0;
5247     };
5248
5249     return 1 if &$scan_config($conf);
5250
5251     undef $skip_drive;
5252
5253     foreach my $snapname (keys %{$conf->{snapshots}}) {
5254         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5255     }
5256
5257     return 0;
5258 }
5259
5260 sub update_disksize {
5261     my ($vmid, $conf, $volid_hash) = @_;
5262
5263     my $changes;
5264
5265     my $used = {};
5266
5267     # Note: it is allowed to define multiple storages with same path (alias), so
5268     # we need to check both 'volid' and real 'path' (two different volid can point
5269     # to the same path).
5270
5271     my $usedpath = {};
5272
5273     # update size info
5274     foreach my $opt (keys %$conf) {
5275         if (is_valid_drivename($opt)) {
5276             my $drive = parse_drive($opt, $conf->{$opt});
5277             my $volid = $drive->{file};
5278             next if !$volid;
5279
5280             $used->{$volid} = 1;
5281             if ($volid_hash->{$volid} &&
5282                 (my $path = $volid_hash->{$volid}->{path})) {
5283                 $usedpath->{$path} = 1;
5284             }
5285
5286             next if drive_is_cdrom($drive);
5287             next if !$volid_hash->{$volid};
5288
5289             $drive->{size} = $volid_hash->{$volid}->{size};
5290             my $new = print_drive($vmid, $drive);
5291             if ($new ne $conf->{$opt}) {
5292                 $changes = 1;
5293                 $conf->{$opt} = $new;
5294             }
5295         }
5296     }
5297
5298     # remove 'unusedX' entry if volume is used
5299     foreach my $opt (keys %$conf) {
5300         next if $opt !~ m/^unused\d+$/;
5301         my $volid = $conf->{$opt};
5302         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5303         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5304             $changes = 1;
5305             delete $conf->{$opt};
5306         }
5307     }
5308
5309     foreach my $volid (sort keys %$volid_hash) {
5310         next if $volid =~ m/vm-$vmid-state-/;
5311         next if $used->{$volid};
5312         my $path = $volid_hash->{$volid}->{path};
5313         next if !$path; # just to be sure
5314         next if $usedpath->{$path};
5315         $changes = 1;
5316         PVE::QemuConfig->add_unused_volume($conf, $volid);
5317         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5318     }
5319
5320     return $changes;
5321 }
5322
5323 sub rescan {
5324     my ($vmid, $nolock) = @_;
5325
5326     my $cfg = PVE::Storage::config();
5327
5328     my $volid_hash = scan_volids($cfg, $vmid);
5329
5330     my $updatefn =  sub {
5331         my ($vmid) = @_;
5332
5333         my $conf = PVE::QemuConfig->load_config($vmid);
5334
5335         PVE::QemuConfig->check_lock($conf);
5336
5337         my $vm_volids = {};
5338         foreach my $volid (keys %$volid_hash) {
5339             my $info = $volid_hash->{$volid};
5340             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5341         }
5342
5343         my $changes = update_disksize($vmid, $conf, $vm_volids);
5344
5345         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5346     };
5347
5348     if (defined($vmid)) {
5349         if ($nolock) {
5350             &$updatefn($vmid);
5351         } else {
5352             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5353         }
5354     } else {
5355         my $vmlist = config_list();
5356         foreach my $vmid (keys %$vmlist) {
5357             if ($nolock) {
5358                 &$updatefn($vmid);
5359             } else {
5360                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5361             }
5362         }
5363     }
5364 }
5365
5366 sub restore_vma_archive {
5367     my ($archive, $vmid, $user, $opts, $comp) = @_;
5368
5369     my $input = $archive eq '-' ? "<&STDIN" : undef;
5370     my $readfrom = $archive;
5371
5372     my $uncomp = '';
5373     if ($comp) {
5374         $readfrom = '-';
5375         my $qarchive = PVE::Tools::shellquote($archive);
5376         if ($comp eq 'gzip') {
5377             $uncomp = "zcat $qarchive|";
5378         } elsif ($comp eq 'lzop') {
5379             $uncomp = "lzop -d -c $qarchive|";
5380         } else {
5381             die "unknown compression method '$comp'\n";
5382         }
5383
5384     }
5385
5386     my $tmpdir = "/var/tmp/vzdumptmp$$";
5387     rmtree $tmpdir;
5388
5389     # disable interrupts (always do cleanups)
5390     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5391         warn "got interrupt - ignored\n";
5392     };
5393
5394     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5395     POSIX::mkfifo($mapfifo, 0600);
5396     my $fifofh;
5397
5398     my $openfifo = sub {
5399         open($fifofh, '>', $mapfifo) || die $!;
5400     };
5401
5402     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5403
5404     my $oldtimeout;
5405     my $timeout = 5;
5406
5407     my $devinfo = {};
5408
5409     my $rpcenv = PVE::RPCEnvironment::get();
5410
5411     my $conffile = PVE::QemuConfig->config_file($vmid);
5412     my $tmpfn = "$conffile.$$.tmp";
5413
5414     # Note: $oldconf is undef if VM does not exists
5415     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5416     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5417
5418     my $print_devmap = sub {
5419         my $virtdev_hash = {};
5420
5421         my $cfgfn = "$tmpdir/qemu-server.conf";
5422
5423         # we can read the config - that is already extracted
5424         my $fh = IO::File->new($cfgfn, "r") ||
5425             "unable to read qemu-server.conf - $!\n";
5426
5427         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5428         if (-f $fwcfgfn) {
5429             my $pve_firewall_dir = '/etc/pve/firewall';
5430             mkdir $pve_firewall_dir; # make sure the dir exists
5431             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5432         }
5433
5434         while (defined(my $line = <$fh>)) {
5435             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5436                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5437                 die "archive does not contain data for drive '$virtdev'\n"
5438                     if !$devinfo->{$devname};
5439                 if (defined($opts->{storage})) {
5440                     $storeid = $opts->{storage} || 'local';
5441                 } elsif (!$storeid) {
5442                     $storeid = 'local';
5443                 }
5444                 $format = 'raw' if !$format;
5445                 $devinfo->{$devname}->{devname} = $devname;
5446                 $devinfo->{$devname}->{virtdev} = $virtdev;
5447                 $devinfo->{$devname}->{format} = $format;
5448                 $devinfo->{$devname}->{storeid} = $storeid;
5449
5450                 # check permission on storage
5451                 my $pool = $opts->{pool}; # todo: do we need that?
5452                 if ($user ne 'root@pam') {
5453                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5454                 }
5455
5456                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5457             }
5458         }
5459
5460         foreach my $devname (keys %$devinfo) {
5461             die "found no device mapping information for device '$devname'\n"
5462                 if !$devinfo->{$devname}->{virtdev};
5463         }
5464
5465         my $cfg = PVE::Storage::config();
5466
5467         # create empty/temp config
5468         if ($oldconf) {
5469             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5470             foreach_drive($oldconf, sub {
5471                 my ($ds, $drive) = @_;
5472
5473                 return if drive_is_cdrom($drive);
5474
5475                 my $volid = $drive->{file};
5476
5477                 return if !$volid || $volid =~ m|^/|;
5478
5479                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5480                 return if !$path || !$owner || ($owner != $vmid);
5481
5482                 # Note: only delete disk we want to restore
5483                 # other volumes will become unused
5484                 if ($virtdev_hash->{$ds}) {
5485                     PVE::Storage::vdisk_free($cfg, $volid);
5486                 }
5487             });
5488
5489             # delete vmstate files
5490             # since after the restore we have no snapshots anymore
5491             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5492                 my $snap = $oldconf->{snapshots}->{$snapname};
5493                 if ($snap->{vmstate}) {
5494                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5495                     if (my $err = $@) {
5496                         warn $err;
5497                     }
5498                 }
5499             }
5500         }
5501
5502         my $map = {};
5503         foreach my $virtdev (sort keys %$virtdev_hash) {
5504             my $d = $virtdev_hash->{$virtdev};
5505             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5506             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5507
5508             # test if requested format is supported
5509             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5510             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5511             $d->{format} = $defFormat if !$supported;
5512
5513             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5514                                                   $d->{format}, undef, $alloc_size);
5515             print STDERR "new volume ID is '$volid'\n";
5516             $d->{volid} = $volid;
5517             my $path = PVE::Storage::path($cfg, $volid);
5518
5519             PVE::Storage::activate_volumes($cfg,[$volid]);
5520
5521             my $write_zeros = 1;
5522             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5523                 $write_zeros = 0;
5524             }
5525
5526             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5527
5528             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5529             $map->{$virtdev} = $volid;
5530         }
5531
5532         $fh->seek(0, 0) || die "seek failed - $!\n";
5533
5534         my $outfd = new IO::File ($tmpfn, "w") ||
5535             die "unable to write config for VM $vmid\n";
5536
5537         my $cookie = { netcount => 0 };
5538         while (defined(my $line = <$fh>)) {
5539             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5540         }
5541
5542         $fh->close();
5543         $outfd->close();
5544     };
5545
5546     eval {
5547         # enable interrupts
5548         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5549             die "interrupted by signal\n";
5550         };
5551         local $SIG{ALRM} = sub { die "got timeout\n"; };
5552
5553         $oldtimeout = alarm($timeout);
5554
5555         my $parser = sub {
5556             my $line = shift;
5557
5558             print "$line\n";
5559
5560             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5561                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5562                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5563             } elsif ($line =~ m/^CTIME: /) {
5564                 # we correctly received the vma config, so we can disable
5565                 # the timeout now for disk allocation (set to 10 minutes, so
5566                 # that we always timeout if something goes wrong)
5567                 alarm(600);
5568                 &$print_devmap();
5569                 print $fifofh "done\n";
5570                 my $tmp = $oldtimeout || 0;
5571                 $oldtimeout = undef;
5572                 alarm($tmp);
5573                 close($fifofh);
5574             }
5575         };
5576
5577         print "restore vma archive: $cmd\n";
5578         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5579     };
5580     my $err = $@;
5581
5582     alarm($oldtimeout) if $oldtimeout;
5583
5584     my $vollist = [];
5585     foreach my $devname (keys %$devinfo) {
5586         my $volid = $devinfo->{$devname}->{volid};
5587         push @$vollist, $volid if $volid;
5588     }
5589
5590     my $cfg = PVE::Storage::config();
5591     PVE::Storage::deactivate_volumes($cfg, $vollist);
5592
5593     unlink $mapfifo;
5594
5595     if ($err) {
5596         rmtree $tmpdir;
5597         unlink $tmpfn;
5598
5599         foreach my $devname (keys %$devinfo) {
5600             my $volid = $devinfo->{$devname}->{volid};
5601             next if !$volid;
5602             eval {
5603                 if ($volid =~ m|^/|) {
5604                     unlink $volid || die 'unlink failed\n';
5605                 } else {
5606                     PVE::Storage::vdisk_free($cfg, $volid);
5607                 }
5608                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5609             };
5610             print STDERR "unable to cleanup '$volid' - $@" if $@;
5611         }
5612         die $err;
5613     }
5614
5615     rmtree $tmpdir;
5616
5617     rename($tmpfn, $conffile) ||
5618         die "unable to commit configuration file '$conffile'\n";
5619
5620     PVE::Cluster::cfs_update(); # make sure we read new file
5621
5622     eval { rescan($vmid, 1); };
5623     warn $@ if $@;
5624 }
5625
5626 sub restore_tar_archive {
5627     my ($archive, $vmid, $user, $opts) = @_;
5628
5629     if ($archive ne '-') {
5630         my $firstfile = tar_archive_read_firstfile($archive);
5631         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5632             if $firstfile ne 'qemu-server.conf';
5633     }
5634
5635     my $storecfg = PVE::Storage::config();
5636
5637     # destroy existing data - keep empty config
5638     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5639     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5640
5641     my $tocmd = "/usr/lib/qemu-server/qmextract";
5642
5643     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5644     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5645     $tocmd .= ' --prealloc' if $opts->{prealloc};
5646     $tocmd .= ' --info' if $opts->{info};
5647
5648     # tar option "xf" does not autodetect compression when read from STDIN,
5649     # so we pipe to zcat
5650     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5651         PVE::Tools::shellquote("--to-command=$tocmd");
5652
5653     my $tmpdir = "/var/tmp/vzdumptmp$$";
5654     mkpath $tmpdir;
5655
5656     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5657     local $ENV{VZDUMP_VMID} = $vmid;
5658     local $ENV{VZDUMP_USER} = $user;
5659
5660     my $conffile = PVE::QemuConfig->config_file($vmid);
5661     my $tmpfn = "$conffile.$$.tmp";
5662
5663     # disable interrupts (always do cleanups)
5664     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5665         print STDERR "got interrupt - ignored\n";
5666     };
5667
5668     eval {
5669         # enable interrupts
5670         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5671             die "interrupted by signal\n";
5672         };
5673
5674         if ($archive eq '-') {
5675             print "extracting archive from STDIN\n";
5676             run_command($cmd, input => "<&STDIN");
5677         } else {
5678             print "extracting archive '$archive'\n";
5679             run_command($cmd);
5680         }
5681
5682         return if $opts->{info};
5683
5684         # read new mapping
5685         my $map = {};
5686         my $statfile = "$tmpdir/qmrestore.stat";
5687         if (my $fd = IO::File->new($statfile, "r")) {
5688             while (defined (my $line = <$fd>)) {
5689                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5690                     $map->{$1} = $2 if $1;
5691                 } else {
5692                     print STDERR "unable to parse line in statfile - $line\n";
5693                 }
5694             }
5695             $fd->close();
5696         }
5697
5698         my $confsrc = "$tmpdir/qemu-server.conf";
5699
5700         my $srcfd = new IO::File($confsrc, "r") ||
5701             die "unable to open file '$confsrc'\n";
5702
5703         my $outfd = new IO::File ($tmpfn, "w") ||
5704             die "unable to write config for VM $vmid\n";
5705
5706         my $cookie = { netcount => 0 };
5707         while (defined (my $line = <$srcfd>)) {
5708             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5709         }
5710
5711         $srcfd->close();
5712         $outfd->close();
5713     };
5714     my $err = $@;
5715
5716     if ($err) {
5717
5718         unlink $tmpfn;
5719
5720         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5721
5722         die $err;
5723     }
5724
5725     rmtree $tmpdir;
5726
5727     rename $tmpfn, $conffile ||
5728         die "unable to commit configuration file '$conffile'\n";
5729
5730     PVE::Cluster::cfs_update(); # make sure we read new file
5731
5732     eval { rescan($vmid, 1); };
5733     warn $@ if $@;
5734 };
5735
5736 sub foreach_writable_storage {
5737     my ($conf, $func) = @_;
5738
5739     my $sidhash = {};
5740
5741     foreach my $ds (keys %$conf) {
5742         next if !is_valid_drivename($ds);
5743
5744         my $drive = parse_drive($ds, $conf->{$ds});
5745         next if !$drive;
5746         next if drive_is_cdrom($drive);
5747
5748         my $volid = $drive->{file};
5749
5750         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5751         $sidhash->{$sid} = $sid if $sid;
5752     }
5753
5754     foreach my $sid (sort keys %$sidhash) {
5755         &$func($sid);
5756     }
5757 }
5758
5759 sub do_snapshots_with_qemu {
5760     my ($storecfg, $volid) = @_;
5761
5762     my $storage_name = PVE::Storage::parse_volume_id($volid);
5763
5764     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5765         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5766         return 1;
5767     }
5768
5769     if ($volid =~ m/\.(qcow2|qed)$/){
5770         return 1;
5771     }
5772
5773     return undef;
5774 }
5775
5776 sub qga_check_running {
5777     my ($vmid) = @_;
5778
5779     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5780     if ($@) {
5781         warn "Qemu Guest Agent are not running - $@";
5782         return 0;
5783     }
5784     return 1;
5785 }
5786
5787 sub template_create {
5788     my ($vmid, $conf, $disk) = @_;
5789
5790     my $storecfg = PVE::Storage::config();
5791
5792     foreach_drive($conf, sub {
5793         my ($ds, $drive) = @_;
5794
5795         return if drive_is_cdrom($drive);
5796         return if $disk && $ds ne $disk;
5797
5798         my $volid = $drive->{file};
5799         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5800
5801         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5802         $drive->{file} = $voliddst;
5803         $conf->{$ds} = print_drive($vmid, $drive);
5804         PVE::QemuConfig->write_config($vmid, $conf);
5805     });
5806 }
5807
5808 sub qemu_img_convert {
5809     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5810
5811     my $storecfg = PVE::Storage::config();
5812     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5813     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5814
5815     if ($src_storeid && $dst_storeid) {
5816
5817         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5818
5819         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5820         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5821
5822         my $src_format = qemu_img_format($src_scfg, $src_volname);
5823         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5824
5825         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5826         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5827
5828         my $cmd = [];
5829         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5830         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5831         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5832         if ($is_zero_initialized) {
5833             push @$cmd, "zeroinit:$dst_path";
5834         } else {
5835             push @$cmd, $dst_path;
5836         }
5837
5838         my $parser = sub {
5839             my $line = shift;
5840             if($line =~ m/\((\S+)\/100\%\)/){
5841                 my $percent = $1;
5842                 my $transferred = int($size * $percent / 100);
5843                 my $remaining = $size - $transferred;
5844
5845                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5846             }
5847
5848         };
5849
5850         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5851         my $err = $@;
5852         die "copy failed: $err" if $err;
5853     }
5854 }
5855
5856 sub qemu_img_format {
5857     my ($scfg, $volname) = @_;
5858
5859     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5860         return $1;
5861     } else {
5862         return "raw";
5863     }
5864 }
5865
5866 sub qemu_drive_mirror {
5867     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5868
5869     my $storecfg = PVE::Storage::config();
5870     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5871
5872     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5873
5874     my $format = qemu_img_format($dst_scfg, $dst_volname);
5875
5876     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5877
5878     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5879
5880     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5881     $opts->{format} = $format if $format;
5882
5883     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5884
5885     eval {
5886     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5887         while (1) {
5888             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5889             my $stat = @$stats[0];
5890             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5891             die "error job is not mirroring" if $stat->{type} ne "mirror";
5892
5893             my $busy = $stat->{busy};
5894             my $ready = $stat->{ready};
5895
5896             if (my $total = $stat->{len}) {
5897                 my $transferred = $stat->{offset} || 0;
5898                 my $remaining = $total - $transferred;
5899                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5900
5901                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5902             }
5903
5904
5905             if ($stat->{ready} eq 'true') {
5906
5907                 last if $vmiddst != $vmid;
5908
5909                 # try to switch the disk if source and destination are on the same guest
5910                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5911                 last if !$@;
5912                 die $@ if $@ !~ m/cannot be completed/;
5913             }
5914             sleep 1;
5915         }
5916
5917
5918     };
5919     my $err = $@;
5920
5921     my $cancel_job = sub {
5922         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5923         while (1) {
5924             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5925             my $stat = @$stats[0];
5926             last if !$stat;
5927             sleep 1;
5928         }
5929     };
5930
5931     if ($err) {
5932         eval { &$cancel_job(); };
5933         die "mirroring error: $err";
5934     }
5935
5936     if ($vmiddst != $vmid) {
5937         # if we clone a disk for a new target vm, we don't switch the disk
5938         &$cancel_job(); # so we call block-job-cancel
5939     }
5940 }
5941
5942 sub clone_disk {
5943     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5944         $newvmid, $storage, $format, $full, $newvollist) = @_;
5945
5946     my $newvolid;
5947
5948     if (!$full) {
5949         print "create linked clone of drive $drivename ($drive->{file})\n";
5950         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5951         push @$newvollist, $newvolid;
5952     } else {
5953         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5954         $storeid = $storage if $storage;
5955
5956         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5957         if (!$format) {
5958             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5959             $format = qemu_img_format($scfg, $volname);
5960         }
5961
5962         # test if requested format is supported - else use default
5963         my $supported = grep { $_ eq $format } @$validFormats;
5964         $format = $defFormat if !$supported;
5965
5966         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5967
5968         print "create full clone of drive $drivename ($drive->{file})\n";
5969         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5970         push @$newvollist, $newvolid;
5971
5972         PVE::Storage::activate_volumes($storecfg, $newvollist);
5973
5974         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5975         if (!$running || $snapname) {
5976             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5977         } else {
5978             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5979         }
5980     }
5981
5982     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5983
5984     my $disk = $drive;
5985     $disk->{format} = undef;
5986     $disk->{file} = $newvolid;
5987     $disk->{size} = $size;
5988
5989     return $disk;
5990 }
5991
5992 # this only works if VM is running
5993 sub get_current_qemu_machine {
5994     my ($vmid) = @_;
5995
5996     my $cmd = { execute => 'query-machines', arguments => {} };
5997     my $res = vm_qmp_command($vmid, $cmd);
5998
5999     my ($current, $default);
6000     foreach my $e (@$res) {
6001         $default = $e->{name} if $e->{'is-default'};
6002         $current = $e->{name} if $e->{'is-current'};
6003     }
6004
6005     # fallback to the default machine if current is not supported by qemu
6006     return $current || $default || 'pc';
6007 }
6008
6009 sub qemu_machine_feature_enabled {
6010     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6011
6012     my $current_major;
6013     my $current_minor;
6014
6015     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6016
6017         $current_major = $3;
6018         $current_minor = $4;
6019
6020     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6021
6022         $current_major = $1;
6023         $current_minor = $2;
6024     }
6025
6026     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6027
6028
6029 }
6030
6031 sub qemu_machine_pxe {
6032     my ($vmid, $conf, $machine) = @_;
6033
6034     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6035
6036     foreach my $opt (keys %$conf) {
6037         next if $opt !~ m/^net(\d+)$/;
6038         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6039         next if !$net;
6040         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6041         return $machine.".pxe" if $romfile =~ m/pxe/;
6042         last;
6043     }
6044
6045     return $machine;
6046 }
6047
6048 sub qemu_use_old_bios_files {
6049     my ($machine_type) = @_;
6050
6051     return if !$machine_type;
6052
6053     my $use_old_bios_files = undef;
6054
6055     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6056         $machine_type = $1;
6057         $use_old_bios_files = 1;
6058     } else {
6059         my $kvmver = kvm_user_version();
6060         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6061         # load new efi bios files on migration. So this hack is required to allow
6062         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6063         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6064         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6065     }
6066
6067     return ($use_old_bios_files, $machine_type);
6068 }
6069
6070 sub lspci {
6071
6072     my $devices = {};
6073
6074     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6075             my (undef, $id, $function) = @_;
6076             my $res = { id => $id, function => $function};
6077             push @{$devices->{$id}}, $res;
6078     });
6079
6080     return $devices;
6081 }
6082
6083 sub vm_iothreads_list {
6084     my ($vmid) = @_;
6085
6086     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6087
6088     my $iothreads = {};
6089     foreach my $iothread (@$res) {
6090         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6091     }
6092
6093     return $iothreads;
6094 }
6095
6096 sub scsihw_infos {
6097     my ($conf, $drive) = @_;
6098
6099     my $maxdev = 0;
6100
6101     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6102         $maxdev = 7;
6103     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6104         $maxdev = 1;
6105     } else {
6106         $maxdev = 256;
6107     }
6108
6109     my $controller = int($drive->{index} / $maxdev);
6110     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6111
6112     return ($maxdev, $controller, $controller_prefix);
6113 }
6114
6115 # bash completion helper
6116
6117 sub complete_backup_archives {
6118     my ($cmdname, $pname, $cvalue) = @_;
6119
6120     my $cfg = PVE::Storage::config();
6121
6122     my $storeid;
6123
6124     if ($cvalue =~ m/^([^:]+):/) {
6125         $storeid = $1;
6126     }
6127
6128     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6129
6130     my $res = [];
6131     foreach my $id (keys %$data) {
6132         foreach my $item (@{$data->{$id}}) {
6133             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6134             push @$res, $item->{volid} if defined($item->{volid});
6135         }
6136     }
6137
6138     return $res;
6139 }
6140
6141 my $complete_vmid_full = sub {
6142     my ($running) = @_;
6143
6144     my $idlist = vmstatus();
6145
6146     my $res = [];
6147
6148     foreach my $id (keys %$idlist) {
6149         my $d = $idlist->{$id};
6150         if (defined($running)) {
6151             next if $d->{template};
6152             next if $running && $d->{status} ne 'running';
6153             next if !$running && $d->{status} eq 'running';
6154         }
6155         push @$res, $id;
6156
6157     }
6158     return $res;
6159 };
6160
6161 sub complete_vmid {
6162     return &$complete_vmid_full();
6163 }
6164
6165 sub complete_vmid_stopped {
6166     return &$complete_vmid_full(0);
6167 }
6168
6169 sub complete_vmid_running {
6170     return &$complete_vmid_full(1);
6171 }
6172
6173 sub complete_storage {
6174
6175     my $cfg = PVE::Storage::config();
6176     my $ids = $cfg->{ids};
6177
6178     my $res = [];
6179     foreach my $sid (keys %$ids) {
6180         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6181         next if !$ids->{$sid}->{content}->{images};
6182         push @$res, $sid;
6183     }
6184
6185     return $res;
6186 }
6187
6188 1;