PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393     bios => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [ qw(seabios ovmf) ],
 397         description => "Select BIOS implementation.",
 398         default => 'seabios',
 399     },
 400 };
 401
 402 # what about other qemu settings ?
 403 #cpu => 'string',
 404 #machine => 'string',
 405 #fda => 'file',
 406 #fdb => 'file',
 407 #mtdblock => 'file',
 408 #sd => 'file',
 409 #pflash => 'file',
 410 #snapshot => 'bool',
 411 #bootp => 'file',
 412 ##tftp => 'dir',
 413 ##smb => 'dir',
 414 #kernel => 'file',
 415 #append => 'string',
 416 #initrd => 'file',
 417 ##soundhw => 'string',
 418
 419 while (my ($k, $v) = each %$confdesc) {
 420     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 421 }
 422
 423 my $MAX_IDE_DISKS = 4;
 424 my $MAX_SCSI_DISKS = 14;
 425 my $MAX_VIRTIO_DISKS = 16;
 426 my $MAX_SATA_DISKS = 6;
 427 my $MAX_USB_DEVICES = 5;
 428 my $MAX_NETS = 32;
 429 my $MAX_UNUSED_DISKS = 8;
 430 my $MAX_HOSTPCI_DEVICES = 4;
 431 my $MAX_SERIAL_PORTS = 4;
 432 my $MAX_PARALLEL_PORTS = 3;
 433 my $MAX_NUMA = 8;
 434 my $MAX_MEM = 4194304;
 435 my $STATICMEM = 1024;
 436
 437 my $numadesc = {
 438     optional => 1,
 439     type => 'string', format => 'pve-qm-numanode',
 440     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 441     description => "numa topology",
 442 };
 443 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 444
 445 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 446     $confdesc->{"numa$i"} = $numadesc;
 447 }
 448
 449 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 450                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 451                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 452 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 453
 454 my $netdesc = {
 455     optional => 1,
 456     type => 'string', format => 'pve-qm-net',
 457     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 458     description => <<EODESCR,
 459 Specify network devices.
 460
 461 MODEL is one of: $nic_model_list_txt
 462
 463 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 464 automatically generated if not specified.
 465
 466 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 467
 468 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 469
 470 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 471
 472 10.0.2.2   Gateway
 473 10.0.2.3   DNS Server
 474 10.0.2.4   SMB Server
 475
 476 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 477
 478 EODESCR
 479 };
 480 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 481
 482 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 483     $confdesc->{"net$i"} = $netdesc;
 484 }
 485
 486 my $drivename_hash;
 487
 488 my %drivedesc_base = (
 489     volume => { alias => 'file' },
 490     file => {
 491         type => 'pve-volume-id',
 492         default_key => 1,
 493         format_description => 'volume',
 494         description => "The drive's backing volume.",
 495     },
 496     media => {
 497         type => 'string',
 498         format_description => 'cdrom|disk',
 499         enum => [qw(cdrom disk)],
 500         description => "The drive's media type.",
 501         default => 'disk',
 502         optional => 1
 503     },
 504     cyls => {
 505         type => 'integer',
 506         format_description => 'count',
 507         description => "Force the drive's physical geometry to have a specific cylinder count.",
 508         optional => 1
 509     },
 510     heads => {
 511         type => 'integer',
 512         format_description => 'count',
 513         description => "Force the drive's physical geometry to have a specific head count.",
 514         optional => 1
 515     },
 516     secs => {
 517         type => 'integer',
 518         format_description => 'count',
 519         description => "Force the drive's physical geometry to have a specific sector count.",
 520         optional => 1
 521     },
 522     trans => {
 523         type => 'string',
 524         format_description => 'none|lba|auto',
 525         enum => [qw(none lba auto)],
 526         description => "Force disk geometry bios translation mode.",
 527         optional => 1,
 528     },
 529     snapshot => {
 530         type => 'boolean',
 531         format_description => 'on|off',
 532         description => "Whether the drive should be included when making snapshots.",
 533         optional => 1,
 534     },
 535     cache => {
 536         type => 'string',
 537         format_description => 'none|writethrough|writeback|unsafe|directsync',
 538         enum => [qw(none writethrough writeback unsafe directsync)],
 539         description => "The drive's cache mode",
 540         optional => 1,
 541     },
 542     format => {
 543         type => 'string',
 544         format_description => 'drive format',
 545         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 546         description => "The drive's backing file's data format.",
 547         optional => 1,
 548     },
 549     size => {
 550         type => 'disk-size',
 551         description => "Disk size. This is purely informational and has no effect.",
 552         optional => 1,
 553     },
 554     backup => {
 555         type => 'boolean',
 556         format_description => 'on|off',
 557         description => "Whether the drive should be included when making backups.",
 558         optional => 1,
 559     },
 560     werror => {
 561         type => 'string',
 562         format_description => 'enospc|ignore|report|stop',
 563         enum => [qw(enospc ignore report stop)],
 564         description => 'Write error action.',
 565         optional => 1,
 566     },
 567     aio => {
 568         type => 'string',
 569         format_description => 'native|threads',
 570         enum => [qw(native threads)],
 571         description => 'AIO type to use.',
 572         optional => 1,
 573     },
 574     discard => {
 575         type => 'string',
 576         format_description => 'ignore|on',
 577         enum => [qw(ignore on)],
 578         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 579         optional => 1,
 580     },
 581     detect_zeroes => {
 582         type => 'boolean',
 583         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 584         optional => 1,
 585     },
 586     serial => {
 587         type => 'string',
 588         format_description => 'serial',
 589         description => "The drive's reported serial number.",
 590         optional => 1,
 591     }
 592 );
 593
 594 my %rerror_fmt = (
 595     rerror => {
 596         type => 'string',
 597         format_description => 'ignore|report|stop',
 598         enum => [qw(ignore report stop)],
 599         description => 'Read error action.',
 600         optional => 1,
 601     },
 602 );
 603
 604 my %iothread_fmt = ( iothread => {
 605         type => 'boolean',
 606         format_description => 'off|on',
 607         description => "Whether to use iothreads for this drive",
 608         optional => 1,
 609 });
 610
 611 my %model_fmt = (
 612     model => {
 613         type => 'string',
 614         format_description => 'model',
 615         description => "The drive's reported model name.",
 616         optional => 1,
 617     },
 618 );
 619
 620 my %queues_fmt = (
 621     queues => {
 622         type => 'integer',
 623         format_description => 'nbqueues',
 624         description => "Number of queues.",
 625         minimum => 2,
 626         optional => 1
 627     }
 628 );
 629
 630 my $add_throttle_desc = sub {
 631     my ($key, $type, $what, $size, $longsize) = @_;
 632     $drivedesc_base{$key} = {
 633         type => $type,
 634         format_description => $size,
 635         description => "Maximum $what speed in $longsize per second.",
 636         optional => 1,
 637     };
 638 };
 639 # throughput: (leaky bucket)
 640 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 641 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 642 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 643 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 644 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 645 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 646 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 647 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 648 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 649
 650 # pools: (pool of IO before throttling starts taking effect)
 651 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 652 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 653 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 654 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 655 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 656 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 657
 658 my $ide_fmt = {
 659     %drivedesc_base,
 660     %rerror_fmt,
 661     %model_fmt,
 662 };
 663
 664 my $idedesc = {
 665     optional => 1,
 666     type => 'string', format => $ide_fmt,
 667     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 668 };
 669 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 670
 671 my $scsi_fmt = {
 672     %drivedesc_base,
 673     %iothread_fmt,
 674     %queues_fmt,
 675 };
 676 my $scsidesc = {
 677     optional => 1,
 678     type => 'string', format => $scsi_fmt,
 679     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 680 };
 681 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 682
 683 my $sata_fmt = {
 684     %drivedesc_base,
 685     %rerror_fmt,
 686 };
 687 my $satadesc = {
 688     optional => 1,
 689     type => 'string', format => $sata_fmt,
 690     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 691 };
 692 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 693
 694 my $virtio_fmt = {
 695     %drivedesc_base,
 696     %iothread_fmt,
 697     %rerror_fmt,
 698 };
 699 my $virtiodesc = {
 700     optional => 1,
 701     type => 'string', format => $virtio_fmt,
 702     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 703 };
 704 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 705
 706 my $alldrive_fmt = {
 707     %drivedesc_base,
 708     %rerror_fmt,
 709     %iothread_fmt,
 710     %model_fmt,
 711     %queues_fmt,
 712 };
 713
 714 my $usbdesc = {
 715     optional => 1,
 716     type => 'string', format => 'pve-qm-usb-device',
 717     typetext => 'host=HOSTUSBDEVICE|spice',
 718     description => <<EODESCR,
 719 Configure an USB device (n is 0 to 4). This can be used to
 720 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 721
 722 'bus-port(.port)*' (decimal numbers) or
 723 'vendor_id:product_id' (hexadeciaml numbers)
 724
 725 You can use the 'lsusb -t' command to list existing usb devices.
 726
 727 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 728
 729 The value 'spice' can be used to add a usb redirection devices for spice.
 730
 731 EODESCR
 732 };
 733 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 734
 735 my $hostpcidesc = {
 736         optional => 1,
 737         type => 'string', format => 'pve-qm-hostpci',
 738         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 739         description => <<EODESCR,
 740 Map host pci devices. HOSTPCIDEVICE syntax is:
 741
 742 'bus:dev.func' (hexadecimal numbers)
 743
 744 You can us the 'lspci' command to list existing pci devices.
 745
 746 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 747
 748 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 749
 750 Experimental: user reported problems with this option.
 751 EODESCR
 752 };
 753 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 754
 755 my $serialdesc = {
 756         optional => 1,
 757         type => 'string',
 758         pattern => '(/dev/.+|socket)',
 759         description =>  <<EODESCR,
 760 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 761
 762 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 763
 764 Experimental: user reported problems with this option.
 765 EODESCR
 766 };
 767
 768 my $paralleldesc= {
 769         optional => 1,
 770         type => 'string',
 771         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 772         description =>  <<EODESCR,
 773 Map host parallel devices (n is 0 to 2).
 774
 775 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 776
 777 Experimental: user reported problems with this option.
 778 EODESCR
 779 };
 780
 781 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 782     $confdesc->{"parallel$i"} = $paralleldesc;
 783 }
 784
 785 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 786     $confdesc->{"serial$i"} = $serialdesc;
 787 }
 788
 789 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 790     $confdesc->{"hostpci$i"} = $hostpcidesc;
 791 }
 792
 793 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 794     $drivename_hash->{"ide$i"} = 1;
 795     $confdesc->{"ide$i"} = $idedesc;
 796 }
 797
 798 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 799     $drivename_hash->{"sata$i"} = 1;
 800     $confdesc->{"sata$i"} = $satadesc;
 801 }
 802
 803 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 804     $drivename_hash->{"scsi$i"} = 1;
 805     $confdesc->{"scsi$i"} = $scsidesc ;
 806 }
 807
 808 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 809     $drivename_hash->{"virtio$i"} = 1;
 810     $confdesc->{"virtio$i"} = $virtiodesc;
 811 }
 812
 813 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 814     $confdesc->{"usb$i"} = $usbdesc;
 815 }
 816
 817 my $unuseddesc = {
 818     optional => 1,
 819     type => 'string', format => 'pve-volume-id',
 820     description => "Reference to unused volumes.",
 821 };
 822
 823 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 824     $confdesc->{"unused$i"} = $unuseddesc;
 825 }
 826
 827 my $kvm_api_version = 0;
 828
 829 sub kvm_version {
 830
 831     return $kvm_api_version if $kvm_api_version;
 832
 833     my $fh = IO::File->new("</dev/kvm") ||
 834         return 0;
 835
 836     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 837         $kvm_api_version = $v;
 838     }
 839
 840     $fh->close();
 841
 842     return  $kvm_api_version;
 843 }
 844
 845 my $kvm_user_version;
 846
 847 sub kvm_user_version {
 848
 849     return $kvm_user_version if $kvm_user_version;
 850
 851     $kvm_user_version = 'unknown';
 852
 853     my $tmp = `kvm -help 2>/dev/null`;
 854
 855     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 856         $kvm_user_version = $2;
 857     }
 858
 859     return $kvm_user_version;
 860
 861 }
 862
 863 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 864
 865 sub disknames {
 866     # order is important - used to autoselect boot disk
 867     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 868             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 869             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 870             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 871 }
 872
 873 sub valid_drivename {
 874     my $dev = shift;
 875
 876     return defined($drivename_hash->{$dev});
 877 }
 878
 879 sub option_exists {
 880     my $key = shift;
 881     return defined($confdesc->{$key});
 882 }
 883
 884 sub nic_models {
 885     return $nic_model_list;
 886 }
 887
 888 sub os_list_description {
 889
 890     return {
 891         other => 'Other',
 892         wxp => 'Windows XP',
 893         w2k => 'Windows 2000',
 894         w2k3 =>, 'Windows 2003',
 895         w2k8 => 'Windows 2008',
 896         wvista => 'Windows Vista',
 897         win7 => 'Windows 7',
 898         win8 => 'Windows 8/2012',
 899         l24 => 'Linux 2.4',
 900         l26 => 'Linux 2.6',
 901     };
 902 }
 903
 904 my $cdrom_path;
 905
 906 sub get_cdrom_path {
 907
 908     return  $cdrom_path if $cdrom_path;
 909
 910     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 911     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 912     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 913 }
 914
 915 sub get_iso_path {
 916     my ($storecfg, $vmid, $cdrom) = @_;
 917
 918     if ($cdrom eq 'cdrom') {
 919         return get_cdrom_path();
 920     } elsif ($cdrom eq 'none') {
 921         return '';
 922     } elsif ($cdrom =~ m|^/|) {
 923         return $cdrom;
 924     } else {
 925         return PVE::Storage::path($storecfg, $cdrom);
 926     }
 927 }
 928
 929 # try to convert old style file names to volume IDs
 930 sub filename_to_volume_id {
 931     my ($vmid, $file, $media) = @_;
 932
 933     if (!($file eq 'none' || $file eq 'cdrom' ||
 934           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 935
 936         return undef if $file =~ m|/|;
 937
 938         if ($media && $media eq 'cdrom') {
 939             $file = "local:iso/$file";
 940         } else {
 941             $file = "local:$vmid/$file";
 942         }
 943     }
 944
 945     return $file;
 946 }
 947
 948 sub verify_media_type {
 949     my ($opt, $vtype, $media) = @_;
 950
 951     return if !$media;
 952
 953     my $etype;
 954     if ($media eq 'disk') {
 955         $etype = 'images';
 956     } elsif ($media eq 'cdrom') {
 957         $etype = 'iso';
 958     } else {
 959         die "internal error";
 960     }
 961
 962     return if ($vtype eq $etype);
 963
 964     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 965 }
 966
 967 sub cleanup_drive_path {
 968     my ($opt, $storecfg, $drive) = @_;
 969
 970     # try to convert filesystem paths to volume IDs
 971
 972     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 973         ($drive->{file} !~ m|^/dev/.+|) &&
 974         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 975         ($drive->{file} !~ m/^\d+$/)) {
 976         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 977         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 978         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 979         verify_media_type($opt, $vtype, $drive->{media});
 980         $drive->{file} = $volid;
 981     }
 982
 983     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 984 }
 985
 986 sub create_conf_nolock {
 987     my ($vmid, $settings) = @_;
 988
 989     my $filename = config_file($vmid);
 990
 991     die "configuration file '$filename' already exists\n" if -f $filename;
 992
 993     my $defaults = load_defaults();
 994
 995     $settings->{name} = "vm$vmid" if !$settings->{name};
 996     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 997
 998     my $data = '';
 999     foreach my $opt (keys %$settings) {
1000         next if !$confdesc->{$opt};
1001
1002         my $value = $settings->{$opt};
1003         next if !$value;
1004
1005         $data .= "$opt: $value\n";
1006     }
1007
1008     PVE::Tools::file_set_contents($filename, $data);
1009 }
1010
1011 sub parse_hotplug_features {
1012     my ($data) = @_;
1013
1014     my $res = {};
1015
1016     return $res if $data eq '0';
1017
1018     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1019
1020     foreach my $feature (PVE::Tools::split_list($data)) {
1021         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1022             $res->{$1} = 1;
1023         } else {
1024             warn "ignoring unknown hotplug feature '$feature'\n";
1025         }
1026     }
1027     return $res;
1028 }
1029
1030 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1031 sub pve_verify_hotplug_features {
1032     my ($value, $noerr) = @_;
1033
1034     return $value if parse_hotplug_features($value);
1035
1036     return undef if $noerr;
1037
1038     die "unable to parse hotplug option\n";
1039 }
1040
1041 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1042 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1043 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1044 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1045 #        [,iothread=on][,serial=serial][,model=model]
1046
1047 sub parse_drive {
1048     my ($key, $data) = @_;
1049
1050     my ($interface, $index);
1051
1052     if ($key =~ m/^([^\d]+)(\d+)$/) {
1053         $interface = $1;
1054         $index = $2;
1055     } else {
1056         return undef;
1057     }
1058
1059     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1060                                      : $confdesc->{$key}->{format};
1061     if (!$desc) {
1062         warn "invalid drive key: $key\n";
1063         return undef;
1064     }
1065     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1066     return undef if !$res;
1067     $res->{interface} = $interface;
1068     $res->{index} = $index;
1069
1070     my $error = 0;
1071     foreach my $opt (qw(bps bps_rd bps_wr)) {
1072         if (my $bps = defined(delete $res->{$opt})) {
1073             if (defined($res->{"m$opt"})) {
1074                 warn "both $opt and m$opt specified\n";
1075                 ++$error;
1076                 next;
1077             }
1078             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1079         }
1080     }
1081     return undef if $error;
1082
1083     return undef if $res->{mbps_rd} && $res->{mbps};
1084     return undef if $res->{mbps_wr} && $res->{mbps};
1085     return undef if $res->{iops_rd} && $res->{iops};
1086     return undef if $res->{iops_wr} && $res->{iops};
1087
1088     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1089         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1090         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1091         return undef if $res->{interface} eq 'virtio';
1092     }
1093
1094     if (my $size = $res->{size}) {
1095         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1096     }
1097
1098     return $res;
1099 }
1100
1101 sub print_drive {
1102     my ($vmid, $drive) = @_;
1103     my $data = { %$drive };
1104     delete $data->{$_} for qw(index interface);
1105     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1106 }
1107
1108 sub scsi_inquiry {
1109     my($fh, $noerr) = @_;
1110
1111     my $SG_IO = 0x2285;
1112     my $SG_GET_VERSION_NUM = 0x2282;
1113
1114     my $versionbuf = "\x00" x 8;
1115     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1116     if (!$ret) {
1117         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1118         return undef;
1119     }
1120     my $version = unpack("I", $versionbuf);
1121     if ($version < 30000) {
1122         die "scsi generic interface too old\n"  if !$noerr;
1123         return undef;
1124     }
1125
1126     my $buf = "\x00" x 36;
1127     my $sensebuf = "\x00" x 8;
1128     my $cmd = pack("C x3 C x1", 0x12, 36);
1129
1130     # see /usr/include/scsi/sg.h
1131     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1132
1133     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1134                       length($sensebuf), 0, length($buf), $buf,
1135                       $cmd, $sensebuf, 6000);
1136
1137     $ret = ioctl($fh, $SG_IO, $packet);
1138     if (!$ret) {
1139         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1140         return undef;
1141     }
1142
1143     my @res = unpack($sg_io_hdr_t, $packet);
1144     if ($res[17] || $res[18]) {
1145         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1146         return undef;
1147     }
1148
1149     my $res = {};
1150     (my $byte0, my $byte1, $res->{vendor},
1151      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1152
1153     $res->{removable} = $byte1 & 128 ? 1 : 0;
1154     $res->{type} = $byte0 & 31;
1155
1156     return $res;
1157 }
1158
1159 sub path_is_scsi {
1160     my ($path) = @_;
1161
1162     my $fh = IO::File->new("+<$path") || return undef;
1163     my $res = scsi_inquiry($fh, 1);
1164     close($fh);
1165
1166     return $res;
1167 }
1168
1169 sub machine_type_is_q35 {
1170     my ($conf) = @_;
1171
1172     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1173 }
1174
1175 sub print_tabletdevice_full {
1176     my ($conf) = @_;
1177
1178     my $q35 = machine_type_is_q35($conf);
1179
1180     # we use uhci for old VMs because tablet driver was buggy in older qemu
1181     my $usbbus = $q35 ? "ehci" : "uhci";
1182
1183     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1184 }
1185
1186 sub print_drivedevice_full {
1187     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1188
1189     my $device = '';
1190     my $maxdev = 0;
1191
1192     if ($drive->{interface} eq 'virtio') {
1193         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1194         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1195         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1196     } elsif ($drive->{interface} eq 'scsi') {
1197
1198         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1199         my $unit = $drive->{index} % $maxdev;
1200         my $devicetype = 'hd';
1201         my $path = '';
1202         if (drive_is_cdrom($drive)) {
1203               $devicetype = 'cd';
1204         } else {
1205               if ($drive->{file} =~ m|^/|) {
1206                   $path = $drive->{file};
1207               } else {
1208                   $path = PVE::Storage::path($storecfg, $drive->{file});
1209               }
1210
1211               if($path =~ m/^iscsi\:\/\//){
1212                   $devicetype = 'generic';
1213               } else {
1214                   if (my $info = path_is_scsi($path)) {
1215                       if ($info->{type} == 0) {
1216                           $devicetype = 'block';
1217                       } elsif ($info->{type} == 1) { # tape
1218                           $devicetype = 'generic';
1219                       }
1220                   }
1221               }
1222          }
1223
1224         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1225             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1226         } else {
1227             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1228         }
1229
1230     } elsif ($drive->{interface} eq 'ide'){
1231         $maxdev = 2;
1232         my $controller = int($drive->{index} / $maxdev);
1233         my $unit = $drive->{index} % $maxdev;
1234         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1235
1236         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1237         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1238             $device .= ",model=$model";
1239         }
1240     } elsif ($drive->{interface} eq 'sata'){
1241         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1242         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1243         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1244     } elsif ($drive->{interface} eq 'usb') {
1245         die "implement me";
1246         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1247     } else {
1248         die "unsupported interface type";
1249     }
1250
1251     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1252
1253     return $device;
1254 }
1255
1256 sub get_initiator_name {
1257     my $initiator;
1258
1259     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1260     while (defined(my $line = <$fh>)) {
1261         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1262         $initiator = $1;
1263         last;
1264     }
1265     $fh->close();
1266
1267     return $initiator;
1268 }
1269
1270 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1271 sub print_drive_full {
1272     my ($storecfg, $vmid, $drive) = @_;
1273
1274     my $path;
1275     my $volid = $drive->{file};
1276     my $format;
1277
1278     if (drive_is_cdrom($drive)) {
1279         $path = get_iso_path($storecfg, $vmid, $volid);
1280     } else {
1281         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1282         if ($storeid) {
1283             $path = PVE::Storage::path($storecfg, $volid);
1284             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1285             $format = qemu_img_format($scfg, $volname);
1286         } else {
1287             $path = $volid;
1288         }
1289    }
1290
1291     my $opts = '';
1292     foreach my $o (@qemu_drive_options) {
1293         next if $o eq 'bootindex';
1294         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1295     }
1296
1297     $opts .= ",format=$format" if $format && !$drive->{format};
1298
1299     foreach my $o (qw(bps bps_rd bps_wr)) {
1300         my $v = $drive->{"m$o"};
1301         $opts .= ",$o=" . int($v*1024*1024) if $v;
1302     }
1303
1304     my $cache_direct = 0;
1305
1306     if (my $cache = $drive->{cache}) {
1307         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1308     } elsif (!drive_is_cdrom($drive)) {
1309         $opts .= ",cache=none";
1310         $cache_direct = 1;
1311     }
1312
1313     # aio native works only with O_DIRECT
1314     if (!$drive->{aio}) {
1315         if($cache_direct) {
1316             $opts .= ",aio=native";
1317         } else {
1318             $opts .= ",aio=threads";
1319         }
1320     }
1321
1322     if (!drive_is_cdrom($drive)) {
1323         my $detectzeroes;
1324         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1325             $detectzeroes = 'off';
1326         } elsif ($drive->{discard}) {
1327             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1328         } else {
1329             # This used to be our default with discard not being specified:
1330             $detectzeroes = 'on';
1331         }
1332         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1333     }
1334
1335     my $pathinfo = $path ? "file=$path," : '';
1336
1337     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1338 }
1339
1340 sub print_netdevice_full {
1341     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1342
1343     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1344
1345     my $device = $net->{model};
1346     if ($net->{model} eq 'virtio') {
1347          $device = 'virtio-net-pci';
1348      };
1349
1350     my $pciaddr = print_pci_addr("$netid", $bridges);
1351     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1352     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1353         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1354         my $vectors = $net->{queues} * 2 + 2;
1355         $tmpstr .= ",vectors=$vectors,mq=on";
1356     }
1357     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1358
1359     if ($use_old_bios_files) {
1360         my $romfile;
1361         if ($device eq 'virtio-net-pci') {
1362             $romfile = 'pxe-virtio.rom';
1363         } elsif ($device eq 'e1000') {
1364             $romfile = 'pxe-e1000.rom';
1365         } elsif ($device eq 'ne2k') {
1366             $romfile = 'pxe-ne2k_pci.rom';
1367         } elsif ($device eq 'pcnet') {
1368             $romfile = 'pxe-pcnet.rom';
1369         } elsif ($device eq 'rtl8139') {
1370             $romfile = 'pxe-rtl8139.rom';
1371         }
1372         $tmpstr .= ",romfile=$romfile" if $romfile;
1373     }
1374
1375     return $tmpstr;
1376 }
1377
1378 sub print_netdev_full {
1379     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1380
1381     my $i = '';
1382     if ($netid =~ m/^net(\d+)$/) {
1383         $i = int($1);
1384     }
1385
1386     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1387
1388     my $ifname = "tap${vmid}i$i";
1389
1390     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1391     die "interface name '$ifname' is too long (max 15 character)\n"
1392         if length($ifname) >= 16;
1393
1394     my $vhostparam = '';
1395     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1396
1397     my $vmname = $conf->{name} || "vm$vmid";
1398
1399     my $netdev = "";
1400     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1401
1402     if ($net->{bridge}) {
1403         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1404     } else {
1405         $netdev = "type=user,id=$netid,hostname=$vmname";
1406     }
1407
1408     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1409
1410     return $netdev;
1411 }
1412
1413 sub drive_is_cdrom {
1414     my ($drive) = @_;
1415
1416     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1417
1418 }
1419
1420 sub parse_numa {
1421     my ($data) = @_;
1422
1423     my $res = {};
1424
1425     foreach my $kvp (split(/,/, $data)) {
1426
1427         if ($kvp =~ m/^memory=(\S+)$/) {
1428             $res->{memory} = $1;
1429         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1430             $res->{policy} = $1;
1431         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1432             $res->{cpus}->{start} = $1;
1433             $res->{cpus}->{end} = $3;
1434         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1435             $res->{hostnodes}->{start} = $1;
1436             $res->{hostnodes}->{end} = $3;
1437         } else {
1438             return undef;
1439         }
1440     }
1441
1442     return $res;
1443 }
1444
1445 sub parse_hostpci {
1446     my ($value) = @_;
1447
1448     return undef if !$value;
1449
1450
1451     my @list = split(/,/, $value);
1452     my $found;
1453
1454     my $res = {};
1455     foreach my $kv (@list) {
1456
1457         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1458             $found = 1;
1459             if(defined($4)){
1460                 push @{$res->{pciid}}, { id => $2 , function => $4};
1461
1462             }else{
1463                 my $pcidevices = lspci($2);
1464                 $res->{pciid} = $pcidevices->{$2};
1465             }
1466         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1467             $res->{rombar} = $1;
1468         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1469             $res->{'x-vga'} = $1;
1470         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1471             $res->{pcie} = 1 if $1 == 1;
1472         } else {
1473             warn "unknown hostpci setting '$kv'\n";
1474         }
1475     }
1476
1477     return undef if !$found;
1478
1479     return $res;
1480 }
1481
1482 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1483 sub parse_net {
1484     my ($data) = @_;
1485
1486     my $res = {};
1487
1488     foreach my $kvp (split(/,/, $data)) {
1489
1490         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1491             my $model = lc($1);
1492             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1493             $res->{model} = $model;
1494             $res->{macaddr} = $mac;
1495         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1496             $res->{bridge} = $1;
1497         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1498             $res->{queues} = $1;
1499         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1500             $res->{rate} = $1;
1501         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1502             $res->{tag} = $1;
1503         } elsif ($kvp =~ m/^firewall=([01])$/) {
1504             $res->{firewall} = $1;
1505         } elsif ($kvp =~ m/^link_down=([01])$/) {
1506             $res->{link_down} = $1;
1507         } else {
1508             return undef;
1509         }
1510
1511     }
1512
1513     return undef if !$res->{model};
1514
1515     return $res;
1516 }
1517
1518 sub print_net {
1519     my $net = shift;
1520
1521     my $res = "$net->{model}";
1522     $res .= "=$net->{macaddr}" if $net->{macaddr};
1523     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1524     $res .= ",rate=$net->{rate}" if $net->{rate};
1525     $res .= ",tag=$net->{tag}" if $net->{tag};
1526     $res .= ",firewall=1" if $net->{firewall};
1527     $res .= ",link_down=1" if $net->{link_down};
1528     $res .= ",queues=$net->{queues}" if $net->{queues};
1529
1530     return $res;
1531 }
1532
1533 sub add_random_macs {
1534     my ($settings) = @_;
1535
1536     foreach my $opt (keys %$settings) {
1537         next if $opt !~ m/^net(\d+)$/;
1538         my $net = parse_net($settings->{$opt});
1539         next if !$net;
1540         $settings->{$opt} = print_net($net);
1541     }
1542 }
1543
1544 sub add_unused_volume {
1545     my ($config, $volid) = @_;
1546
1547     my $key;
1548     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1549         my $test = "unused$ind";
1550         if (my $vid = $config->{$test}) {
1551             return if $vid eq $volid; # do not add duplicates
1552         } else {
1553             $key = $test;
1554         }
1555     }
1556
1557     die "To many unused volume - please delete them first.\n" if !$key;
1558
1559     $config->{$key} = $volid;
1560
1561     return $key;
1562 }
1563
1564 sub vm_is_volid_owner {
1565     my ($storecfg, $vmid, $volid) = @_;
1566
1567     if ($volid !~  m|^/|) {
1568         my ($path, $owner);
1569         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1570         if ($owner && ($owner == $vmid)) {
1571             return 1;
1572         }
1573     }
1574
1575     return undef;
1576 }
1577
1578 sub split_flagged_list {
1579     my $text = shift || '';
1580     $text =~ s/[,;]/ /g;
1581     $text =~ s/^\s+//;
1582     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1583 }
1584
1585 sub join_flagged_list {
1586     my ($how, $lst) = @_;
1587     join $how, map { $lst->{$_} . $_ } keys %$lst;
1588 }
1589
1590 sub vmconfig_delete_pending_option {
1591     my ($conf, $key, $force) = @_;
1592
1593     delete $conf->{pending}->{$key};
1594     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1595     $pending_delete_hash->{$key} = $force ? '!' : '';
1596     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1597 }
1598
1599 sub vmconfig_undelete_pending_option {
1600     my ($conf, $key) = @_;
1601
1602     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1603     delete $pending_delete_hash->{$key};
1604
1605     if (%$pending_delete_hash) {
1606         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1607     } else {
1608         delete $conf->{pending}->{delete};
1609     }
1610 }
1611
1612 sub vmconfig_register_unused_drive {
1613     my ($storecfg, $vmid, $conf, $drive) = @_;
1614
1615     if (!drive_is_cdrom($drive)) {
1616         my $volid = $drive->{file};
1617         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1618             add_unused_volume($conf, $volid, $vmid);
1619         }
1620     }
1621 }
1622
1623 sub vmconfig_cleanup_pending {
1624     my ($conf) = @_;
1625
1626     # remove pending changes when nothing changed
1627     my $changes;
1628     foreach my $opt (keys %{$conf->{pending}}) {
1629         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1630             $changes = 1;
1631             delete $conf->{pending}->{$opt};
1632         }
1633     }
1634
1635     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1636     my $pending_delete_hash = {};
1637     while (my ($opt, $force) = each %$current_delete_hash) {
1638         if (defined($conf->{$opt})) {
1639             $pending_delete_hash->{$opt} = $force;
1640         } else {
1641             $changes = 1;
1642         }
1643     }
1644
1645     if (%$pending_delete_hash) {
1646         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1647     } else {
1648         delete $conf->{pending}->{delete};
1649     }
1650
1651     return $changes;
1652 }
1653
1654 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1655 my $smbios1_desc = {
1656     uuid => {
1657         type => 'string',
1658         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1659         format_description => 'UUID',
1660         optional => 1,
1661     },
1662     version => {
1663         type => 'string',
1664         pattern => '\S+',
1665         format_description => 'str',
1666         optional => 1,
1667     },
1668     serial => {
1669         type => 'string',
1670         pattern => '\S+',
1671         format_description => 'str',
1672         optional => 1,
1673     },
1674     manufacturer => {
1675         type => 'string',
1676         pattern => '\S+',
1677         format_description => 'name',
1678         optional => 1,
1679     },
1680     product => {
1681         type => 'string',
1682         pattern => '\S+',
1683         format_description => 'name',
1684         optional => 1,
1685     },
1686     sku => {
1687         type => 'string',
1688         pattern => '\S+',
1689         format_description => 'str',
1690         optional => 1,
1691     },
1692     family => {
1693         type => 'string',
1694         pattern => '\S+',
1695         format_description => 'str',
1696         optional => 1,
1697     },
1698 };
1699
1700 sub parse_smbios1 {
1701     my ($data) = @_;
1702
1703     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1704     warn $@ if $@;
1705     return $res;
1706 }
1707
1708 sub print_smbios1 {
1709     my ($smbios1) = @_;
1710     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1711 }
1712
1713 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1714
1715 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1716 sub verify_bootdisk {
1717     my ($value, $noerr) = @_;
1718
1719     return $value if valid_drivename($value);
1720
1721     return undef if $noerr;
1722
1723     die "invalid boot disk '$value'\n";
1724 }
1725
1726 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1727 sub verify_numa {
1728     my ($value, $noerr) = @_;
1729
1730     return $value if parse_numa($value);
1731
1732     return undef if $noerr;
1733
1734     die "unable to parse numa options\n";
1735 }
1736
1737 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1738 sub verify_net {
1739     my ($value, $noerr) = @_;
1740
1741     return $value if parse_net($value);
1742
1743     return undef if $noerr;
1744
1745     die "unable to parse network options\n";
1746 }
1747
1748 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1749 sub verify_hostpci {
1750     my ($value, $noerr) = @_;
1751
1752     return $value if parse_hostpci($value);
1753
1754     return undef if $noerr;
1755
1756     die "unable to parse pci id\n";
1757 }
1758
1759 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1760 sub verify_watchdog {
1761     my ($value, $noerr) = @_;
1762
1763     return $value if parse_watchdog($value);
1764
1765     return undef if $noerr;
1766
1767     die "unable to parse watchdog options\n";
1768 }
1769
1770 sub parse_watchdog {
1771     my ($value) = @_;
1772
1773     return undef if !$value;
1774
1775     my $res = {};
1776
1777     foreach my $p (split(/,/, $value)) {
1778         next if $p =~ m/^\s*$/;
1779
1780         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1781             $res->{model} = $2;
1782         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1783             $res->{action} = $2;
1784         } else {
1785             return undef;
1786         }
1787     }
1788
1789     return $res;
1790 }
1791
1792 sub parse_usb_device {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my @dl = split(/,/, $value);
1798     my $found;
1799
1800     my $res = {};
1801     foreach my $v (@dl) {
1802         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1803             $found = 1;
1804             $res->{vendorid} = $2;
1805             $res->{productid} = $4;
1806         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1807             $found = 1;
1808             $res->{hostbus} = $1;
1809             $res->{hostport} = $2;
1810         } elsif ($v =~ m/^spice$/) {
1811             $found = 1;
1812             $res->{spice} = 1;
1813         } else {
1814             return undef;
1815         }
1816     }
1817     return undef if !$found;
1818
1819     return $res;
1820 }
1821
1822 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1823 sub verify_usb_device {
1824     my ($value, $noerr) = @_;
1825
1826     return $value if parse_usb_device($value);
1827
1828     return undef if $noerr;
1829
1830     die "unable to parse usb device\n";
1831 }
1832
1833 # add JSON properties for create and set function
1834 sub json_config_properties {
1835     my $prop = shift;
1836
1837     foreach my $opt (keys %$confdesc) {
1838         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1839         $prop->{$opt} = $confdesc->{$opt};
1840     }
1841
1842     return $prop;
1843 }
1844
1845 sub check_type {
1846     my ($key, $value) = @_;
1847
1848     die "unknown setting '$key'\n" if !$confdesc->{$key};
1849
1850     my $type = $confdesc->{$key}->{type};
1851
1852     if (!defined($value)) {
1853         die "got undefined value\n";
1854     }
1855
1856     if ($value =~ m/[\n\r]/) {
1857         die "property contains a line feed\n";
1858     }
1859
1860     if ($type eq 'boolean') {
1861         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1862         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1863         die "type check ('boolean') failed - got '$value'\n";
1864     } elsif ($type eq 'integer') {
1865         return int($1) if $value =~ m/^(\d+)$/;
1866         die "type check ('integer') failed - got '$value'\n";
1867     } elsif ($type eq 'number') {
1868         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1869         die "type check ('number') failed - got '$value'\n";
1870     } elsif ($type eq 'string') {
1871         if (my $fmt = $confdesc->{$key}->{format}) {
1872             if ($fmt eq 'pve-qm-drive') {
1873                 # special case - we need to pass $key to parse_drive()
1874                 my $drive = parse_drive($key, $value);
1875                 return $value if $drive;
1876                 die "unable to parse drive options\n";
1877             }
1878             PVE::JSONSchema::check_format($fmt, $value);
1879             return $value;
1880         }
1881         $value =~ s/^\"(.*)\"$/$1/;
1882         return $value;
1883     } else {
1884         die "internal error"
1885     }
1886 }
1887
1888 sub lock_config_full {
1889     my ($vmid, $timeout, $code, @param) = @_;
1890
1891     my $filename = config_file_lock($vmid);
1892
1893     my $res = lock_file($filename, $timeout, $code, @param);
1894
1895     die $@ if $@;
1896
1897     return $res;
1898 }
1899
1900 sub lock_config_mode {
1901     my ($vmid, $timeout, $shared, $code, @param) = @_;
1902
1903     my $filename = config_file_lock($vmid);
1904
1905     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1906
1907     die $@ if $@;
1908
1909     return $res;
1910 }
1911
1912 sub lock_config {
1913     my ($vmid, $code, @param) = @_;
1914
1915     return lock_config_full($vmid, 10, $code, @param);
1916 }
1917
1918 sub cfs_config_path {
1919     my ($vmid, $node) = @_;
1920
1921     $node = $nodename if !$node;
1922     return "nodes/$node/qemu-server/$vmid.conf";
1923 }
1924
1925 sub check_iommu_support{
1926     #fixme : need to check IOMMU support
1927     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1928
1929     my $iommu=1;
1930     return $iommu;
1931
1932 }
1933
1934 sub config_file {
1935     my ($vmid, $node) = @_;
1936
1937     my $cfspath = cfs_config_path($vmid, $node);
1938     return "/etc/pve/$cfspath";
1939 }
1940
1941 sub config_file_lock {
1942     my ($vmid) = @_;
1943
1944     return "$lock_dir/lock-$vmid.conf";
1945 }
1946
1947 sub touch_config {
1948     my ($vmid) = @_;
1949
1950     my $conf = config_file($vmid);
1951     utime undef, undef, $conf;
1952 }
1953
1954 sub destroy_vm {
1955     my ($storecfg, $vmid, $keep_empty_config) = @_;
1956
1957     my $conffile = config_file($vmid);
1958
1959     my $conf = load_config($vmid);
1960
1961     check_lock($conf);
1962
1963     # only remove disks owned by this VM
1964     foreach_drive($conf, sub {
1965         my ($ds, $drive) = @_;
1966
1967         return if drive_is_cdrom($drive);
1968
1969         my $volid = $drive->{file};
1970
1971         return if !$volid || $volid =~ m|^/|;
1972
1973         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1974         return if !$path || !$owner || ($owner != $vmid);
1975
1976         PVE::Storage::vdisk_free($storecfg, $volid);
1977     });
1978
1979     if ($keep_empty_config) {
1980         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1981     } else {
1982         unlink $conffile;
1983     }
1984
1985     # also remove unused disk
1986     eval {
1987         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1988
1989         eval {
1990             PVE::Storage::foreach_volid($dl, sub {
1991                 my ($volid, $sid, $volname, $d) = @_;
1992                 PVE::Storage::vdisk_free($storecfg, $volid);
1993             });
1994         };
1995         warn $@ if $@;
1996
1997     };
1998     warn $@ if $@;
1999 }
2000
2001 sub load_config {
2002     my ($vmid, $node) = @_;
2003
2004     my $cfspath = cfs_config_path($vmid, $node);
2005
2006     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2007
2008     die "no such VM ('$vmid')\n" if !defined($conf);
2009
2010     return $conf;
2011 }
2012
2013 sub parse_vm_config {
2014     my ($filename, $raw) = @_;
2015
2016     return undef if !defined($raw);
2017
2018     my $res = {
2019         digest => Digest::SHA::sha1_hex($raw),
2020         snapshots => {},
2021         pending => {},
2022     };
2023
2024     $filename =~ m|/qemu-server/(\d+)\.conf$|
2025         || die "got strange filename '$filename'";
2026
2027     my $vmid = $1;
2028
2029     my $conf = $res;
2030     my $descr;
2031     my $section = '';
2032
2033     my @lines = split(/\n/, $raw);
2034     foreach my $line (@lines) {
2035         next if $line =~ m/^\s*$/;
2036
2037         if ($line =~ m/^\[PENDING\]\s*$/i) {
2038             $section = 'pending';
2039             if (defined($descr)) {
2040                 $descr =~ s/\s+$//;
2041                 $conf->{description} = $descr;
2042             }
2043             $descr = undef;
2044             $conf = $res->{$section} = {};
2045             next;
2046
2047         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2048             $section = $1;
2049             if (defined($descr)) {
2050                 $descr =~ s/\s+$//;
2051                 $conf->{description} = $descr;
2052             }
2053             $descr = undef;
2054             $conf = $res->{snapshots}->{$section} = {};
2055             next;
2056         }
2057
2058         if ($line =~ m/^\#(.*)\s*$/) {
2059             $descr = '' if !defined($descr);
2060             $descr .= PVE::Tools::decode_text($1) . "\n";
2061             next;
2062         }
2063
2064         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2065             $descr = '' if !defined($descr);
2066             $descr .= PVE::Tools::decode_text($2);
2067         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2068             $conf->{snapstate} = $1;
2069         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2070             my $key = $1;
2071             my $value = $2;
2072             $conf->{$key} = $value;
2073         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2074             my $value = $1;
2075             if ($section eq 'pending') {
2076                 $conf->{delete} = $value; # we parse this later
2077             } else {
2078                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2079             }
2080         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2081             my $key = $1;
2082             my $value = $2;
2083             eval { $value = check_type($key, $value); };
2084             if ($@) {
2085                 warn "vm $vmid - unable to parse value of '$key' - $@";
2086             } else {
2087                 my $fmt = $confdesc->{$key}->{format};
2088                 if ($fmt && $fmt eq 'pve-qm-drive') {
2089                     my $v = parse_drive($key, $value);
2090                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2091                         $v->{file} = $volid;
2092                         $value = print_drive($vmid, $v);
2093                     } else {
2094                         warn "vm $vmid - unable to parse value of '$key'\n";
2095                         next;
2096                     }
2097                 }
2098
2099                 if ($key eq 'cdrom') {
2100                     $conf->{ide2} = $value;
2101                 } else {
2102                     $conf->{$key} = $value;
2103                 }
2104             }
2105         }
2106     }
2107
2108     if (defined($descr)) {
2109         $descr =~ s/\s+$//;
2110         $conf->{description} = $descr;
2111     }
2112     delete $res->{snapstate}; # just to be sure
2113
2114     return $res;
2115 }
2116
2117 sub write_vm_config {
2118     my ($filename, $conf) = @_;
2119
2120     delete $conf->{snapstate}; # just to be sure
2121
2122     if ($conf->{cdrom}) {
2123         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2124         $conf->{ide2} = $conf->{cdrom};
2125         delete $conf->{cdrom};
2126     }
2127
2128     # we do not use 'smp' any longer
2129     if ($conf->{sockets}) {
2130         delete $conf->{smp};
2131     } elsif ($conf->{smp}) {
2132         $conf->{sockets} = $conf->{smp};
2133         delete $conf->{cores};
2134         delete $conf->{smp};
2135     }
2136
2137     my $used_volids = {};
2138
2139     my $cleanup_config = sub {
2140         my ($cref, $pending, $snapname) = @_;
2141
2142         foreach my $key (keys %$cref) {
2143             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2144                 $key eq 'snapstate' || $key eq 'pending';
2145             my $value = $cref->{$key};
2146             if ($key eq 'delete') {
2147                 die "propertry 'delete' is only allowed in [PENDING]\n"
2148                     if !$pending;
2149                 # fixme: check syntax?
2150                 next;
2151             }
2152             eval { $value = check_type($key, $value); };
2153             die "unable to parse value of '$key' - $@" if $@;
2154
2155             $cref->{$key} = $value;
2156
2157             if (!$snapname && valid_drivename($key)) {
2158                 my $drive = parse_drive($key, $value);
2159                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2160             }
2161         }
2162     };
2163
2164     &$cleanup_config($conf);
2165
2166     &$cleanup_config($conf->{pending}, 1);
2167
2168     foreach my $snapname (keys %{$conf->{snapshots}}) {
2169         die "internal error" if $snapname eq 'pending';
2170         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2171     }
2172
2173     # remove 'unusedX' settings if we re-add a volume
2174     foreach my $key (keys %$conf) {
2175         my $value = $conf->{$key};
2176         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2177             delete $conf->{$key};
2178         }
2179     }
2180
2181     my $generate_raw_config = sub {
2182         my ($conf, $pending) = @_;
2183
2184         my $raw = '';
2185
2186         # add description as comment to top of file
2187         if (defined(my $descr = $conf->{description})) {
2188             if ($descr) {
2189                 foreach my $cl (split(/\n/, $descr)) {
2190                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2191                 }
2192             } else {
2193                 $raw .= "#\n" if $pending;
2194             }
2195         }
2196
2197         foreach my $key (sort keys %$conf) {
2198             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2199             $raw .= "$key: $conf->{$key}\n";
2200         }
2201         return $raw;
2202     };
2203
2204     my $raw = &$generate_raw_config($conf);
2205
2206     if (scalar(keys %{$conf->{pending}})){
2207         $raw .= "\n[PENDING]\n";
2208         $raw .= &$generate_raw_config($conf->{pending}, 1);
2209     }
2210
2211     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2212         $raw .= "\n[$snapname]\n";
2213         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2214     }
2215
2216     return $raw;
2217 }
2218
2219 sub update_config_nolock {
2220     my ($vmid, $conf, $skiplock) = @_;
2221
2222     check_lock($conf) if !$skiplock;
2223
2224     my $cfspath = cfs_config_path($vmid);
2225
2226     PVE::Cluster::cfs_write_file($cfspath, $conf);
2227 }
2228
2229 sub update_config {
2230     my ($vmid, $conf, $skiplock) = @_;
2231
2232     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2233 }
2234
2235 sub load_defaults {
2236
2237     my $res = {};
2238
2239     # we use static defaults from our JSON schema configuration
2240     foreach my $key (keys %$confdesc) {
2241         if (defined(my $default = $confdesc->{$key}->{default})) {
2242             $res->{$key} = $default;
2243         }
2244     }
2245
2246     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2247     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2248
2249     return $res;
2250 }
2251
2252 sub config_list {
2253     my $vmlist = PVE::Cluster::get_vmlist();
2254     my $res = {};
2255     return $res if !$vmlist || !$vmlist->{ids};
2256     my $ids = $vmlist->{ids};
2257
2258     foreach my $vmid (keys %$ids) {
2259         my $d = $ids->{$vmid};
2260         next if !$d->{node} || $d->{node} ne $nodename;
2261         next if !$d->{type} || $d->{type} ne 'qemu';
2262         $res->{$vmid}->{exists} = 1;
2263     }
2264     return $res;
2265 }
2266
2267 # test if VM uses local resources (to prevent migration)
2268 sub check_local_resources {
2269     my ($conf, $noerr) = @_;
2270
2271     my $loc_res = 0;
2272
2273     $loc_res = 1 if $conf->{hostusb}; # old syntax
2274     $loc_res = 1 if $conf->{hostpci}; # old syntax
2275
2276     foreach my $k (keys %$conf) {
2277         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2278         # sockets are safe: they will recreated be on the target side post-migrate
2279         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2280         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2281     }
2282
2283     die "VM uses local resources\n" if $loc_res && !$noerr;
2284
2285     return $loc_res;
2286 }
2287
2288 # check if used storages are available on all nodes (use by migrate)
2289 sub check_storage_availability {
2290     my ($storecfg, $conf, $node) = @_;
2291
2292     foreach_drive($conf, sub {
2293         my ($ds, $drive) = @_;
2294
2295         my $volid = $drive->{file};
2296         return if !$volid;
2297
2298         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2299         return if !$sid;
2300
2301         # check if storage is available on both nodes
2302         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2303         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2304    });
2305 }
2306
2307 # list nodes where all VM images are available (used by has_feature API)
2308 sub shared_nodes {
2309     my ($conf, $storecfg) = @_;
2310
2311     my $nodelist = PVE::Cluster::get_nodelist();
2312     my $nodehash = { map { $_ => 1 } @$nodelist };
2313     my $nodename = PVE::INotify::nodename();
2314
2315     foreach_drive($conf, sub {
2316         my ($ds, $drive) = @_;
2317
2318         my $volid = $drive->{file};
2319         return if !$volid;
2320
2321         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2322         if ($storeid) {
2323             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2324             if ($scfg->{disable}) {
2325                 $nodehash = {};
2326             } elsif (my $avail = $scfg->{nodes}) {
2327                 foreach my $node (keys %$nodehash) {
2328                     delete $nodehash->{$node} if !$avail->{$node};
2329                 }
2330             } elsif (!$scfg->{shared}) {
2331                 foreach my $node (keys %$nodehash) {
2332                     delete $nodehash->{$node} if $node ne $nodename
2333                 }
2334             }
2335         }
2336     });
2337
2338     return $nodehash
2339 }
2340
2341 sub check_lock {
2342     my ($conf) = @_;
2343
2344     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2345 }
2346
2347 sub check_cmdline {
2348     my ($pidfile, $pid) = @_;
2349
2350     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2351     if (defined($fh)) {
2352         my $line = <$fh>;
2353         $fh->close;
2354         return undef if !$line;
2355         my @param = split(/\0/, $line);
2356
2357         my $cmd = $param[0];
2358         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2359
2360         for (my $i = 0; $i < scalar (@param); $i++) {
2361             my $p = $param[$i];
2362             next if !$p;
2363             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2364                 my $p = $param[$i+1];
2365                 return 1 if $p && ($p eq $pidfile);
2366                 return undef;
2367             }
2368         }
2369     }
2370     return undef;
2371 }
2372
2373 sub check_running {
2374     my ($vmid, $nocheck, $node) = @_;
2375
2376     my $filename = config_file($vmid, $node);
2377
2378     die "unable to find configuration file for VM $vmid - no such machine\n"
2379         if !$nocheck && ! -f $filename;
2380
2381     my $pidfile = pidfile_name($vmid);
2382
2383     if (my $fd = IO::File->new("<$pidfile")) {
2384         my $st = stat($fd);
2385         my $line = <$fd>;
2386         close($fd);
2387
2388         my $mtime = $st->mtime;
2389         if ($mtime > time()) {
2390             warn "file '$filename' modified in future\n";
2391         }
2392
2393         if ($line =~ m/^(\d+)$/) {
2394             my $pid = $1;
2395             if (check_cmdline($pidfile, $pid)) {
2396                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2397                     return $pid;
2398                 }
2399             }
2400         }
2401     }
2402
2403     return undef;
2404 }
2405
2406 sub vzlist {
2407
2408     my $vzlist = config_list();
2409
2410     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2411
2412     while (defined(my $de = $fd->read)) {
2413         next if $de !~ m/^(\d+)\.pid$/;
2414         my $vmid = $1;
2415         next if !defined($vzlist->{$vmid});
2416         if (my $pid = check_running($vmid)) {
2417             $vzlist->{$vmid}->{pid} = $pid;
2418         }
2419     }
2420
2421     return $vzlist;
2422 }
2423
2424 sub disksize {
2425     my ($storecfg, $conf) = @_;
2426
2427     my $bootdisk = $conf->{bootdisk};
2428     return undef if !$bootdisk;
2429     return undef if !valid_drivename($bootdisk);
2430
2431     return undef if !$conf->{$bootdisk};
2432
2433     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2434     return undef if !defined($drive);
2435
2436     return undef if drive_is_cdrom($drive);
2437
2438     my $volid = $drive->{file};
2439     return undef if !$volid;
2440
2441     return $drive->{size};
2442 }
2443
2444 my $last_proc_pid_stat;
2445
2446 # get VM status information
2447 # This must be fast and should not block ($full == false)
2448 # We only query KVM using QMP if $full == true (this can be slow)
2449 sub vmstatus {
2450     my ($opt_vmid, $full) = @_;
2451
2452     my $res = {};
2453
2454     my $storecfg = PVE::Storage::config();
2455
2456     my $list = vzlist();
2457     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2458
2459     my $cpucount = $cpuinfo->{cpus} || 1;
2460
2461     foreach my $vmid (keys %$list) {
2462         next if $opt_vmid && ($vmid ne $opt_vmid);
2463
2464         my $cfspath = cfs_config_path($vmid);
2465         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2466
2467         my $d = {};
2468         $d->{pid} = $list->{$vmid}->{pid};
2469
2470         # fixme: better status?
2471         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2472
2473         my $size = disksize($storecfg, $conf);
2474         if (defined($size)) {
2475             $d->{disk} = 0; # no info available
2476             $d->{maxdisk} = $size;
2477         } else {
2478             $d->{disk} = 0;
2479             $d->{maxdisk} = 0;
2480         }
2481
2482         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2483         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2484         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2485
2486         $d->{name} = $conf->{name} || "VM $vmid";
2487         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2488
2489         if ($conf->{balloon}) {
2490             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2491             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2492         }
2493
2494         $d->{uptime} = 0;
2495         $d->{cpu} = 0;
2496         $d->{mem} = 0;
2497
2498         $d->{netout} = 0;
2499         $d->{netin} = 0;
2500
2501         $d->{diskread} = 0;
2502         $d->{diskwrite} = 0;
2503
2504         $d->{template} = is_template($conf);
2505
2506         $res->{$vmid} = $d;
2507     }
2508
2509     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2510     foreach my $dev (keys %$netdev) {
2511         next if $dev !~ m/^tap([1-9]\d*)i/;
2512         my $vmid = $1;
2513         my $d = $res->{$vmid};
2514         next if !$d;
2515
2516         $d->{netout} += $netdev->{$dev}->{receive};
2517         $d->{netin} += $netdev->{$dev}->{transmit};
2518
2519         if ($full) {
2520             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2521             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2522         }
2523
2524     }
2525
2526     my $ctime = gettimeofday;
2527
2528     foreach my $vmid (keys %$list) {
2529
2530         my $d = $res->{$vmid};
2531         my $pid = $d->{pid};
2532         next if !$pid;
2533
2534         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2535         next if !$pstat; # not running
2536
2537         my $used = $pstat->{utime} + $pstat->{stime};
2538
2539         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2540
2541         if ($pstat->{vsize}) {
2542             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2543         }
2544
2545         my $old = $last_proc_pid_stat->{$pid};
2546         if (!$old) {
2547             $last_proc_pid_stat->{$pid} = {
2548                 time => $ctime,
2549                 used => $used,
2550                 cpu => 0,
2551             };
2552             next;
2553         }
2554
2555         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2556
2557         if ($dtime > 1000) {
2558             my $dutime = $used -  $old->{used};
2559
2560             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2561             $last_proc_pid_stat->{$pid} = {
2562                 time => $ctime,
2563                 used => $used,
2564                 cpu => $d->{cpu},
2565             };
2566         } else {
2567             $d->{cpu} = $old->{cpu};
2568         }
2569     }
2570
2571     return $res if !$full;
2572
2573     my $qmpclient = PVE::QMPClient->new();
2574
2575     my $ballooncb = sub {
2576         my ($vmid, $resp) = @_;
2577
2578         my $info = $resp->{'return'};
2579         return if !$info->{max_mem};
2580
2581         my $d = $res->{$vmid};
2582
2583         # use memory assigned to VM
2584         $d->{maxmem} = $info->{max_mem};
2585         $d->{balloon} = $info->{actual};
2586
2587         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2588             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2589             $d->{freemem} = $info->{free_mem};
2590         }
2591
2592         $d->{ballooninfo} = $info;
2593     };
2594
2595     my $blockstatscb = sub {
2596         my ($vmid, $resp) = @_;
2597         my $data = $resp->{'return'} || [];
2598         my $totalrdbytes = 0;
2599         my $totalwrbytes = 0;
2600
2601         for my $blockstat (@$data) {
2602             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2603             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2604
2605             $blockstat->{device} =~ s/drive-//;
2606             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2607         }
2608         $res->{$vmid}->{diskread} = $totalrdbytes;
2609         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2610     };
2611
2612     my $statuscb = sub {
2613         my ($vmid, $resp) = @_;
2614
2615         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2616         # this fails if ballon driver is not loaded, so this must be
2617         # the last commnand (following command are aborted if this fails).
2618         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2619
2620         my $status = 'unknown';
2621         if (!defined($status = $resp->{'return'}->{status})) {
2622             warn "unable to get VM status\n";
2623             return;
2624         }
2625
2626         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2627     };
2628
2629     foreach my $vmid (keys %$list) {
2630         next if $opt_vmid && ($vmid ne $opt_vmid);
2631         next if !$res->{$vmid}->{pid}; # not running
2632         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2633     }
2634
2635     $qmpclient->queue_execute(undef, 1);
2636
2637     foreach my $vmid (keys %$list) {
2638         next if $opt_vmid && ($vmid ne $opt_vmid);
2639         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2640     }
2641
2642     return $res;
2643 }
2644
2645 sub foreach_dimm {
2646     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2647
2648     my $dimm_id = 0;
2649     my $current_size = 1024;
2650     my $dimm_size = 512;
2651     return if $current_size == $memory;
2652
2653     for (my $j = 0; $j < 8; $j++) {
2654         for (my $i = 0; $i < 32; $i++) {
2655             my $name = "dimm${dimm_id}";
2656             $dimm_id++;
2657             my $numanode = $i % $sockets;
2658             $current_size += $dimm_size;
2659             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2660             return  $current_size if $current_size >= $memory;
2661         }
2662         $dimm_size *= 2;
2663     }
2664 }
2665
2666 sub foreach_reverse_dimm {
2667     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2668
2669     my $dimm_id = 253;
2670     my $current_size = 4177920;
2671     my $dimm_size = 65536;
2672     return if $current_size == $memory;
2673
2674     for (my $j = 0; $j < 8; $j++) {
2675         for (my $i = 0; $i < 32; $i++) {
2676             my $name = "dimm${dimm_id}";
2677             $dimm_id--;
2678             my $numanode = $i % $sockets;
2679             $current_size -= $dimm_size;
2680             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2681             return  $current_size if $current_size <= $memory;
2682         }
2683         $dimm_size /= 2;
2684     }
2685 }
2686
2687 sub foreach_drive {
2688     my ($conf, $func) = @_;
2689
2690     foreach my $ds (keys %$conf) {
2691         next if !valid_drivename($ds);
2692
2693         my $drive = parse_drive($ds, $conf->{$ds});
2694         next if !$drive;
2695
2696         &$func($ds, $drive);
2697     }
2698 }
2699
2700 sub foreach_volid {
2701     my ($conf, $func) = @_;
2702
2703     my $volhash = {};
2704
2705     my $test_volid = sub {
2706         my ($volid, $is_cdrom) = @_;
2707
2708         return if !$volid;
2709
2710         $volhash->{$volid} = $is_cdrom || 0;
2711     };
2712
2713     foreach_drive($conf, sub {
2714         my ($ds, $drive) = @_;
2715         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2716     });
2717
2718     foreach my $snapname (keys %{$conf->{snapshots}}) {
2719         my $snap = $conf->{snapshots}->{$snapname};
2720         &$test_volid($snap->{vmstate}, 0);
2721         foreach_drive($snap, sub {
2722             my ($ds, $drive) = @_;
2723             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2724         });
2725     }
2726
2727     foreach my $volid (keys %$volhash) {
2728         &$func($volid, $volhash->{$volid});
2729     }
2730 }
2731
2732 sub vga_conf_has_spice {
2733     my ($vga) = @_;
2734
2735     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2736
2737     return $1 || 1;
2738 }
2739
2740 sub config_to_command {
2741     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2742
2743     my $cmd = [];
2744     my $globalFlags = [];
2745     my $machineFlags = [];
2746     my $rtcFlags = [];
2747     my $cpuFlags = [];
2748     my $devices = [];
2749     my $pciaddr = '';
2750     my $bridges = {};
2751     my $kvmver = kvm_user_version();
2752     my $vernum = 0; # unknown
2753     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2754         $vernum = $1*1000000+$2*1000;
2755     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2756         $vernum = $1*1000000+$2*1000+$3;
2757     }
2758
2759     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2760
2761     my $have_ovz = -f '/proc/vz/vestat';
2762
2763     my $q35 = machine_type_is_q35($conf);
2764     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2765     my $machine_type = $forcemachine || $conf->{machine};
2766     my $use_old_bios_files = undef;
2767     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2768
2769     my $cpuunits = defined($conf->{cpuunits}) ?
2770             $conf->{cpuunits} : $defaults->{cpuunits};
2771
2772     push @$cmd, '/usr/bin/systemd-run';
2773     push @$cmd, '--scope';
2774     push @$cmd, '--slice', "qemu";
2775     push @$cmd, '--unit', $vmid;
2776     # set KillMode=none, so that systemd don't kill those scopes
2777     # at shutdown (pve-manager service should stop the VMs instead)
2778     push @$cmd, '-p', "KillMode=none";
2779     push @$cmd, '-p', "CPUShares=$cpuunits";
2780     if ($conf->{cpulimit}) {
2781         my $cpulimit = int($conf->{cpulimit} * 100);
2782         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2783     }
2784
2785     push @$cmd, '/usr/bin/kvm';
2786
2787     push @$cmd, '-id', $vmid;
2788
2789     my $use_virtio = 0;
2790
2791     my $qmpsocket = qmp_socket($vmid);
2792     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2793     push @$cmd, '-mon', "chardev=qmp,mode=control";
2794
2795     my $socket = vnc_socket($vmid);
2796     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2797
2798     push @$cmd, '-pidfile' , pidfile_name($vmid);
2799
2800     push @$cmd, '-daemonize';
2801
2802     if ($conf->{smbios1}) {
2803         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2804     }
2805
2806     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2807         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2808         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2809         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2810         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2811         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2812         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2813     }
2814
2815     if ($q35) {
2816         # the q35 chipset support native usb2, so we enable usb controller
2817         # by default for this machine type
2818         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2819     } else {
2820         $pciaddr = print_pci_addr("piix3", $bridges);
2821         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2822
2823         my $use_usb2 = 0;
2824         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2825             next if !$conf->{"usb$i"};
2826             $use_usb2 = 1;
2827         }
2828         # include usb device config
2829         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2830     }
2831
2832     my $vga = $conf->{vga};
2833
2834     my $qxlnum = vga_conf_has_spice($vga);
2835     $vga = 'qxl' if $qxlnum;
2836
2837     if (!$vga) {
2838         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2839                                 $conf->{ostype} eq 'win7' ||
2840                                 $conf->{ostype} eq 'w2k8')) {
2841             $vga = 'std';
2842         } else {
2843             $vga = 'cirrus';
2844         }
2845     }
2846
2847     # enable absolute mouse coordinates (needed by vnc)
2848     my $tablet;
2849     if (defined($conf->{tablet})) {
2850         $tablet = $conf->{tablet};
2851     } else {
2852         $tablet = $defaults->{tablet};
2853         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2854         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2855     }
2856
2857     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2858
2859     my $nohyperv;
2860     # host pci devices
2861     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2862         my $d = parse_hostpci($conf->{"hostpci$i"});
2863         next if !$d;
2864
2865         my $pcie = $d->{pcie};
2866         if($pcie){
2867             die "q35 machine model is not enabled" if !$q35;
2868             $pciaddr = print_pcie_addr("hostpci$i");
2869         }else{
2870             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2871         }
2872
2873         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2874         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2875         if ($xvga && $xvga ne '') {
2876             push @$cpuFlags, 'kvm=off';
2877             $vga = 'none';
2878             $nohyperv = 1;
2879             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2880                 $xvga = "";
2881             }
2882         }
2883         my $pcidevices = $d->{pciid};
2884         my $multifunction = 1 if @$pcidevices > 1;
2885
2886         my $j=0;
2887         foreach my $pcidevice (@$pcidevices) {
2888
2889             my $id = "hostpci$i";
2890             $id .= ".$j" if $multifunction;
2891             my $addr = $pciaddr;
2892             $addr .= ".$j" if $multifunction;
2893             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2894
2895             if($j == 0){
2896                 $devicestr .= "$rombar$xvga";
2897                 $devicestr .= ",multifunction=on" if $multifunction;
2898             }
2899
2900             push @$devices, '-device', $devicestr;
2901             $j++;
2902         }
2903     }
2904
2905     # usb devices
2906     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2907         my $d = parse_usb_device($conf->{"usb$i"});
2908         next if !$d;
2909         if ($d->{vendorid} && $d->{productid}) {
2910             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2911         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2912             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2913         } elsif ($d->{spice}) {
2914             # usb redir support for spice
2915             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2916             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2917         }
2918     }
2919
2920     # serial devices
2921     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2922         if (my $path = $conf->{"serial$i"}) {
2923             if ($path eq 'socket') {
2924                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2925                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2926                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2927             } else {
2928                 die "no such serial device\n" if ! -c $path;
2929                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2930                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2931             }
2932         }
2933     }
2934
2935     # parallel devices
2936     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2937         if (my $path = $conf->{"parallel$i"}) {
2938             die "no such parallel device\n" if ! -c $path;
2939             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2940             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2941             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2942         }
2943     }
2944
2945     my $vmname = $conf->{name} || "vm$vmid";
2946
2947     push @$cmd, '-name', $vmname;
2948
2949     my $sockets = 1;
2950     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2951     $sockets = $conf->{sockets} if  $conf->{sockets};
2952
2953     my $cores = $conf->{cores} || 1;
2954
2955     my $maxcpus = $sockets * $cores;
2956
2957     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2958
2959     my $allowed_vcpus = $cpuinfo->{cpus};
2960
2961     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2962         if ($allowed_vcpus < $maxcpus);
2963
2964     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2965
2966     push @$cmd, '-nodefaults';
2967
2968     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2969
2970     my $bootindex_hash = {};
2971     my $i = 1;
2972     foreach my $o (split(//, $bootorder)) {
2973         $bootindex_hash->{$o} = $i*100;
2974         $i++;
2975     }
2976
2977     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2978
2979     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2980
2981     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2982
2983     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2984
2985     # time drift fix
2986     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2987
2988     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2989     my $useLocaltime = $conf->{localtime};
2990
2991     if (my $ost = $conf->{ostype}) {
2992         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2993
2994         if ($ost =~ m/^w/) { # windows
2995             $useLocaltime = 1 if !defined($conf->{localtime});
2996
2997             # use time drift fix when acpi is enabled
2998             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2999                 $tdf = 1 if !defined($conf->{tdf});
3000             }
3001         }
3002
3003         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
3004             $ost eq 'wvista') {
3005             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3006             push @$cmd, '-no-hpet';
3007             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3008                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
3009                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
3010                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
3011
3012             } else {
3013                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
3014             }
3015         }
3016
3017         if ($ost eq 'win7' || $ost eq 'win8') {
3018             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
3019         }
3020     }
3021
3022     push @$rtcFlags, 'driftfix=slew' if $tdf;
3023
3024     if ($nokvm) {
3025         push @$machineFlags, 'accel=tcg';
3026     } else {
3027         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3028     }
3029
3030     if ($machine_type) {
3031         push @$machineFlags, "type=${machine_type}";
3032     }
3033
3034     if ($conf->{startdate}) {
3035         push @$rtcFlags, "base=$conf->{startdate}";
3036     } elsif ($useLocaltime) {
3037         push @$rtcFlags, 'base=localtime';
3038     }
3039
3040     my $cpu = $nokvm ? "qemu64" : "kvm64";
3041     $cpu = $conf->{cpu} if $conf->{cpu};
3042
3043     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3044
3045     push @$cpuFlags , '-x2apic'
3046         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3047
3048     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3049
3050     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3051
3052     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3053
3054         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3055         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3056     }
3057
3058     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3059
3060     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3061
3062     push @$cmd, '-cpu', $cpu;
3063
3064     my $memory = $conf->{memory} || $defaults->{memory};
3065     my $static_memory = 0;
3066     my $dimm_memory = 0;
3067
3068     if ($hotplug_features->{memory}) {
3069         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3070         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3071         $static_memory = $STATICMEM;
3072         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3073         $dimm_memory = $memory - $static_memory;
3074         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3075
3076     } else {
3077
3078         $static_memory = $memory;
3079         push @$cmd, '-m', $static_memory;
3080     }
3081
3082     if ($conf->{numa}) {
3083
3084         my $numa_totalmemory = undef;
3085         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3086             next if !$conf->{"numa$i"};
3087             my $numa = parse_numa($conf->{"numa$i"});
3088             next if !$numa;
3089             # memory
3090             die "missing numa node$i memory value\n" if !$numa->{memory};
3091             my $numa_memory = $numa->{memory};
3092             $numa_totalmemory += $numa_memory;
3093             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3094
3095             # cpus
3096             my $cpus_start = $numa->{cpus}->{start};
3097             die "missing numa node$i cpus\n" if !defined($cpus_start);
3098             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3099             my $cpus = $cpus_start;
3100             if (defined($cpus_end)) {
3101                 $cpus .= "-$cpus_end";
3102                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3103             }
3104
3105             # hostnodes
3106             my $hostnodes_start = $numa->{hostnodes}->{start};
3107             if (defined($hostnodes_start)) {
3108                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3109                 my $hostnodes = $hostnodes_start;
3110                 if (defined($hostnodes_end)) {
3111                     $hostnodes .= "-$hostnodes_end";
3112                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3113                 }
3114
3115                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3116                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3117                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3118                 }
3119
3120                 # policy
3121                 my $policy = $numa->{policy};
3122                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3123                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3124             }
3125
3126             push @$cmd, '-object', $numa_object;
3127             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3128         }
3129
3130         die "total memory for NUMA nodes must be equal to vm static memory\n"
3131             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3132
3133         #if no custom tology, we split memory and cores across numa nodes
3134         if(!$numa_totalmemory) {
3135
3136             my $numa_memory = ($static_memory / $sockets) . "M";
3137
3138             for (my $i = 0; $i < $sockets; $i++)  {
3139
3140                 my $cpustart = ($cores * $i);
3141                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3142                 my $cpus = $cpustart;
3143                 $cpus .= "-$cpuend" if $cpuend;
3144
3145                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3146                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3147             }
3148         }
3149     }
3150
3151     if ($hotplug_features->{memory}) {
3152         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3153             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3154             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3155             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3156
3157             #if dimm_memory is not aligned to dimm map
3158             if($current_size > $memory) {
3159                  $conf->{memory} = $current_size;
3160                  update_config_nolock($vmid, $conf, 1);
3161             }
3162         });
3163     }
3164
3165     push @$cmd, '-S' if $conf->{freeze};
3166
3167     # set keyboard layout
3168     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3169     push @$cmd, '-k', $kb if $kb;
3170
3171     # enable sound
3172     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3173     #push @$cmd, '-soundhw', 'es1370';
3174     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3175
3176     if($conf->{agent}) {
3177         my $qgasocket = qmp_socket($vmid, 1);
3178         my $pciaddr = print_pci_addr("qga0", $bridges);
3179         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3180         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3181         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3182     }
3183
3184     my $spice_port;
3185
3186     if ($qxlnum) {
3187         if ($qxlnum > 1) {
3188             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3189                 for(my $i = 1; $i < $qxlnum; $i++){
3190                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3191                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3192                 }
3193             } else {
3194                 # assume other OS works like Linux
3195                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3196                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3197             }
3198         }
3199
3200         my $pciaddr = print_pci_addr("spice", $bridges);
3201
3202         my $nodename = PVE::INotify::nodename();
3203         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3204         $spice_port = PVE::Tools::next_spice_port($pfamily);
3205
3206         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3207
3208         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3209         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3210         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3211     }
3212
3213     # enable balloon by default, unless explicitly disabled
3214     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3215         $pciaddr = print_pci_addr("balloon0", $bridges);
3216         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3217     }
3218
3219     if ($conf->{watchdog}) {
3220         my $wdopts = parse_watchdog($conf->{watchdog});
3221         $pciaddr = print_pci_addr("watchdog", $bridges);
3222         my $watchdog = $wdopts->{model} || 'i6300esb';
3223         push @$devices, '-device', "$watchdog$pciaddr";
3224         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3225     }
3226
3227     my $vollist = [];
3228     my $scsicontroller = {};
3229     my $ahcicontroller = {};
3230     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3231
3232     # Add iscsi initiator name if available
3233     if (my $initiator = get_initiator_name()) {
3234         push @$devices, '-iscsi', "initiator-name=$initiator";
3235     }
3236
3237     foreach_drive($conf, sub {
3238         my ($ds, $drive) = @_;
3239
3240         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3241             push @$vollist, $drive->{file};
3242         }
3243
3244         $use_virtio = 1 if $ds =~ m/^virtio/;
3245
3246         if (drive_is_cdrom ($drive)) {
3247             if ($bootindex_hash->{d}) {
3248                 $drive->{bootindex} = $bootindex_hash->{d};
3249                 $bootindex_hash->{d} += 1;
3250             }
3251         } else {
3252             if ($bootindex_hash->{c}) {
3253                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3254                 $bootindex_hash->{c} += 1;
3255             }
3256         }
3257
3258         if($drive->{interface} eq 'virtio'){
3259            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3260         }
3261
3262         if ($drive->{interface} eq 'scsi') {
3263
3264             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3265
3266             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3267             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3268
3269             my $iothread = '';
3270             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3271                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3272                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3273             }
3274
3275             my $queues = '';
3276             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3277                 $queues = ",num_queues=$drive->{queues}";
3278             }
3279
3280             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3281             $scsicontroller->{$controller}=1;
3282         }
3283
3284         if ($drive->{interface} eq 'sata') {
3285            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3286            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3287            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3288            $ahcicontroller->{$controller}=1;
3289         }
3290
3291         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3292         push @$devices, '-drive',$drive_cmd;
3293         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3294     });
3295
3296     for (my $i = 0; $i < $MAX_NETS; $i++) {
3297          next if !$conf->{"net$i"};
3298          my $d = parse_net($conf->{"net$i"});
3299          next if !$d;
3300
3301          $use_virtio = 1 if $d->{model} eq 'virtio';
3302
3303          if ($bootindex_hash->{n}) {
3304             $d->{bootindex} = $bootindex_hash->{n};
3305             $bootindex_hash->{n} += 1;
3306          }
3307
3308          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3309          push @$devices, '-netdev', $netdevfull;
3310
3311          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3312          push @$devices, '-device', $netdevicefull;
3313     }
3314
3315     if (!$q35) {
3316         # add pci bridges
3317         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3318            $bridges->{1} = 1;
3319            $bridges->{2} = 1;
3320         }
3321
3322         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3323
3324         while (my ($k, $v) = each %$bridges) {
3325             $pciaddr = print_pci_addr("pci.$k");
3326             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3327         }
3328     }
3329
3330     # add custom args
3331     if ($conf->{args}) {
3332         my $aa = PVE::Tools::split_args($conf->{args});
3333         push @$cmd, @$aa;
3334     }
3335
3336     push @$cmd, @$devices;
3337     push @$cmd, '-rtc', join(',', @$rtcFlags)
3338         if scalar(@$rtcFlags);
3339     push @$cmd, '-machine', join(',', @$machineFlags)
3340         if scalar(@$machineFlags);
3341     push @$cmd, '-global', join(',', @$globalFlags)
3342         if scalar(@$globalFlags);
3343
3344     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3345 }
3346
3347 sub vnc_socket {
3348     my ($vmid) = @_;
3349     return "${var_run_tmpdir}/$vmid.vnc";
3350 }
3351
3352 sub spice_port {
3353     my ($vmid) = @_;
3354
3355     my $res = vm_mon_cmd($vmid, 'query-spice');
3356
3357     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3358 }
3359
3360 sub qmp_socket {
3361     my ($vmid, $qga) = @_;
3362     my $sockettype = $qga ? 'qga' : 'qmp';
3363     return "${var_run_tmpdir}/$vmid.$sockettype";
3364 }
3365
3366 sub pidfile_name {
3367     my ($vmid) = @_;
3368     return "${var_run_tmpdir}/$vmid.pid";
3369 }
3370
3371 sub vm_devices_list {
3372     my ($vmid) = @_;
3373
3374     my $res = vm_mon_cmd($vmid, 'query-pci');
3375     my $devices = {};
3376     foreach my $pcibus (@$res) {
3377         foreach my $device (@{$pcibus->{devices}}) {
3378             next if !$device->{'qdev_id'};
3379             if ($device->{'pci_bridge'}) {
3380                 $devices->{$device->{'qdev_id'}} = 1;
3381                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3382                     next if !$bridge_device->{'qdev_id'};
3383                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3384                     $devices->{$device->{'qdev_id'}}++;
3385                 }
3386             } else {
3387                 $devices->{$device->{'qdev_id'}} = 1;
3388             }
3389         }
3390     }
3391
3392     my $resblock = vm_mon_cmd($vmid, 'query-block');
3393     foreach my $block (@$resblock) {
3394         if($block->{device} =~ m/^drive-(\S+)/){
3395                 $devices->{$1} = 1;
3396         }
3397     }
3398
3399     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3400     foreach my $mice (@$resmice) {
3401         if ($mice->{name} eq 'QEMU HID Tablet') {
3402             $devices->{tablet} = 1;
3403             last;
3404         }
3405     }
3406
3407     return $devices;
3408 }
3409
3410 sub vm_deviceplug {
3411     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3412
3413     my $q35 = machine_type_is_q35($conf);
3414
3415     my $devices_list = vm_devices_list($vmid);
3416     return 1 if defined($devices_list->{$deviceid});
3417
3418     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3419
3420     if ($deviceid eq 'tablet') {
3421
3422         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3423
3424     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3425
3426         qemu_iothread_add($vmid, $deviceid, $device);
3427
3428         qemu_driveadd($storecfg, $vmid, $device);
3429         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3430
3431         qemu_deviceadd($vmid, $devicefull);
3432         eval { qemu_deviceaddverify($vmid, $deviceid); };
3433         if (my $err = $@) {
3434             eval { qemu_drivedel($vmid, $deviceid); };
3435             warn $@ if $@;
3436             die $err;
3437         }
3438
3439     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3440
3441
3442         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3443         my $pciaddr = print_pci_addr($deviceid);
3444         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3445
3446         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3447
3448         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3449             qemu_iothread_add($vmid, $deviceid, $device);
3450             $devicefull .= ",iothread=iothread-$deviceid";
3451         }
3452
3453         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3454             $devicefull .= ",num_queues=$device->{queues}";
3455         }
3456
3457         qemu_deviceadd($vmid, $devicefull);
3458         qemu_deviceaddverify($vmid, $deviceid);
3459
3460     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3461
3462         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3463         qemu_driveadd($storecfg, $vmid, $device);
3464
3465         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3466         eval { qemu_deviceadd($vmid, $devicefull); };
3467         if (my $err = $@) {
3468             eval { qemu_drivedel($vmid, $deviceid); };
3469             warn $@ if $@;
3470             die $err;
3471         }
3472
3473     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3474
3475         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3476
3477         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3478         my $use_old_bios_files = undef;
3479         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3480
3481         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3482         qemu_deviceadd($vmid, $netdevicefull);
3483         eval { qemu_deviceaddverify($vmid, $deviceid); };
3484         if (my $err = $@) {
3485             eval { qemu_netdevdel($vmid, $deviceid); };
3486             warn $@ if $@;
3487             die $err;
3488         }
3489
3490     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3491
3492         my $bridgeid = $2;
3493         my $pciaddr = print_pci_addr($deviceid);
3494         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3495
3496         qemu_deviceadd($vmid, $devicefull);
3497         qemu_deviceaddverify($vmid, $deviceid);
3498
3499     } else {
3500         die "can't hotplug device '$deviceid'\n";
3501     }
3502
3503     return 1;
3504 }
3505
3506 # fixme: this should raise exceptions on error!
3507 sub vm_deviceunplug {
3508     my ($vmid, $conf, $deviceid) = @_;
3509
3510     my $devices_list = vm_devices_list($vmid);
3511     return 1 if !defined($devices_list->{$deviceid});
3512
3513     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3514
3515     if ($deviceid eq 'tablet') {
3516
3517         qemu_devicedel($vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_devicedelverify($vmid, $deviceid);
3523         qemu_drivedel($vmid, $deviceid);
3524         qemu_iothread_del($conf, $vmid, $deviceid);
3525
3526     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3527
3528         qemu_devicedel($vmid, $deviceid);
3529         qemu_devicedelverify($vmid, $deviceid);
3530         qemu_iothread_del($conf, $vmid, $deviceid);
3531
3532     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3533
3534         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3535         my $device = parse_drive($deviceid, $conf->{$deviceid});
3536         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3537
3538         qemu_devicedel($vmid, $deviceid);
3539         qemu_drivedel($vmid, $deviceid);
3540         qemu_deletescsihw($conf, $vmid, $deviceid);
3541
3542     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3543
3544         qemu_devicedel($vmid, $deviceid);
3545         qemu_devicedelverify($vmid, $deviceid);
3546         qemu_netdevdel($vmid, $deviceid);
3547
3548     } else {
3549         die "can't unplug device '$deviceid'\n";
3550     }
3551
3552     return 1;
3553 }
3554
3555 sub qemu_deviceadd {
3556     my ($vmid, $devicefull) = @_;
3557
3558     $devicefull = "driver=".$devicefull;
3559     my %options =  split(/[=,]/, $devicefull);
3560
3561     vm_mon_cmd($vmid, "device_add" , %options);
3562 }
3563
3564 sub qemu_devicedel {
3565     my ($vmid, $deviceid) = @_;
3566
3567     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3568 }
3569
3570 sub qemu_iothread_add {
3571     my($vmid, $deviceid, $device) = @_;
3572
3573     if ($device->{iothread}) {
3574         my $iothreads = vm_iothreads_list($vmid);
3575         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3576     }
3577 }
3578
3579 sub qemu_iothread_del {
3580     my($conf, $vmid, $deviceid) = @_;
3581
3582     my $device = parse_drive($deviceid, $conf->{$deviceid});
3583     if ($device->{iothread}) {
3584         my $iothreads = vm_iothreads_list($vmid);
3585         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3586     }
3587 }
3588
3589 sub qemu_objectadd {
3590     my($vmid, $objectid, $qomtype) = @_;
3591
3592     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3593
3594     return 1;
3595 }
3596
3597 sub qemu_objectdel {
3598     my($vmid, $objectid) = @_;
3599
3600     vm_mon_cmd($vmid, "object-del", id => $objectid);
3601
3602     return 1;
3603 }
3604
3605 sub qemu_driveadd {
3606     my ($storecfg, $vmid, $device) = @_;
3607
3608     my $drive = print_drive_full($storecfg, $vmid, $device);
3609     $drive =~ s/\\/\\\\/g;
3610     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3611
3612     # If the command succeeds qemu prints: "OK"
3613     return 1 if $ret =~ m/OK/s;
3614
3615     die "adding drive failed: $ret\n";
3616 }
3617
3618 sub qemu_drivedel {
3619     my($vmid, $deviceid) = @_;
3620
3621     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3622     $ret =~ s/^\s+//;
3623
3624     return 1 if $ret eq "";
3625
3626     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3627     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3628
3629     die "deleting drive $deviceid failed : $ret\n";
3630 }
3631
3632 sub qemu_deviceaddverify {
3633     my ($vmid, $deviceid) = @_;
3634
3635     for (my $i = 0; $i <= 5; $i++) {
3636          my $devices_list = vm_devices_list($vmid);
3637          return 1 if defined($devices_list->{$deviceid});
3638          sleep 1;
3639     }
3640
3641     die "error on hotplug device '$deviceid'\n";
3642 }
3643
3644
3645 sub qemu_devicedelverify {
3646     my ($vmid, $deviceid) = @_;
3647
3648     # need to verify that the device is correctly removed as device_del
3649     # is async and empty return is not reliable
3650
3651     for (my $i = 0; $i <= 5; $i++) {
3652          my $devices_list = vm_devices_list($vmid);
3653          return 1 if !defined($devices_list->{$deviceid});
3654          sleep 1;
3655     }
3656
3657     die "error on hot-unplugging device '$deviceid'\n";
3658 }
3659
3660 sub qemu_findorcreatescsihw {
3661     my ($storecfg, $conf, $vmid, $device) = @_;
3662
3663     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3664
3665     my $scsihwid="$controller_prefix$controller";
3666     my $devices_list = vm_devices_list($vmid);
3667
3668     if(!defined($devices_list->{$scsihwid})) {
3669         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3670     }
3671
3672     return 1;
3673 }
3674
3675 sub qemu_deletescsihw {
3676     my ($conf, $vmid, $opt) = @_;
3677
3678     my $device = parse_drive($opt, $conf->{$opt});
3679
3680     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3681         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3682         return 1;
3683     }
3684
3685     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3686
3687     my $devices_list = vm_devices_list($vmid);
3688     foreach my $opt (keys %{$devices_list}) {
3689         if (PVE::QemuServer::valid_drivename($opt)) {
3690             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3691             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3692                 return 1;
3693             }
3694         }
3695     }
3696
3697     my $scsihwid="scsihw$controller";
3698
3699     vm_deviceunplug($vmid, $conf, $scsihwid);
3700
3701     return 1;
3702 }
3703
3704 sub qemu_add_pci_bridge {
3705     my ($storecfg, $conf, $vmid, $device) = @_;
3706
3707     my $bridges = {};
3708
3709     my $bridgeid;
3710
3711     print_pci_addr($device, $bridges);
3712
3713     while (my ($k, $v) = each %$bridges) {
3714         $bridgeid = $k;
3715     }
3716     return 1 if !defined($bridgeid) || $bridgeid < 1;
3717
3718     my $bridge = "pci.$bridgeid";
3719     my $devices_list = vm_devices_list($vmid);
3720
3721     if (!defined($devices_list->{$bridge})) {
3722         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3723     }
3724
3725     return 1;
3726 }
3727
3728 sub qemu_set_link_status {
3729     my ($vmid, $device, $up) = @_;
3730
3731     vm_mon_cmd($vmid, "set_link", name => $device,
3732                up => $up ? JSON::true : JSON::false);
3733 }
3734
3735 sub qemu_netdevadd {
3736     my ($vmid, $conf, $device, $deviceid) = @_;
3737
3738     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3739     my %options =  split(/[=,]/, $netdev);
3740
3741     vm_mon_cmd($vmid, "netdev_add",  %options);
3742     return 1;
3743 }
3744
3745 sub qemu_netdevdel {
3746     my ($vmid, $deviceid) = @_;
3747
3748     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3749 }
3750
3751 sub qemu_cpu_hotplug {
3752     my ($vmid, $conf, $vcpus) = @_;
3753
3754     my $sockets = 1;
3755     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3756     $sockets = $conf->{sockets} if  $conf->{sockets};
3757     my $cores = $conf->{cores} || 1;
3758     my $maxcpus = $sockets * $cores;
3759
3760     $vcpus = $maxcpus if !$vcpus;
3761
3762     die "you can't add more vcpus than maxcpus\n"
3763         if $vcpus > $maxcpus;
3764
3765     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3766     die "online cpu unplug is not yet possible\n"
3767         if $vcpus < $currentvcpus;
3768
3769     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3770     die "vcpus in running vm is different than configuration\n"
3771         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3772
3773     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3774         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3775     }
3776 }
3777
3778 sub qemu_memory_hotplug {
3779     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3780
3781     return $value if !check_running($vmid);
3782
3783     my $memory = $conf->{memory} || $defaults->{memory};
3784     $value = $defaults->{memory} if !$value;
3785     return $value if $value == $memory;
3786
3787     my $static_memory = $STATICMEM;
3788     my $dimm_memory = $memory - $static_memory;
3789
3790     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3791     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3792
3793
3794     my $sockets = 1;
3795     $sockets = $conf->{sockets} if $conf->{sockets};
3796
3797     if($value > $memory) {
3798
3799         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3800             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3801
3802                 return if $current_size <= $conf->{memory};
3803
3804                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3805                 if (my $err = $@) {
3806                     eval { qemu_objectdel($vmid, "mem-$name"); };
3807                     die $err;
3808                 }
3809
3810                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3811                 if (my $err = $@) {
3812                     eval { qemu_objectdel($vmid, "mem-$name"); };
3813                     die $err;
3814                 }
3815                 #update conf after each succesful module hotplug
3816                 $conf->{memory} = $current_size;
3817                 update_config_nolock($vmid, $conf, 1);
3818         });
3819
3820     } else {
3821
3822         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3823             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3824
3825                 return if $current_size >= $conf->{memory};
3826                 print "try to unplug memory dimm $name\n";
3827
3828                 my $retry = 0;
3829                 while (1) {
3830                     eval { qemu_devicedel($vmid, $name) };
3831                     sleep 3;
3832                     my $dimm_list = qemu_dimm_list($vmid);
3833                     last if !$dimm_list->{$name};
3834                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3835                     $retry++;
3836                 }
3837
3838                 #update conf after each succesful module unplug
3839                 $conf->{memory} = $current_size;
3840
3841                 eval { qemu_objectdel($vmid, "mem-$name"); };
3842                 update_config_nolock($vmid, $conf, 1);
3843         });
3844     }
3845 }
3846
3847 sub qemu_dimm_list {
3848     my ($vmid) = @_;
3849
3850     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3851     my $dimms = {};
3852
3853     foreach my $dimm (@$dimmarray) {
3854
3855         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3856         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3857         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3858         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3859         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3860     }
3861     return $dimms;
3862 }
3863
3864 sub qemu_block_set_io_throttle {
3865     my ($vmid, $deviceid,
3866         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3867         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3868
3869     return if !check_running($vmid) ;
3870
3871     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3872         bps => int($bps),
3873         bps_rd => int($bps_rd),
3874         bps_wr => int($bps_wr),
3875         iops => int($iops),
3876         iops_rd => int($iops_rd),
3877         iops_wr => int($iops_wr),
3878         bps_max => int($bps_max),
3879         bps_rd_max => int($bps_rd_max),
3880         bps_wr_max => int($bps_wr_max),
3881         iops_max => int($iops_max),
3882         iops_rd_max => int($iops_rd_max),
3883         iops_wr_max => int($iops_wr_max)
3884     );
3885
3886 }
3887
3888 # old code, only used to shutdown old VM after update
3889 sub __read_avail {
3890     my ($fh, $timeout) = @_;
3891
3892     my $sel = new IO::Select;
3893     $sel->add($fh);
3894
3895     my $res = '';
3896     my $buf;
3897
3898     my @ready;
3899     while (scalar (@ready = $sel->can_read($timeout))) {
3900         my $count;
3901         if ($count = $fh->sysread($buf, 8192)) {
3902             if ($buf =~ /^(.*)\(qemu\) $/s) {
3903                 $res .= $1;
3904                 last;
3905             } else {
3906                 $res .= $buf;
3907             }
3908         } else {
3909             if (!defined($count)) {
3910                 die "$!\n";
3911             }
3912             last;
3913         }
3914     }
3915
3916     die "monitor read timeout\n" if !scalar(@ready);
3917
3918     return $res;
3919 }
3920
3921 # old code, only used to shutdown old VM after update
3922 sub vm_monitor_command {
3923     my ($vmid, $cmdstr, $nocheck) = @_;
3924
3925     my $res;
3926
3927     eval {
3928         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3929
3930         my $sname = "${var_run_tmpdir}/$vmid.mon";
3931
3932         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3933             die "unable to connect to VM $vmid socket - $!\n";
3934
3935         my $timeout = 3;
3936
3937         # hack: migrate sometime blocks the monitor (when migrate_downtime
3938         # is set)
3939         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3940             $timeout = 60*60; # 1 hour
3941         }
3942
3943         # read banner;
3944         my $data = __read_avail($sock, $timeout);
3945
3946         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3947             die "got unexpected qemu monitor banner\n";
3948         }
3949
3950         my $sel = new IO::Select;
3951         $sel->add($sock);
3952
3953         if (!scalar(my @ready = $sel->can_write($timeout))) {
3954             die "monitor write error - timeout";
3955         }
3956
3957         my $fullcmd = "$cmdstr\r";
3958
3959         # syslog('info', "VM $vmid monitor command: $cmdstr");
3960
3961         my $b;
3962         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3963             die "monitor write error - $!";
3964         }
3965
3966         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3967
3968         $timeout = 20;
3969
3970         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3971             $timeout = 60*60; # 1 hour
3972         } elsif ($cmdstr =~ m/^(eject|change)/) {
3973             $timeout = 60; # note: cdrom mount command is slow
3974         }
3975         if ($res = __read_avail($sock, $timeout)) {
3976
3977             my @lines = split("\r?\n", $res);
3978
3979             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3980
3981             $res = join("\n", @lines);
3982             $res .= "\n";
3983         }
3984     };
3985
3986     my $err = $@;
3987
3988     if ($err) {
3989         syslog("err", "VM $vmid monitor command failed - $err");
3990         die $err;
3991     }
3992
3993     return $res;
3994 }
3995
3996 sub qemu_block_resize {
3997     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3998
3999     my $running = check_running($vmid);
4000
4001     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4002
4003     return if !$running;
4004
4005     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4006
4007 }
4008
4009 sub qemu_volume_snapshot {
4010     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4011
4012     my $running = check_running($vmid);
4013
4014     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4015         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4016     } else {
4017         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4018     }
4019 }
4020
4021 sub qemu_volume_snapshot_delete {
4022     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4023
4024     my $running = check_running($vmid);
4025
4026     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4027
4028     return if !$running;
4029
4030     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4031 }
4032
4033 sub set_migration_caps {
4034     my ($vmid) = @_;
4035
4036     my $cap_ref = [];
4037
4038     my $enabled_cap = {
4039         "auto-converge" => 1,
4040         "xbzrle" => 1,
4041         "x-rdma-pin-all" => 0,
4042         "zero-blocks" => 0,
4043         "compress" => 0
4044     };
4045
4046     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4047
4048     for my $supported_capability (@$supported_capabilities) {
4049         push @$cap_ref, {
4050             capability => $supported_capability->{capability},
4051             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4052         };
4053     }
4054
4055     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4056 }
4057
4058 my $fast_plug_option = {
4059     'lock' => 1,
4060     'name' => 1,
4061     'onboot' => 1,
4062     'shares' => 1,
4063     'startup' => 1,
4064     'description' => 1,
4065 };
4066
4067 # hotplug changes in [PENDING]
4068 # $selection hash can be used to only apply specified options, for
4069 # example: { cores => 1 } (only apply changed 'cores')
4070 # $errors ref is used to return error messages
4071 sub vmconfig_hotplug_pending {
4072     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4073
4074     my $defaults = load_defaults();
4075
4076     # commit values which do not have any impact on running VM first
4077     # Note: those option cannot raise errors, we we do not care about
4078     # $selection and always apply them.
4079
4080     my $add_error = sub {
4081         my ($opt, $msg) = @_;
4082         $errors->{$opt} = "hotplug problem - $msg";
4083     };
4084
4085     my $changes = 0;
4086     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4087         if ($fast_plug_option->{$opt}) {
4088             $conf->{$opt} = $conf->{pending}->{$opt};
4089             delete $conf->{pending}->{$opt};
4090             $changes = 1;
4091         }
4092     }
4093
4094     if ($changes) {
4095         update_config_nolock($vmid, $conf, 1);
4096         $conf = load_config($vmid); # update/reload
4097     }
4098
4099     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4100
4101     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4102     while (my ($opt, $force) = each %$pending_delete_hash) {
4103         next if $selection && !$selection->{$opt};
4104         eval {
4105             if ($opt eq 'hotplug') {
4106                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4107             } elsif ($opt eq 'tablet') {
4108                 die "skip\n" if !$hotplug_features->{usb};
4109                 if ($defaults->{tablet}) {
4110                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4111                 } else {
4112                     vm_deviceunplug($vmid, $conf, $opt);
4113                 }
4114             } elsif ($opt eq 'vcpus') {
4115                 die "skip\n" if !$hotplug_features->{cpu};
4116                 qemu_cpu_hotplug($vmid, $conf, undef);
4117             } elsif ($opt eq 'balloon') {
4118                 # enable balloon device is not hotpluggable
4119                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4120             } elsif ($fast_plug_option->{$opt}) {
4121                 # do nothing
4122             } elsif ($opt =~ m/^net(\d+)$/) {
4123                 die "skip\n" if !$hotplug_features->{network};
4124                 vm_deviceunplug($vmid, $conf, $opt);
4125             } elsif (valid_drivename($opt)) {
4126                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4127                 vm_deviceunplug($vmid, $conf, $opt);
4128                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4129             } elsif ($opt =~ m/^memory$/) {
4130                 die "skip\n" if !$hotplug_features->{memory};
4131                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4132             } elsif ($opt eq 'cpuunits') {
4133                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4134             } elsif ($opt eq 'cpulimit') {
4135                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4136             } else {
4137                 die "skip\n";
4138             }
4139         };
4140         if (my $err = $@) {
4141             &$add_error($opt, $err) if $err ne "skip\n";
4142         } else {
4143             # save new config if hotplug was successful
4144             delete $conf->{$opt};
4145             vmconfig_undelete_pending_option($conf, $opt);
4146             update_config_nolock($vmid, $conf, 1);
4147             $conf = load_config($vmid); # update/reload
4148         }
4149     }
4150
4151     foreach my $opt (keys %{$conf->{pending}}) {
4152         next if $selection && !$selection->{$opt};
4153         my $value = $conf->{pending}->{$opt};
4154         eval {
4155             if ($opt eq 'hotplug') {
4156                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4157             } elsif ($opt eq 'tablet') {
4158                 die "skip\n" if !$hotplug_features->{usb};
4159                 if ($value == 1) {
4160                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4161                 } elsif ($value == 0) {
4162                     vm_deviceunplug($vmid, $conf, $opt);
4163                 }
4164             } elsif ($opt eq 'vcpus') {
4165                 die "skip\n" if !$hotplug_features->{cpu};
4166                 qemu_cpu_hotplug($vmid, $conf, $value);
4167             } elsif ($opt eq 'balloon') {
4168                 # enable/disable balloning device is not hotpluggable
4169                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4170                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4171                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4172
4173                 # allow manual ballooning if shares is set to zero
4174                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4175                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4176                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4177                 }
4178             } elsif ($opt =~ m/^net(\d+)$/) {
4179                 # some changes can be done without hotplug
4180                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4181                                     $vmid, $opt, $value);
4182             } elsif (valid_drivename($opt)) {
4183                 # some changes can be done without hotplug
4184                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4185                                      $vmid, $opt, $value, 1);
4186             } elsif ($opt =~ m/^memory$/) { #dimms
4187                 die "skip\n" if !$hotplug_features->{memory};
4188                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4189             } elsif ($opt eq 'cpuunits') {
4190                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4191             } elsif ($opt eq 'cpulimit') {
4192                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4193                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4194             } else {
4195                 die "skip\n";  # skip non-hot-pluggable options
4196             }
4197         };
4198         if (my $err = $@) {
4199             &$add_error($opt, $err) if $err ne "skip\n";
4200         } else {
4201             # save new config if hotplug was successful
4202             $conf->{$opt} = $value;
4203             delete $conf->{pending}->{$opt};
4204             update_config_nolock($vmid, $conf, 1);
4205             $conf = load_config($vmid); # update/reload
4206         }
4207     }
4208 }
4209
4210 sub try_deallocate_drive {
4211     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4212
4213     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4214         my $volid = $drive->{file};
4215         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4216             my $sid = PVE::Storage::parse_volume_id($volid);
4217             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4218
4219             # check if the disk is really unused
4220             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4221                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4222             PVE::Storage::vdisk_free($storecfg, $volid);
4223             return 1;
4224         } else {
4225             # If vm is not owner of this disk remove from config
4226             return 1;
4227         }
4228     }
4229
4230     return undef;
4231 }
4232
4233 sub vmconfig_delete_or_detach_drive {
4234     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4235
4236     my $drive = parse_drive($opt, $conf->{$opt});
4237
4238     my $rpcenv = PVE::RPCEnvironment::get();
4239     my $authuser = $rpcenv->get_user();
4240
4241     if ($force) {
4242         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4243         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4244     } else {
4245         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4246     }
4247 }
4248
4249 sub vmconfig_apply_pending {
4250     my ($vmid, $conf, $storecfg) = @_;
4251
4252     # cold plug
4253
4254     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4255     while (my ($opt, $force) = each %$pending_delete_hash) {
4256         die "internal error" if $opt =~ m/^unused/;
4257         $conf = load_config($vmid); # update/reload
4258         if (!defined($conf->{$opt})) {
4259             vmconfig_undelete_pending_option($conf, $opt);
4260             update_config_nolock($vmid, $conf, 1);
4261         } elsif (valid_drivename($opt)) {
4262             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4263             vmconfig_undelete_pending_option($conf, $opt);
4264             delete $conf->{$opt};
4265             update_config_nolock($vmid, $conf, 1);
4266         } else {
4267             vmconfig_undelete_pending_option($conf, $opt);
4268             delete $conf->{$opt};
4269             update_config_nolock($vmid, $conf, 1);
4270         }
4271     }
4272
4273     $conf = load_config($vmid); # update/reload
4274
4275     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4276         $conf = load_config($vmid); # update/reload
4277
4278         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4279             # skip if nothing changed
4280         } elsif (valid_drivename($opt)) {
4281             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4282                 if defined($conf->{$opt});
4283             $conf->{$opt} = $conf->{pending}->{$opt};
4284         } else {
4285             $conf->{$opt} = $conf->{pending}->{$opt};
4286         }
4287
4288         delete $conf->{pending}->{$opt};
4289         update_config_nolock($vmid, $conf, 1);
4290     }
4291 }
4292
4293 my $safe_num_ne = sub {
4294     my ($a, $b) = @_;
4295
4296     return 0 if !defined($a) && !defined($b);
4297     return 1 if !defined($a);
4298     return 1 if !defined($b);
4299
4300     return $a != $b;
4301 };
4302
4303 my $safe_string_ne = sub {
4304     my ($a, $b) = @_;
4305
4306     return 0 if !defined($a) && !defined($b);
4307     return 1 if !defined($a);
4308     return 1 if !defined($b);
4309
4310     return $a ne $b;
4311 };
4312
4313 sub vmconfig_update_net {
4314     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4315
4316     my $newnet = parse_net($value);
4317
4318     if ($conf->{$opt}) {
4319         my $oldnet = parse_net($conf->{$opt});
4320
4321         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4322             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4323             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4324             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4325
4326             # for non online change, we try to hot-unplug
4327             die "skip\n" if !$hotplug;
4328             vm_deviceunplug($vmid, $conf, $opt);
4329         } else {
4330
4331             die "internal error" if $opt !~ m/net(\d+)/;
4332             my $iface = "tap${vmid}i$1";
4333
4334             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4335                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4336             }
4337
4338             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4339                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4340                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4341                 PVE::Network::tap_unplug($iface);
4342                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4343             }
4344
4345             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4346                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4347             }
4348
4349             return 1;
4350         }
4351     }
4352
4353     if ($hotplug) {
4354         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4355     } else {
4356         die "skip\n";
4357     }
4358 }
4359
4360 sub vmconfig_update_disk {
4361     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4362
4363     # fixme: do we need force?
4364
4365     my $drive = parse_drive($opt, $value);
4366
4367     if ($conf->{$opt}) {
4368
4369         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4370
4371             my $media = $drive->{media} || 'disk';
4372             my $oldmedia = $old_drive->{media} || 'disk';
4373             die "unable to change media type\n" if $media ne $oldmedia;
4374
4375             if (!drive_is_cdrom($old_drive)) {
4376
4377                 if ($drive->{file} ne $old_drive->{file}) {
4378
4379                     die "skip\n" if !$hotplug;
4380
4381                     # unplug and register as unused
4382                     vm_deviceunplug($vmid, $conf, $opt);
4383                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4384
4385                 } else {
4386                     # update existing disk
4387
4388                     # skip non hotpluggable value
4389                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4390                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4391                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4392                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4393                         die "skip\n";
4394                     }
4395
4396                     # apply throttle
4397                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4398                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4399                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4400                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4401                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4402                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4403                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4404                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4405                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4406                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4407                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4408                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4409
4410                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4411                                                    ($drive->{mbps} || 0)*1024*1024,
4412                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4413                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4414                                                    $drive->{iops} || 0,
4415                                                    $drive->{iops_rd} || 0,
4416                                                    $drive->{iops_wr} || 0,
4417                                                    ($drive->{mbps_max} || 0)*1024*1024,
4418                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4419                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4420                                                    $drive->{iops_max} || 0,
4421                                                    $drive->{iops_rd_max} || 0,
4422                                                    $drive->{iops_wr_max} || 0);
4423
4424                     }
4425
4426                     return 1;
4427                 }
4428
4429             } else { # cdrom
4430
4431                 if ($drive->{file} eq 'none') {
4432                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4433                 } else {
4434                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4435                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4436                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4437                 }
4438
4439                 return 1;
4440             }
4441         }
4442     }
4443
4444     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4445     # hotplug new disks
4446     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4447     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4448 }
4449
4450 sub vm_start {
4451     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4452         $forcemachine, $spice_ticket) = @_;
4453
4454     lock_config($vmid, sub {
4455         my $conf = load_config($vmid, $migratedfrom);
4456
4457         die "you can't start a vm if it's a template\n" if is_template($conf);
4458
4459         check_lock($conf) if !$skiplock;
4460
4461         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4462
4463         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4464             vmconfig_apply_pending($vmid, $conf, $storecfg);
4465             $conf = load_config($vmid); # update/reload
4466         }
4467
4468         my $defaults = load_defaults();
4469
4470         # set environment variable useful inside network script
4471         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4472
4473         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4474
4475         my $migrate_port = 0;
4476         my $migrate_uri;
4477         if ($statefile) {
4478             if ($statefile eq 'tcp') {
4479                 my $localip = "localhost";
4480                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4481                 my $nodename = PVE::INotify::nodename();
4482                 if ($datacenterconf->{migration_unsecure}) {
4483                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4484                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4485                 }
4486                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4487                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4488                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4489                 push @$cmd, '-incoming', $migrate_uri;
4490                 push @$cmd, '-S';
4491             } else {
4492                 push @$cmd, '-loadstate', $statefile;
4493             }
4494         } elsif ($paused) {
4495             push @$cmd, '-S';
4496         }
4497
4498         # host pci devices
4499         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4500           my $d = parse_hostpci($conf->{"hostpci$i"});
4501           next if !$d;
4502           my $pcidevices = $d->{pciid};
4503           foreach my $pcidevice (@$pcidevices) {
4504                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4505
4506                 my $info = pci_device_info("0000:$pciid");
4507                 die "IOMMU not present\n" if !check_iommu_support();
4508                 die "no pci device info for device '$pciid'\n" if !$info;
4509                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4510                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4511           }
4512         }
4513
4514         PVE::Storage::activate_volumes($storecfg, $vollist);
4515
4516         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4517                     umask => 0077); };
4518         my $err = $@;
4519         die "start failed: $err" if $err;
4520
4521         print "migration listens on $migrate_uri\n" if $migrate_uri;
4522
4523         if ($statefile && $statefile ne 'tcp')  {
4524             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4525             warn $@ if $@;
4526         }
4527
4528         if ($migratedfrom) {
4529
4530             eval {
4531                 set_migration_caps($vmid);
4532             };
4533             warn $@ if $@;
4534
4535             if ($spice_port) {
4536                 print "spice listens on port $spice_port\n";
4537                 if ($spice_ticket) {
4538                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4539                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4540                 }
4541             }
4542
4543         } else {
4544
4545             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4546                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4547                     if $conf->{balloon};
4548             }
4549
4550             foreach my $opt (keys %$conf) {
4551                 next if $opt !~  m/^net\d+$/;
4552                 my $nicconf = parse_net($conf->{$opt});
4553                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4554             }
4555         }
4556
4557         vm_mon_cmd_nocheck($vmid, 'qom-set',
4558                     path => "machine/peripheral/balloon0",
4559                     property => "guest-stats-polling-interval",
4560                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4561
4562     });
4563 }
4564
4565 sub vm_mon_cmd {
4566     my ($vmid, $execute, %params) = @_;
4567
4568     my $cmd = { execute => $execute, arguments => \%params };
4569     vm_qmp_command($vmid, $cmd);
4570 }
4571
4572 sub vm_mon_cmd_nocheck {
4573     my ($vmid, $execute, %params) = @_;
4574
4575     my $cmd = { execute => $execute, arguments => \%params };
4576     vm_qmp_command($vmid, $cmd, 1);
4577 }
4578
4579 sub vm_qmp_command {
4580     my ($vmid, $cmd, $nocheck) = @_;
4581
4582     my $res;
4583
4584     my $timeout;
4585     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4586         $timeout = $cmd->{arguments}->{timeout};
4587         delete $cmd->{arguments}->{timeout};
4588     }
4589
4590     eval {
4591         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4592         my $sname = qmp_socket($vmid);
4593         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4594             my $qmpclient = PVE::QMPClient->new();
4595
4596             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4597         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4598             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4599                 if scalar(%{$cmd->{arguments}});
4600             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4601         } else {
4602             die "unable to open monitor socket\n";
4603         }
4604     };
4605     if (my $err = $@) {
4606         syslog("err", "VM $vmid qmp command failed - $err");
4607         die $err;
4608     }
4609
4610     return $res;
4611 }
4612
4613 sub vm_human_monitor_command {
4614     my ($vmid, $cmdline) = @_;
4615
4616     my $res;
4617
4618     my $cmd = {
4619         execute => 'human-monitor-command',
4620         arguments => { 'command-line' => $cmdline},
4621     };
4622
4623     return vm_qmp_command($vmid, $cmd);
4624 }
4625
4626 sub vm_commandline {
4627     my ($storecfg, $vmid) = @_;
4628
4629     my $conf = load_config($vmid);
4630
4631     my $defaults = load_defaults();
4632
4633     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4634
4635     return join(' ', @$cmd);
4636 }
4637
4638 sub vm_reset {
4639     my ($vmid, $skiplock) = @_;
4640
4641     lock_config($vmid, sub {
4642
4643         my $conf = load_config($vmid);
4644
4645         check_lock($conf) if !$skiplock;
4646
4647         vm_mon_cmd($vmid, "system_reset");
4648     });
4649 }
4650
4651 sub get_vm_volumes {
4652     my ($conf) = @_;
4653
4654     my $vollist = [];
4655     foreach_volid($conf, sub {
4656         my ($volid, $is_cdrom) = @_;
4657
4658         return if $volid =~ m|^/|;
4659
4660         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4661         return if !$sid;
4662
4663         push @$vollist, $volid;
4664     });
4665
4666     return $vollist;
4667 }
4668
4669 sub vm_stop_cleanup {
4670     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4671
4672     eval {
4673
4674         if (!$keepActive) {
4675             my $vollist = get_vm_volumes($conf);
4676             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4677         }
4678
4679         foreach my $ext (qw(mon qmp pid vnc qga)) {
4680             unlink "/var/run/qemu-server/${vmid}.$ext";
4681         }
4682
4683         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4684     };
4685     warn $@ if $@; # avoid errors - just warn
4686 }
4687
4688 # Note: use $nockeck to skip tests if VM configuration file exists.
4689 # We need that when migration VMs to other nodes (files already moved)
4690 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4691 sub vm_stop {
4692     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4693
4694     $force = 1 if !defined($force) && !$shutdown;
4695
4696     if ($migratedfrom){
4697         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4698         kill 15, $pid if $pid;
4699         my $conf = load_config($vmid, $migratedfrom);
4700         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4701         return;
4702     }
4703
4704     lock_config($vmid, sub {
4705
4706         my $pid = check_running($vmid, $nocheck);
4707         return if !$pid;
4708
4709         my $conf;
4710         if (!$nocheck) {
4711             $conf = load_config($vmid);
4712             check_lock($conf) if !$skiplock;
4713             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4714                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4715                 $timeout = $opts->{down} if $opts->{down};
4716             }
4717         }
4718
4719         $timeout = 60 if !defined($timeout);
4720
4721         eval {
4722             if ($shutdown) {
4723                 if (defined($conf) && $conf->{agent}) {
4724                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4725                 } else {
4726                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4727                 }
4728             } else {
4729                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4730             }
4731         };
4732         my $err = $@;
4733
4734         if (!$err) {
4735             my $count = 0;
4736             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4737                 $count++;
4738                 sleep 1;
4739             }
4740
4741             if ($count >= $timeout) {
4742                 if ($force) {
4743                     warn "VM still running - terminating now with SIGTERM\n";
4744                     kill 15, $pid;
4745                 } else {
4746                     die "VM quit/powerdown failed - got timeout\n";
4747                 }
4748             } else {
4749                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4750                 return;
4751             }
4752         } else {
4753             if ($force) {
4754                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4755                 kill 15, $pid;
4756             } else {
4757                 die "VM quit/powerdown failed\n";
4758             }
4759         }
4760
4761         # wait again
4762         $timeout = 10;
4763
4764         my $count = 0;
4765         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4766             $count++;
4767             sleep 1;
4768         }
4769
4770         if ($count >= $timeout) {
4771             warn "VM still running - terminating now with SIGKILL\n";
4772             kill 9, $pid;
4773             sleep 1;
4774         }
4775
4776         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4777    });
4778 }
4779
4780 sub vm_suspend {
4781     my ($vmid, $skiplock) = @_;
4782
4783     lock_config($vmid, sub {
4784
4785         my $conf = load_config($vmid);
4786
4787         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4788
4789         vm_mon_cmd($vmid, "stop");
4790     });
4791 }
4792
4793 sub vm_resume {
4794     my ($vmid, $skiplock, $nocheck) = @_;
4795
4796     lock_config($vmid, sub {
4797
4798         if (!$nocheck) {
4799
4800             my $conf = load_config($vmid);
4801
4802             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4803
4804             vm_mon_cmd($vmid, "cont");
4805
4806         } else {
4807             vm_mon_cmd_nocheck($vmid, "cont");
4808         }
4809     });
4810 }
4811
4812 sub vm_sendkey {
4813     my ($vmid, $skiplock, $key) = @_;
4814
4815     lock_config($vmid, sub {
4816
4817         my $conf = load_config($vmid);
4818
4819         # there is no qmp command, so we use the human monitor command
4820         vm_human_monitor_command($vmid, "sendkey $key");
4821     });
4822 }
4823
4824 sub vm_destroy {
4825     my ($storecfg, $vmid, $skiplock) = @_;
4826
4827     lock_config($vmid, sub {
4828
4829         my $conf = load_config($vmid);
4830
4831         check_lock($conf) if !$skiplock;
4832
4833         if (!check_running($vmid)) {
4834             destroy_vm($storecfg, $vmid);
4835         } else {
4836             die "VM $vmid is running - destroy failed\n";
4837         }
4838     });
4839 }
4840
4841 # pci helpers
4842
4843 sub file_write {
4844     my ($filename, $buf) = @_;
4845
4846     my $fh = IO::File->new($filename, "w");
4847     return undef if !$fh;
4848
4849     my $res = print $fh $buf;
4850
4851     $fh->close();
4852
4853     return $res;
4854 }
4855
4856 sub pci_device_info {
4857     my ($name) = @_;
4858
4859     my $res;
4860
4861     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4862     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4863
4864     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4865     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4866
4867     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4868     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4869
4870     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4871     return undef if !defined($product) || $product !~ s/^0x//;
4872
4873     $res = {
4874         name => $name,
4875         vendor => $vendor,
4876         product => $product,
4877         domain => $domain,
4878         bus => $bus,
4879         slot => $slot,
4880         func => $func,
4881         irq => $irq,
4882         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4883     };
4884
4885     return $res;
4886 }
4887
4888 sub pci_dev_reset {
4889     my ($dev) = @_;
4890
4891     my $name = $dev->{name};
4892
4893     my $fn = "$pcisysfs/devices/$name/reset";
4894
4895     return file_write($fn, "1");
4896 }
4897
4898 sub pci_dev_bind_to_vfio {
4899     my ($dev) = @_;
4900
4901     my $name = $dev->{name};
4902
4903     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4904
4905     if (!-d $vfio_basedir) {
4906         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4907     }
4908     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4909
4910     my $testdir = "$vfio_basedir/$name";
4911     return 1 if -d $testdir;
4912
4913     my $data = "$dev->{vendor} $dev->{product}";
4914     return undef if !file_write("$vfio_basedir/new_id", $data);
4915
4916     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4917     if (!file_write($fn, $name)) {
4918         return undef if -f $fn;
4919     }
4920
4921     $fn = "$vfio_basedir/bind";
4922     if (! -d $testdir) {
4923         return undef if !file_write($fn, $name);
4924     }
4925
4926     return -d $testdir;
4927 }
4928
4929 sub pci_dev_group_bind_to_vfio {
4930     my ($pciid) = @_;
4931
4932     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4933
4934     if (!-d $vfio_basedir) {
4935         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4936     }
4937     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4938
4939     # get IOMMU group devices
4940     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4941       my @devs = grep /^0000:/, readdir($D);
4942     closedir($D);
4943
4944     foreach my $pciid (@devs) {
4945         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4946
4947         # pci bridges, switches or root ports are not supported
4948         # they have a pci_bus subdirectory so skip them
4949         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4950
4951         my $info = pci_device_info($1);
4952         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4953     }
4954
4955     return 1;
4956 }
4957
4958 sub print_pci_addr {
4959     my ($id, $bridges) = @_;
4960
4961     my $res = '';
4962     my $devices = {
4963         piix3 => { bus => 0, addr => 1 },
4964         #addr2 : first videocard
4965         balloon0 => { bus => 0, addr => 3 },
4966         watchdog => { bus => 0, addr => 4 },
4967         scsihw0 => { bus => 0, addr => 5 },
4968         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4969         scsihw1 => { bus => 0, addr => 6 },
4970         ahci0 => { bus => 0, addr => 7 },
4971         qga0 => { bus => 0, addr => 8 },
4972         spice => { bus => 0, addr => 9 },
4973         virtio0 => { bus => 0, addr => 10 },
4974         virtio1 => { bus => 0, addr => 11 },
4975         virtio2 => { bus => 0, addr => 12 },
4976         virtio3 => { bus => 0, addr => 13 },
4977         virtio4 => { bus => 0, addr => 14 },
4978         virtio5 => { bus => 0, addr => 15 },
4979         hostpci0 => { bus => 0, addr => 16 },
4980         hostpci1 => { bus => 0, addr => 17 },
4981         net0 => { bus => 0, addr => 18 },
4982         net1 => { bus => 0, addr => 19 },
4983         net2 => { bus => 0, addr => 20 },
4984         net3 => { bus => 0, addr => 21 },
4985         net4 => { bus => 0, addr => 22 },
4986         net5 => { bus => 0, addr => 23 },
4987         vga1 => { bus => 0, addr => 24 },
4988         vga2 => { bus => 0, addr => 25 },
4989         vga3 => { bus => 0, addr => 26 },
4990         hostpci2 => { bus => 0, addr => 27 },
4991         hostpci3 => { bus => 0, addr => 28 },
4992         #addr29 : usb-host (pve-usb.cfg)
4993         'pci.1' => { bus => 0, addr => 30 },
4994         'pci.2' => { bus => 0, addr => 31 },
4995         'net6' => { bus => 1, addr => 1 },
4996         'net7' => { bus => 1, addr => 2 },
4997         'net8' => { bus => 1, addr => 3 },
4998         'net9' => { bus => 1, addr => 4 },
4999         'net10' => { bus => 1, addr => 5 },
5000         'net11' => { bus => 1, addr => 6 },
5001         'net12' => { bus => 1, addr => 7 },
5002         'net13' => { bus => 1, addr => 8 },
5003         'net14' => { bus => 1, addr => 9 },
5004         'net15' => { bus => 1, addr => 10 },
5005         'net16' => { bus => 1, addr => 11 },
5006         'net17' => { bus => 1, addr => 12 },
5007         'net18' => { bus => 1, addr => 13 },
5008         'net19' => { bus => 1, addr => 14 },
5009         'net20' => { bus => 1, addr => 15 },
5010         'net21' => { bus => 1, addr => 16 },
5011         'net22' => { bus => 1, addr => 17 },
5012         'net23' => { bus => 1, addr => 18 },
5013         'net24' => { bus => 1, addr => 19 },
5014         'net25' => { bus => 1, addr => 20 },
5015         'net26' => { bus => 1, addr => 21 },
5016         'net27' => { bus => 1, addr => 22 },
5017         'net28' => { bus => 1, addr => 23 },
5018         'net29' => { bus => 1, addr => 24 },
5019         'net30' => { bus => 1, addr => 25 },
5020         'net31' => { bus => 1, addr => 26 },
5021         'virtio6' => { bus => 2, addr => 1 },
5022         'virtio7' => { bus => 2, addr => 2 },
5023         'virtio8' => { bus => 2, addr => 3 },
5024         'virtio9' => { bus => 2, addr => 4 },
5025         'virtio10' => { bus => 2, addr => 5 },
5026         'virtio11' => { bus => 2, addr => 6 },
5027         'virtio12' => { bus => 2, addr => 7 },
5028         'virtio13' => { bus => 2, addr => 8 },
5029         'virtio14' => { bus => 2, addr => 9 },
5030         'virtio15' => { bus => 2, addr => 10 },
5031         'virtioscsi0' => { bus => 3, addr => 1 },
5032         'virtioscsi1' => { bus => 3, addr => 2 },
5033         'virtioscsi2' => { bus => 3, addr => 3 },
5034         'virtioscsi3' => { bus => 3, addr => 4 },
5035         'virtioscsi4' => { bus => 3, addr => 5 },
5036         'virtioscsi5' => { bus => 3, addr => 6 },
5037         'virtioscsi6' => { bus => 3, addr => 7 },
5038         'virtioscsi7' => { bus => 3, addr => 8 },
5039         'virtioscsi8' => { bus => 3, addr => 9 },
5040         'virtioscsi9' => { bus => 3, addr => 10 },
5041         'virtioscsi10' => { bus => 3, addr => 11 },
5042         'virtioscsi11' => { bus => 3, addr => 12 },
5043         'virtioscsi12' => { bus => 3, addr => 13 },
5044         'virtioscsi13' => { bus => 3, addr => 14 },
5045         'virtioscsi14' => { bus => 3, addr => 15 },
5046         'virtioscsi15' => { bus => 3, addr => 16 },
5047         'virtioscsi16' => { bus => 3, addr => 17 },
5048         'virtioscsi17' => { bus => 3, addr => 18 },
5049         'virtioscsi18' => { bus => 3, addr => 19 },
5050         'virtioscsi19' => { bus => 3, addr => 20 },
5051         'virtioscsi20' => { bus => 3, addr => 21 },
5052         'virtioscsi21' => { bus => 3, addr => 22 },
5053         'virtioscsi22' => { bus => 3, addr => 23 },
5054         'virtioscsi23' => { bus => 3, addr => 24 },
5055         'virtioscsi24' => { bus => 3, addr => 25 },
5056         'virtioscsi25' => { bus => 3, addr => 26 },
5057         'virtioscsi26' => { bus => 3, addr => 27 },
5058         'virtioscsi27' => { bus => 3, addr => 28 },
5059         'virtioscsi28' => { bus => 3, addr => 29 },
5060         'virtioscsi29' => { bus => 3, addr => 30 },
5061         'virtioscsi30' => { bus => 3, addr => 31 },
5062
5063     };
5064
5065     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5066            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5067            my $bus = $devices->{$id}->{bus};
5068            $res = ",bus=pci.$bus,addr=$addr";
5069            $bridges->{$bus} = 1 if $bridges;
5070     }
5071     return $res;
5072
5073 }
5074
5075 sub print_pcie_addr {
5076     my ($id) = @_;
5077
5078     my $res = '';
5079     my $devices = {
5080         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5081         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5082         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5083         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5084     };
5085
5086     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5087            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5088            my $bus = $devices->{$id}->{bus};
5089            $res = ",bus=$bus,addr=$addr";
5090     }
5091     return $res;
5092
5093 }
5094
5095 # vzdump restore implementaion
5096
5097 sub tar_archive_read_firstfile {
5098     my $archive = shift;
5099
5100     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5101
5102     # try to detect archive type first
5103     my $pid = open (TMP, "tar tf '$archive'|") ||
5104         die "unable to open file '$archive'\n";
5105     my $firstfile = <TMP>;
5106     kill 15, $pid;
5107     close TMP;
5108
5109     die "ERROR: archive contaions no data\n" if !$firstfile;
5110     chomp $firstfile;
5111
5112     return $firstfile;
5113 }
5114
5115 sub tar_restore_cleanup {
5116     my ($storecfg, $statfile) = @_;
5117
5118     print STDERR "starting cleanup\n";
5119
5120     if (my $fd = IO::File->new($statfile, "r")) {
5121         while (defined(my $line = <$fd>)) {
5122             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5123                 my $volid = $2;
5124                 eval {
5125                     if ($volid =~ m|^/|) {
5126                         unlink $volid || die 'unlink failed\n';
5127                     } else {
5128                         PVE::Storage::vdisk_free($storecfg, $volid);
5129                     }
5130                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5131                 };
5132                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5133             } else {
5134                 print STDERR "unable to parse line in statfile - $line";
5135             }
5136         }
5137         $fd->close();
5138     }
5139 }
5140
5141 sub restore_archive {
5142     my ($archive, $vmid, $user, $opts) = @_;
5143
5144     my $format = $opts->{format};
5145     my $comp;
5146
5147     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5148         $format = 'tar' if !$format;
5149         $comp = 'gzip';
5150     } elsif ($archive =~ m/\.tar$/) {
5151         $format = 'tar' if !$format;
5152     } elsif ($archive =~ m/.tar.lzo$/) {
5153         $format = 'tar' if !$format;
5154         $comp = 'lzop';
5155     } elsif ($archive =~ m/\.vma$/) {
5156         $format = 'vma' if !$format;
5157     } elsif ($archive =~ m/\.vma\.gz$/) {
5158         $format = 'vma' if !$format;
5159         $comp = 'gzip';
5160     } elsif ($archive =~ m/\.vma\.lzo$/) {
5161         $format = 'vma' if !$format;
5162         $comp = 'lzop';
5163     } else {
5164         $format = 'vma' if !$format; # default
5165     }
5166
5167     # try to detect archive format
5168     if ($format eq 'tar') {
5169         return restore_tar_archive($archive, $vmid, $user, $opts);
5170     } else {
5171         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5172     }
5173 }
5174
5175 sub restore_update_config_line {
5176     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5177
5178     return if $line =~ m/^\#qmdump\#/;
5179     return if $line =~ m/^\#vzdump\#/;
5180     return if $line =~ m/^lock:/;
5181     return if $line =~ m/^unused\d+:/;
5182     return if $line =~ m/^parent:/;
5183     return if $line =~ m/^template:/; # restored VM is never a template
5184
5185     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5186         # try to convert old 1.X settings
5187         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5188         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5189             my ($model, $macaddr) = split(/\=/, $devconfig);
5190             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5191             my $net = {
5192                 model => $model,
5193                 bridge => "vmbr$ind",
5194                 macaddr => $macaddr,
5195             };
5196             my $netstr = print_net($net);
5197
5198             print $outfd "net$cookie->{netcount}: $netstr\n";
5199             $cookie->{netcount}++;
5200         }
5201     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5202         my ($id, $netstr) = ($1, $2);
5203         my $net = parse_net($netstr);
5204         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5205         $netstr = print_net($net);
5206         print $outfd "$id: $netstr\n";
5207     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5208         my $virtdev = $1;
5209         my $value = $3;
5210         if ($line =~ m/backup=no/) {
5211             print $outfd "#$line";
5212         } elsif ($virtdev && $map->{$virtdev}) {
5213             my $di = parse_drive($virtdev, $value);
5214             delete $di->{format}; # format can change on restore
5215             $di->{file} = $map->{$virtdev};
5216             $value = print_drive($vmid, $di);
5217             print $outfd "$virtdev: $value\n";
5218         } else {
5219             print $outfd $line;
5220         }
5221     } else {
5222         print $outfd $line;
5223     }
5224 }
5225
5226 sub scan_volids {
5227     my ($cfg, $vmid) = @_;
5228
5229     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5230
5231     my $volid_hash = {};
5232     foreach my $storeid (keys %$info) {
5233         foreach my $item (@{$info->{$storeid}}) {
5234             next if !($item->{volid} && $item->{size});
5235             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5236             $volid_hash->{$item->{volid}} = $item;
5237         }
5238     }
5239
5240     return $volid_hash;
5241 }
5242
5243 sub is_volume_in_use {
5244     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5245
5246     my $path = PVE::Storage::path($storecfg, $volid);
5247
5248     my $scan_config = sub {
5249         my ($cref, $snapname) = @_;
5250
5251         foreach my $key (keys %$cref) {
5252             my $value = $cref->{$key};
5253             if (valid_drivename($key)) {
5254                 next if $skip_drive && $key eq $skip_drive;
5255                 my $drive = parse_drive($key, $value);
5256                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5257                 return 1 if $volid eq $drive->{file};
5258                 if ($drive->{file} =~ m!^/!) {
5259                     return 1 if $drive->{file} eq $path;
5260                 } else {
5261                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5262                     next if !$storeid;
5263                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5264                     next if !$scfg;
5265                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5266                 }
5267             }
5268         }
5269
5270         return 0;
5271     };
5272
5273     return 1 if &$scan_config($conf);
5274
5275     undef $skip_drive;
5276
5277     foreach my $snapname (keys %{$conf->{snapshots}}) {
5278         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5279     }
5280
5281     return 0;
5282 }
5283
5284 sub update_disksize {
5285     my ($vmid, $conf, $volid_hash) = @_;
5286
5287     my $changes;
5288
5289     my $used = {};
5290
5291     # Note: it is allowed to define multiple storages with same path (alias), so
5292     # we need to check both 'volid' and real 'path' (two different volid can point
5293     # to the same path).
5294
5295     my $usedpath = {};
5296
5297     # update size info
5298     foreach my $opt (keys %$conf) {
5299         if (valid_drivename($opt)) {
5300             my $drive = parse_drive($opt, $conf->{$opt});
5301             my $volid = $drive->{file};
5302             next if !$volid;
5303
5304             $used->{$volid} = 1;
5305             if ($volid_hash->{$volid} &&
5306                 (my $path = $volid_hash->{$volid}->{path})) {
5307                 $usedpath->{$path} = 1;
5308             }
5309
5310             next if drive_is_cdrom($drive);
5311             next if !$volid_hash->{$volid};
5312
5313             $drive->{size} = $volid_hash->{$volid}->{size};
5314             my $new = print_drive($vmid, $drive);
5315             if ($new ne $conf->{$opt}) {
5316                 $changes = 1;
5317                 $conf->{$opt} = $new;
5318             }
5319         }
5320     }
5321
5322     # remove 'unusedX' entry if volume is used
5323     foreach my $opt (keys %$conf) {
5324         next if $opt !~ m/^unused\d+$/;
5325         my $volid = $conf->{$opt};
5326         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5327         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5328             $changes = 1;
5329             delete $conf->{$opt};
5330         }
5331     }
5332
5333     foreach my $volid (sort keys %$volid_hash) {
5334         next if $volid =~ m/vm-$vmid-state-/;
5335         next if $used->{$volid};
5336         my $path = $volid_hash->{$volid}->{path};
5337         next if !$path; # just to be sure
5338         next if $usedpath->{$path};
5339         $changes = 1;
5340         add_unused_volume($conf, $volid);
5341         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5342     }
5343
5344     return $changes;
5345 }
5346
5347 sub rescan {
5348     my ($vmid, $nolock) = @_;
5349
5350     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5351
5352     my $volid_hash = scan_volids($cfg, $vmid);
5353
5354     my $updatefn =  sub {
5355         my ($vmid) = @_;
5356
5357         my $conf = load_config($vmid);
5358
5359         check_lock($conf);
5360
5361         my $vm_volids = {};
5362         foreach my $volid (keys %$volid_hash) {
5363             my $info = $volid_hash->{$volid};
5364             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5365         }
5366
5367         my $changes = update_disksize($vmid, $conf, $vm_volids);
5368
5369         update_config_nolock($vmid, $conf, 1) if $changes;
5370     };
5371
5372     if (defined($vmid)) {
5373         if ($nolock) {
5374             &$updatefn($vmid);
5375         } else {
5376             lock_config($vmid, $updatefn, $vmid);
5377         }
5378     } else {
5379         my $vmlist = config_list();
5380         foreach my $vmid (keys %$vmlist) {
5381             if ($nolock) {
5382                 &$updatefn($vmid);
5383             } else {
5384                 lock_config($vmid, $updatefn, $vmid);
5385             }
5386         }
5387     }
5388 }
5389
5390 sub restore_vma_archive {
5391     my ($archive, $vmid, $user, $opts, $comp) = @_;
5392
5393     my $input = $archive eq '-' ? "<&STDIN" : undef;
5394     my $readfrom = $archive;
5395
5396     my $uncomp = '';
5397     if ($comp) {
5398         $readfrom = '-';
5399         my $qarchive = PVE::Tools::shellquote($archive);
5400         if ($comp eq 'gzip') {
5401             $uncomp = "zcat $qarchive|";
5402         } elsif ($comp eq 'lzop') {
5403             $uncomp = "lzop -d -c $qarchive|";
5404         } else {
5405             die "unknown compression method '$comp'\n";
5406         }
5407
5408     }
5409
5410     my $tmpdir = "/var/tmp/vzdumptmp$$";
5411     rmtree $tmpdir;
5412
5413     # disable interrupts (always do cleanups)
5414     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5415         warn "got interrupt - ignored\n";
5416     };
5417
5418     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5419     POSIX::mkfifo($mapfifo, 0600);
5420     my $fifofh;
5421
5422     my $openfifo = sub {
5423         open($fifofh, '>', $mapfifo) || die $!;
5424     };
5425
5426     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5427
5428     my $oldtimeout;
5429     my $timeout = 5;
5430
5431     my $devinfo = {};
5432
5433     my $rpcenv = PVE::RPCEnvironment::get();
5434
5435     my $conffile = config_file($vmid);
5436     my $tmpfn = "$conffile.$$.tmp";
5437
5438     # Note: $oldconf is undef if VM does not exists
5439     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5440
5441     my $print_devmap = sub {
5442         my $virtdev_hash = {};
5443
5444         my $cfgfn = "$tmpdir/qemu-server.conf";
5445
5446         # we can read the config - that is already extracted
5447         my $fh = IO::File->new($cfgfn, "r") ||
5448             "unable to read qemu-server.conf - $!\n";
5449
5450         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5451         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5452             if -f $fwcfgfn;
5453
5454         while (defined(my $line = <$fh>)) {
5455             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5456                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5457                 die "archive does not contain data for drive '$virtdev'\n"
5458                     if !$devinfo->{$devname};
5459                 if (defined($opts->{storage})) {
5460                     $storeid = $opts->{storage} || 'local';
5461                 } elsif (!$storeid) {
5462                     $storeid = 'local';
5463                 }
5464                 $format = 'raw' if !$format;
5465                 $devinfo->{$devname}->{devname} = $devname;
5466                 $devinfo->{$devname}->{virtdev} = $virtdev;
5467                 $devinfo->{$devname}->{format} = $format;
5468                 $devinfo->{$devname}->{storeid} = $storeid;
5469
5470                 # check permission on storage
5471                 my $pool = $opts->{pool}; # todo: do we need that?
5472                 if ($user ne 'root@pam') {
5473                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5474                 }
5475
5476                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5477             }
5478         }
5479
5480         foreach my $devname (keys %$devinfo) {
5481             die "found no device mapping information for device '$devname'\n"
5482                 if !$devinfo->{$devname}->{virtdev};
5483         }
5484
5485         my $cfg = cfs_read_file('storage.cfg');
5486
5487         # create empty/temp config
5488         if ($oldconf) {
5489             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5490             foreach_drive($oldconf, sub {
5491                 my ($ds, $drive) = @_;
5492
5493                 return if drive_is_cdrom($drive);
5494
5495                 my $volid = $drive->{file};
5496
5497                 return if !$volid || $volid =~ m|^/|;
5498
5499                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5500                 return if !$path || !$owner || ($owner != $vmid);
5501
5502                 # Note: only delete disk we want to restore
5503                 # other volumes will become unused
5504                 if ($virtdev_hash->{$ds}) {
5505                     PVE::Storage::vdisk_free($cfg, $volid);
5506                 }
5507             });
5508         }
5509
5510         my $map = {};
5511         foreach my $virtdev (sort keys %$virtdev_hash) {
5512             my $d = $virtdev_hash->{$virtdev};
5513             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5514             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5515
5516             # test if requested format is supported
5517             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5518             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5519             $d->{format} = $defFormat if !$supported;
5520
5521             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5522                                                   $d->{format}, undef, $alloc_size);
5523             print STDERR "new volume ID is '$volid'\n";
5524             $d->{volid} = $volid;
5525             my $path = PVE::Storage::path($cfg, $volid);
5526
5527             PVE::Storage::activate_volumes($cfg,[$volid]);
5528
5529             my $write_zeros = 1;
5530             # fixme: what other storages types initialize volumes with zero?
5531             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5532                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5533                 $write_zeros = 0;
5534             }
5535
5536             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5537
5538             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5539             $map->{$virtdev} = $volid;
5540         }
5541
5542         $fh->seek(0, 0) || die "seek failed - $!\n";
5543
5544         my $outfd = new IO::File ($tmpfn, "w") ||
5545             die "unable to write config for VM $vmid\n";
5546
5547         my $cookie = { netcount => 0 };
5548         while (defined(my $line = <$fh>)) {
5549             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5550         }
5551
5552         $fh->close();
5553         $outfd->close();
5554     };
5555
5556     eval {
5557         # enable interrupts
5558         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5559             die "interrupted by signal\n";
5560         };
5561         local $SIG{ALRM} = sub { die "got timeout\n"; };
5562
5563         $oldtimeout = alarm($timeout);
5564
5565         my $parser = sub {
5566             my $line = shift;
5567
5568             print "$line\n";
5569
5570             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5571                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5572                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5573             } elsif ($line =~ m/^CTIME: /) {
5574                 # we correctly received the vma config, so we can disable
5575                 # the timeout now for disk allocation (set to 10 minutes, so
5576                 # that we always timeout if something goes wrong)
5577                 alarm(600);
5578                 &$print_devmap();
5579                 print $fifofh "done\n";
5580                 my $tmp = $oldtimeout || 0;
5581                 $oldtimeout = undef;
5582                 alarm($tmp);
5583                 close($fifofh);
5584             }
5585         };
5586
5587         print "restore vma archive: $cmd\n";
5588         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5589     };
5590     my $err = $@;
5591
5592     alarm($oldtimeout) if $oldtimeout;
5593
5594     my $vollist = [];
5595     foreach my $devname (keys %$devinfo) {
5596         my $volid = $devinfo->{$devname}->{volid};
5597         push @$vollist, $volid if $volid;
5598     }
5599
5600     my $cfg = cfs_read_file('storage.cfg');
5601     PVE::Storage::deactivate_volumes($cfg, $vollist);
5602
5603     unlink $mapfifo;
5604
5605     if ($err) {
5606         rmtree $tmpdir;
5607         unlink $tmpfn;
5608
5609         foreach my $devname (keys %$devinfo) {
5610             my $volid = $devinfo->{$devname}->{volid};
5611             next if !$volid;
5612             eval {
5613                 if ($volid =~ m|^/|) {
5614                     unlink $volid || die 'unlink failed\n';
5615                 } else {
5616                     PVE::Storage::vdisk_free($cfg, $volid);
5617                 }
5618                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5619             };
5620             print STDERR "unable to cleanup '$volid' - $@" if $@;
5621         }
5622         die $err;
5623     }
5624
5625     rmtree $tmpdir;
5626
5627     rename($tmpfn, $conffile) ||
5628         die "unable to commit configuration file '$conffile'\n";
5629
5630     PVE::Cluster::cfs_update(); # make sure we read new file
5631
5632     eval { rescan($vmid, 1); };
5633     warn $@ if $@;
5634 }
5635
5636 sub restore_tar_archive {
5637     my ($archive, $vmid, $user, $opts) = @_;
5638
5639     if ($archive ne '-') {
5640         my $firstfile = tar_archive_read_firstfile($archive);
5641         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5642             if $firstfile ne 'qemu-server.conf';
5643     }
5644
5645     my $storecfg = cfs_read_file('storage.cfg');
5646
5647     # destroy existing data - keep empty config
5648     my $vmcfgfn = config_file($vmid);
5649     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5650
5651     my $tocmd = "/usr/lib/qemu-server/qmextract";
5652
5653     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5654     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5655     $tocmd .= ' --prealloc' if $opts->{prealloc};
5656     $tocmd .= ' --info' if $opts->{info};
5657
5658     # tar option "xf" does not autodetect compression when read from STDIN,
5659     # so we pipe to zcat
5660     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5661         PVE::Tools::shellquote("--to-command=$tocmd");
5662
5663     my $tmpdir = "/var/tmp/vzdumptmp$$";
5664     mkpath $tmpdir;
5665
5666     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5667     local $ENV{VZDUMP_VMID} = $vmid;
5668     local $ENV{VZDUMP_USER} = $user;
5669
5670     my $conffile = config_file($vmid);
5671     my $tmpfn = "$conffile.$$.tmp";
5672
5673     # disable interrupts (always do cleanups)
5674     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5675         print STDERR "got interrupt - ignored\n";
5676     };
5677
5678     eval {
5679         # enable interrupts
5680         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5681             die "interrupted by signal\n";
5682         };
5683
5684         if ($archive eq '-') {
5685             print "extracting archive from STDIN\n";
5686             run_command($cmd, input => "<&STDIN");
5687         } else {
5688             print "extracting archive '$archive'\n";
5689             run_command($cmd);
5690         }
5691
5692         return if $opts->{info};
5693
5694         # read new mapping
5695         my $map = {};
5696         my $statfile = "$tmpdir/qmrestore.stat";
5697         if (my $fd = IO::File->new($statfile, "r")) {
5698             while (defined (my $line = <$fd>)) {
5699                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5700                     $map->{$1} = $2 if $1;
5701                 } else {
5702                     print STDERR "unable to parse line in statfile - $line\n";
5703                 }
5704             }
5705             $fd->close();
5706         }
5707
5708         my $confsrc = "$tmpdir/qemu-server.conf";
5709
5710         my $srcfd = new IO::File($confsrc, "r") ||
5711             die "unable to open file '$confsrc'\n";
5712
5713         my $outfd = new IO::File ($tmpfn, "w") ||
5714             die "unable to write config for VM $vmid\n";
5715
5716         my $cookie = { netcount => 0 };
5717         while (defined (my $line = <$srcfd>)) {
5718             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5719         }
5720
5721         $srcfd->close();
5722         $outfd->close();
5723     };
5724     my $err = $@;
5725
5726     if ($err) {
5727
5728         unlink $tmpfn;
5729
5730         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5731
5732         die $err;
5733     }
5734
5735     rmtree $tmpdir;
5736
5737     rename $tmpfn, $conffile ||
5738         die "unable to commit configuration file '$conffile'\n";
5739
5740     PVE::Cluster::cfs_update(); # make sure we read new file
5741
5742     eval { rescan($vmid, 1); };
5743     warn $@ if $@;
5744 };
5745
5746
5747 # Internal snapshots
5748
5749 # NOTE: Snapshot create/delete involves several non-atomic
5750 # action, and can take a long time.
5751 # So we try to avoid locking the file and use 'lock' variable
5752 # inside the config file instead.
5753
5754 my $snapshot_copy_config = sub {
5755     my ($source, $dest) = @_;
5756
5757     foreach my $k (keys %$source) {
5758         next if $k eq 'snapshots';
5759         next if $k eq 'snapstate';
5760         next if $k eq 'snaptime';
5761         next if $k eq 'vmstate';
5762         next if $k eq 'lock';
5763         next if $k eq 'digest';
5764         next if $k eq 'description';
5765         next if $k =~ m/^unused\d+$/;
5766
5767         $dest->{$k} = $source->{$k};
5768     }
5769 };
5770
5771 my $snapshot_apply_config = sub {
5772     my ($conf, $snap) = @_;
5773
5774     # copy snapshot list
5775     my $newconf = {
5776         snapshots => $conf->{snapshots},
5777     };
5778
5779     # keep description and list of unused disks
5780     foreach my $k (keys %$conf) {
5781         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5782         $newconf->{$k} = $conf->{$k};
5783     }
5784
5785     &$snapshot_copy_config($snap, $newconf);
5786
5787     return $newconf;
5788 };
5789
5790 sub foreach_writable_storage {
5791     my ($conf, $func) = @_;
5792
5793     my $sidhash = {};
5794
5795     foreach my $ds (keys %$conf) {
5796         next if !valid_drivename($ds);
5797
5798         my $drive = parse_drive($ds, $conf->{$ds});
5799         next if !$drive;
5800         next if drive_is_cdrom($drive);
5801
5802         my $volid = $drive->{file};
5803
5804         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5805         $sidhash->{$sid} = $sid if $sid;
5806     }
5807
5808     foreach my $sid (sort keys %$sidhash) {
5809         &$func($sid);
5810     }
5811 }
5812
5813 my $alloc_vmstate_volid = sub {
5814     my ($storecfg, $vmid, $conf, $snapname) = @_;
5815
5816     # Note: we try to be smart when selecting a $target storage
5817
5818     my $target;
5819
5820     # search shared storage first
5821     foreach_writable_storage($conf, sub {
5822         my ($sid) = @_;
5823         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5824         return if !$scfg->{shared};
5825
5826         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5827     });
5828
5829     if (!$target) {
5830         # now search local storage
5831         foreach_writable_storage($conf, sub {
5832             my ($sid) = @_;
5833             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5834             return if $scfg->{shared};
5835
5836             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5837         });
5838     }
5839
5840     $target = 'local' if !$target;
5841
5842     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5843     # we abort live save after $conf->{memory}, so we need at max twice that space
5844     my $size = $conf->{memory}*2 + $driver_state_size;
5845
5846     my $name = "vm-$vmid-state-$snapname";
5847     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5848     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5849     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5850
5851     return $volid;
5852 };
5853
5854 my $snapshot_prepare = sub {
5855     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5856
5857     my $snap;
5858
5859     my $updatefn =  sub {
5860
5861         my $conf = load_config($vmid);
5862
5863         die "you can't take a snapshot if it's a template\n"
5864             if is_template($conf);
5865
5866         check_lock($conf);
5867
5868         $conf->{lock} = 'snapshot';
5869
5870         die "snapshot name '$snapname' already used\n"
5871             if defined($conf->{snapshots}->{$snapname});
5872
5873         my $storecfg = PVE::Storage::config();
5874         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5875
5876         $snap = $conf->{snapshots}->{$snapname} = {};
5877
5878         if ($save_vmstate && check_running($vmid)) {
5879             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5880         }
5881
5882         &$snapshot_copy_config($conf, $snap);
5883
5884         $snap->{snapstate} = "prepare";
5885         $snap->{snaptime} = time();
5886         $snap->{description} = $comment if $comment;
5887
5888         # always overwrite machine if we save vmstate. This makes sure we
5889         # can restore it later using correct machine type
5890         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5891
5892         update_config_nolock($vmid, $conf, 1);
5893     };
5894
5895     lock_config($vmid, $updatefn);
5896
5897     return $snap;
5898 };
5899
5900 my $snapshot_commit = sub {
5901     my ($vmid, $snapname) = @_;
5902
5903     my $updatefn = sub {
5904
5905         my $conf = load_config($vmid);
5906
5907         die "missing snapshot lock\n"
5908             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5909
5910         my $has_machine_config = defined($conf->{machine});
5911
5912         my $snap = $conf->{snapshots}->{$snapname};
5913
5914         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5915
5916         die "wrong snapshot state\n"
5917             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5918
5919         delete $snap->{snapstate};
5920         delete $conf->{lock};
5921
5922         my $newconf = &$snapshot_apply_config($conf, $snap);
5923
5924         delete $newconf->{machine} if !$has_machine_config;
5925
5926         $newconf->{parent} = $snapname;
5927
5928         update_config_nolock($vmid, $newconf, 1);
5929     };
5930
5931     lock_config($vmid, $updatefn);
5932 };
5933
5934 sub snapshot_rollback {
5935     my ($vmid, $snapname) = @_;
5936
5937     my $prepare = 1;
5938
5939     my $storecfg = PVE::Storage::config();
5940
5941     my $conf = load_config($vmid);
5942
5943     my $get_snapshot_config = sub {
5944
5945         die "you can't rollback if vm is a template\n" if is_template($conf);
5946
5947         my $res = $conf->{snapshots}->{$snapname};
5948
5949         die "snapshot '$snapname' does not exist\n" if !defined($res);
5950
5951         return $res;
5952     };
5953
5954     my $snap = &$get_snapshot_config();
5955
5956     foreach_drive($snap, sub {
5957         my ($ds, $drive) = @_;
5958
5959         return if drive_is_cdrom($drive);
5960
5961         my $volid = $drive->{file};
5962
5963         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5964     });
5965
5966     my $updatefn = sub {
5967
5968         $conf = load_config($vmid);
5969
5970         $snap = &$get_snapshot_config();
5971
5972         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5973             if $snap->{snapstate};
5974
5975         if ($prepare) {
5976             check_lock($conf);
5977             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5978         }
5979
5980         die "unable to rollback vm $vmid: vm is running\n"
5981             if check_running($vmid);
5982
5983         if ($prepare) {
5984             $conf->{lock} = 'rollback';
5985         } else {
5986             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5987             delete $conf->{lock};
5988         }
5989
5990         my $forcemachine;
5991
5992         if (!$prepare) {
5993             my $has_machine_config = defined($conf->{machine});
5994
5995             # copy snapshot config to current config
5996             $conf = &$snapshot_apply_config($conf, $snap);
5997             $conf->{parent} = $snapname;
5998
5999             # Note: old code did not store 'machine', so we try to be smart
6000             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6001             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6002             # we remove the 'machine' configuration if not explicitly specified
6003             # in the original config.
6004             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6005         }
6006
6007         update_config_nolock($vmid, $conf, 1);
6008
6009         if (!$prepare && $snap->{vmstate}) {
6010             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6011             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6012         }
6013     };
6014
6015     lock_config($vmid, $updatefn);
6016
6017     foreach_drive($snap, sub {
6018         my ($ds, $drive) = @_;
6019
6020         return if drive_is_cdrom($drive);
6021
6022         my $volid = $drive->{file};
6023         my $device = "drive-$ds";
6024
6025         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6026     });
6027
6028     $prepare = 0;
6029     lock_config($vmid, $updatefn);
6030 }
6031
6032 my $savevm_wait = sub {
6033     my ($vmid) = @_;
6034
6035     for(;;) {
6036         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6037         if (!$stat->{status}) {
6038             die "savevm not active\n";
6039         } elsif ($stat->{status} eq 'active') {
6040             sleep(1);
6041             next;
6042         } elsif ($stat->{status} eq 'completed') {
6043             last;
6044         } else {
6045             die "query-savevm returned status '$stat->{status}'\n";
6046         }
6047     }
6048 };
6049
6050 sub do_snapshots_with_qemu {
6051     my ($storecfg, $volid) = @_;
6052
6053     my $storage_name = PVE::Storage::parse_volume_id($volid);
6054
6055     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6056         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6057         return 1;
6058     }
6059
6060     if ($volid =~ m/\.(qcow2|qed)$/){
6061         return 1;
6062     }
6063
6064     return undef;
6065 }
6066
6067 sub snapshot_create {
6068     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6069
6070     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6071
6072     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6073
6074     my $config = load_config($vmid);
6075
6076     my $running = check_running($vmid);
6077
6078     my $freezefs = $running && $config->{agent};
6079     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6080
6081     my $drivehash = {};
6082
6083     if ($freezefs) {
6084         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6085         warn "guest-fsfreeze-freeze problems - $@" if $@;
6086     }
6087
6088     eval {
6089         # create internal snapshots of all drives
6090
6091         my $storecfg = PVE::Storage::config();
6092
6093         if ($running) {
6094             if ($snap->{vmstate}) {
6095                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6096                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6097                 &$savevm_wait($vmid);
6098             } else {
6099                 vm_mon_cmd($vmid, "savevm-start");
6100             }
6101         };
6102
6103         foreach_drive($snap, sub {
6104             my ($ds, $drive) = @_;
6105
6106             return if drive_is_cdrom($drive);
6107
6108             my $volid = $drive->{file};
6109             my $device = "drive-$ds";
6110
6111             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6112             $drivehash->{$ds} = 1;
6113        });
6114     };
6115     my $err = $@;
6116
6117     if ($running) {
6118         eval { vm_mon_cmd($vmid, "savevm-end")  };
6119         warn $@ if $@;
6120
6121         if ($freezefs) {
6122             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6123             warn "guest-fsfreeze-thaw problems - $@" if $@;
6124         }
6125
6126         # savevm-end is async, we need to wait
6127         for (;;) {
6128             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6129             if (!$stat->{bytes}) {
6130                 last;
6131             } else {
6132                 print "savevm not yet finished\n";
6133                 sleep(1);
6134                 next;
6135             }
6136         }
6137     }
6138
6139     if ($err) {
6140         warn "snapshot create failed: starting cleanup\n";
6141         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6142         warn $@ if $@;
6143         die $err;
6144     }
6145
6146     &$snapshot_commit($vmid, $snapname);
6147 }
6148
6149 # Note: $drivehash is only set when called from snapshot_create.
6150 sub snapshot_delete {
6151     my ($vmid, $snapname, $force, $drivehash) = @_;
6152
6153     my $prepare = 1;
6154
6155     my $snap;
6156     my $unused = [];
6157
6158     my $unlink_parent = sub {
6159         my ($confref, $new_parent) = @_;
6160
6161         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6162             if ($new_parent) {
6163                 $confref->{parent} = $new_parent;
6164             } else {
6165                 delete $confref->{parent};
6166             }
6167         }
6168     };
6169
6170     my $updatefn =  sub {
6171         my ($remove_drive) = @_;
6172
6173         my $conf = load_config($vmid);
6174
6175         if (!$drivehash) {
6176             check_lock($conf);
6177             die "you can't delete a snapshot if vm is a template\n"
6178                 if is_template($conf);
6179         }
6180
6181         $snap = $conf->{snapshots}->{$snapname};
6182
6183         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6184
6185         # remove parent refs
6186         if (!$prepare) {
6187             &$unlink_parent($conf, $snap->{parent});
6188             foreach my $sn (keys %{$conf->{snapshots}}) {
6189                 next if $sn eq $snapname;
6190                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6191             }
6192         }
6193
6194         if ($remove_drive) {
6195             if ($remove_drive eq 'vmstate') {
6196                 delete $snap->{$remove_drive};
6197             } else {
6198                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6199                 my $volid = $drive->{file};
6200                 delete $snap->{$remove_drive};
6201                 add_unused_volume($conf, $volid);
6202             }
6203         }
6204
6205         if ($prepare) {
6206             $snap->{snapstate} = 'delete';
6207         } else {
6208             delete $conf->{snapshots}->{$snapname};
6209             delete $conf->{lock} if $drivehash;
6210             foreach my $volid (@$unused) {
6211                 add_unused_volume($conf, $volid);
6212             }
6213         }
6214
6215         update_config_nolock($vmid, $conf, 1);
6216     };
6217
6218     lock_config($vmid, $updatefn);
6219
6220     # now remove vmstate file
6221
6222     my $storecfg = PVE::Storage::config();
6223
6224     if ($snap->{vmstate}) {
6225         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6226         if (my $err = $@) {
6227             die $err if !$force;
6228             warn $err;
6229         }
6230         # save changes (remove vmstate from snapshot)
6231         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6232     };
6233
6234     # now remove all internal snapshots
6235     foreach_drive($snap, sub {
6236         my ($ds, $drive) = @_;
6237
6238         return if drive_is_cdrom($drive);
6239
6240         my $volid = $drive->{file};
6241         my $device = "drive-$ds";
6242
6243         if (!$drivehash || $drivehash->{$ds}) {
6244             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6245             if (my $err = $@) {
6246                 die $err if !$force;
6247                 warn $err;
6248             }
6249         }
6250
6251         # save changes (remove drive fron snapshot)
6252         lock_config($vmid, $updatefn, $ds) if !$force;
6253         push @$unused, $volid;
6254     });
6255
6256     # now cleanup config
6257     $prepare = 0;
6258     lock_config($vmid, $updatefn);
6259 }
6260
6261 sub has_feature {
6262     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6263
6264     my $err;
6265     foreach_drive($conf, sub {
6266         my ($ds, $drive) = @_;
6267
6268         return if drive_is_cdrom($drive);
6269         my $volid = $drive->{file};
6270         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6271     });
6272
6273     return $err ? 0 : 1;
6274 }
6275
6276 sub template_create {
6277     my ($vmid, $conf, $disk) = @_;
6278
6279     my $storecfg = PVE::Storage::config();
6280
6281     foreach_drive($conf, sub {
6282         my ($ds, $drive) = @_;
6283
6284         return if drive_is_cdrom($drive);
6285         return if $disk && $ds ne $disk;
6286
6287         my $volid = $drive->{file};
6288         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6289
6290         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6291         $drive->{file} = $voliddst;
6292         $conf->{$ds} = print_drive($vmid, $drive);
6293         update_config_nolock($vmid, $conf, 1);
6294     });
6295 }
6296
6297 sub is_template {
6298     my ($conf) = @_;
6299
6300     return 1 if defined $conf->{template} && $conf->{template} == 1;
6301 }
6302
6303 sub qemu_img_convert {
6304     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6305
6306     my $storecfg = PVE::Storage::config();
6307     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6308     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6309
6310     if ($src_storeid && $dst_storeid) {
6311
6312         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6313
6314         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6315         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6316
6317         my $src_format = qemu_img_format($src_scfg, $src_volname);
6318         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6319
6320         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6321         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6322
6323         my $cmd = [];
6324         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6325         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6326         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6327
6328         my $parser = sub {
6329             my $line = shift;
6330             if($line =~ m/\((\S+)\/100\%\)/){
6331                 my $percent = $1;
6332                 my $transferred = int($size * $percent / 100);
6333                 my $remaining = $size - $transferred;
6334
6335                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6336             }
6337
6338         };
6339
6340         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6341         my $err = $@;
6342         die "copy failed: $err" if $err;
6343     }
6344 }
6345
6346 sub qemu_img_format {
6347     my ($scfg, $volname) = @_;
6348
6349     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6350         return $1;
6351     } else {
6352         return "raw";
6353     }
6354 }
6355
6356 sub qemu_drive_mirror {
6357     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6358
6359     my $storecfg = PVE::Storage::config();
6360     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6361
6362     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6363
6364     my $format = qemu_img_format($dst_scfg, $dst_volname);
6365
6366     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6367
6368     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6369     $opts->{format} = $format if $format;
6370
6371     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6372
6373     eval {
6374     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6375         while (1) {
6376             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6377             my $stat = @$stats[0];
6378             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6379             die "error job is not mirroring" if $stat->{type} ne "mirror";
6380
6381             my $busy = $stat->{busy};
6382             my $ready = $stat->{ready};
6383
6384             if (my $total = $stat->{len}) {
6385                 my $transferred = $stat->{offset} || 0;
6386                 my $remaining = $total - $transferred;
6387                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6388
6389                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6390             }
6391
6392
6393             if ($stat->{ready} eq 'true') {
6394
6395                 last if $vmiddst != $vmid;
6396
6397                 # try to switch the disk if source and destination are on the same guest
6398                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6399                 last if !$@;
6400                 die $@ if $@ !~ m/cannot be completed/;
6401             }
6402             sleep 1;
6403         }
6404
6405
6406     };
6407     my $err = $@;
6408
6409     my $cancel_job = sub {
6410         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6411         while (1) {
6412             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6413             my $stat = @$stats[0];
6414             last if !$stat;
6415             sleep 1;
6416         }
6417     };
6418
6419     if ($err) {
6420         eval { &$cancel_job(); };
6421         die "mirroring error: $err";
6422     }
6423
6424     if ($vmiddst != $vmid) {
6425         # if we clone a disk for a new target vm, we don't switch the disk
6426         &$cancel_job(); # so we call block-job-cancel
6427     }
6428 }
6429
6430 sub clone_disk {
6431     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6432         $newvmid, $storage, $format, $full, $newvollist) = @_;
6433
6434     my $newvolid;
6435
6436     if (!$full) {
6437         print "create linked clone of drive $drivename ($drive->{file})\n";
6438         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6439         push @$newvollist, $newvolid;
6440     } else {
6441         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6442         $storeid = $storage if $storage;
6443
6444         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6445         if (!$format) {
6446             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6447             $format = qemu_img_format($scfg, $volname);
6448         }
6449
6450         # test if requested format is supported - else use default
6451         my $supported = grep { $_ eq $format } @$validFormats;
6452         $format = $defFormat if !$supported;
6453
6454         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6455
6456         print "create full clone of drive $drivename ($drive->{file})\n";
6457         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6458         push @$newvollist, $newvolid;
6459
6460         PVE::Storage::activate_volumes($storecfg, $newvollist);
6461
6462         if (!$running || $snapname) {
6463             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6464         } else {
6465             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6466         }
6467     }
6468
6469     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6470
6471     my $disk = $drive;
6472     $disk->{format} = undef;
6473     $disk->{file} = $newvolid;
6474     $disk->{size} = $size;
6475
6476     return $disk;
6477 }
6478
6479 # this only works if VM is running
6480 sub get_current_qemu_machine {
6481     my ($vmid) = @_;
6482
6483     my $cmd = { execute => 'query-machines', arguments => {} };
6484     my $res = vm_qmp_command($vmid, $cmd);
6485
6486     my ($current, $default);
6487     foreach my $e (@$res) {
6488         $default = $e->{name} if $e->{'is-default'};
6489         $current = $e->{name} if $e->{'is-current'};
6490     }
6491
6492     # fallback to the default machine if current is not supported by qemu
6493     return $current || $default || 'pc';
6494 }
6495
6496 sub qemu_machine_feature_enabled {
6497     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6498
6499     my $current_major;
6500     my $current_minor;
6501
6502     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6503
6504         $current_major = $3;
6505         $current_minor = $4;
6506
6507     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6508
6509         $current_major = $1;
6510         $current_minor = $2;
6511     }
6512
6513     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6514
6515
6516 }
6517
6518 sub qemu_machine_pxe {
6519     my ($vmid, $conf, $machine) = @_;
6520
6521     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6522
6523     foreach my $opt (keys %$conf) {
6524         next if $opt !~ m/^net(\d+)$/;
6525         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6526         next if !$net;
6527         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6528         return $machine.".pxe" if $romfile =~ m/pxe/;
6529         last;
6530     }
6531
6532 }
6533
6534 sub qemu_use_old_bios_files {
6535     my ($machine_type) = @_;
6536
6537     return if !$machine_type;
6538
6539     my $use_old_bios_files = undef;
6540
6541     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6542         $machine_type = $1;
6543         $use_old_bios_files = 1;
6544     } else {
6545         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6546         # load new efi bios files on migration. So this hack is required to allow
6547         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6548         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6549         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6550     }
6551
6552     return ($use_old_bios_files, $machine_type);
6553 }
6554
6555 sub lspci {
6556
6557     my $devices = {};
6558
6559     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6560             my (undef, $id, $function) = @_;
6561             my $res = { id => $id, function => $function};
6562             push @{$devices->{$id}}, $res;
6563     });
6564
6565     return $devices;
6566 }
6567
6568 sub vm_iothreads_list {
6569     my ($vmid) = @_;
6570
6571     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6572
6573     my $iothreads = {};
6574     foreach my $iothread (@$res) {
6575         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6576     }
6577
6578     return $iothreads;
6579 }
6580
6581 sub scsihw_infos {
6582     my ($conf, $drive) = @_;
6583
6584     my $maxdev = 0;
6585
6586     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6587         $maxdev = 7;
6588     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6589         $maxdev = 1;
6590     } else {
6591         $maxdev = 256;
6592     }
6593
6594     my $controller = int($drive->{index} / $maxdev);
6595     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6596
6597     return ($maxdev, $controller, $controller_prefix);
6598 }
6599
6600 # bash completion helper
6601
6602 sub complete_backup_archives {
6603     my ($cmdname, $pname, $cvalue) = @_;
6604
6605     my $cfg = PVE::Storage::config();
6606
6607     my $storeid;
6608
6609     if ($cvalue =~ m/^([^:]+):/) {
6610         $storeid = $1;
6611     }
6612
6613     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6614
6615     my $res = [];
6616     foreach my $id (keys %$data) {
6617         foreach my $item (@{$data->{$id}}) {
6618             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6619             push @$res, $item->{volid} if defined($item->{volid});
6620         }
6621     }
6622
6623     return $res;
6624 }
6625
6626 my $complete_vmid_full = sub {
6627     my ($running) = @_;
6628
6629     my $idlist = vmstatus();
6630
6631     my $res = [];
6632
6633     foreach my $id (keys %$idlist) {
6634         my $d = $idlist->{$id};
6635         if (defined($running)) {
6636             next if $d->{template};
6637             next if $running && $d->{status} ne 'running';
6638             next if !$running && $d->{status} eq 'running';
6639         }
6640         push @$res, $id;
6641
6642     }
6643     return $res;
6644 };
6645
6646 sub complete_vmid {
6647     return &$complete_vmid_full();
6648 }
6649
6650 sub complete_vmid_stopped {
6651     return &$complete_vmid_full(0);
6652 }
6653
6654 sub complete_vmid_running {
6655     return &$complete_vmid_full(1);
6656 }
6657
6658 sub complete_storage {
6659
6660     my $cfg = PVE::Storage::config();
6661     my $ids = $cfg->{ids};
6662
6663     my $res = [];
6664     foreach my $sid (keys %$ids) {
6665         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6666         next if !$ids->{$sid}->{content}->{images};
6667         push @$res, $sid;
6668     }
6669
6670     return $res;
6671 }
6672
6673 1;