PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393     bios => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [ qw(seabios ovmf) ],
 397         description => "Select BIOS implementation.",
 398         default => 'seabios',
 399     },
 400 };
 401
 402 # what about other qemu settings ?
 403 #cpu => 'string',
 404 #machine => 'string',
 405 #fda => 'file',
 406 #fdb => 'file',
 407 #mtdblock => 'file',
 408 #sd => 'file',
 409 #pflash => 'file',
 410 #snapshot => 'bool',
 411 #bootp => 'file',
 412 ##tftp => 'dir',
 413 ##smb => 'dir',
 414 #kernel => 'file',
 415 #append => 'string',
 416 #initrd => 'file',
 417 ##soundhw => 'string',
 418
 419 while (my ($k, $v) = each %$confdesc) {
 420     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 421 }
 422
 423 my $MAX_IDE_DISKS = 4;
 424 my $MAX_SCSI_DISKS = 14;
 425 my $MAX_VIRTIO_DISKS = 16;
 426 my $MAX_SATA_DISKS = 6;
 427 my $MAX_USB_DEVICES = 5;
 428 my $MAX_NETS = 32;
 429 my $MAX_UNUSED_DISKS = 8;
 430 my $MAX_HOSTPCI_DEVICES = 4;
 431 my $MAX_SERIAL_PORTS = 4;
 432 my $MAX_PARALLEL_PORTS = 3;
 433 my $MAX_NUMA = 8;
 434 my $MAX_MEM = 4194304;
 435 my $STATICMEM = 1024;
 436
 437 my $numadesc = {
 438     optional => 1,
 439     type => 'string', format => 'pve-qm-numanode',
 440     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 441     description => "numa topology",
 442 };
 443 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 444
 445 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 446     $confdesc->{"numa$i"} = $numadesc;
 447 }
 448
 449 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 450                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 451                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 452 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 453
 454 my $netdesc = {
 455     optional => 1,
 456     type => 'string', format => 'pve-qm-net',
 457     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 458     description => <<EODESCR,
 459 Specify network devices.
 460
 461 MODEL is one of: $nic_model_list_txt
 462
 463 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 464 automatically generated if not specified.
 465
 466 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 467
 468 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 469
 470 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 471
 472 10.0.2.2   Gateway
 473 10.0.2.3   DNS Server
 474 10.0.2.4   SMB Server
 475
 476 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 477
 478 EODESCR
 479 };
 480 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 481
 482 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 483     $confdesc->{"net$i"} = $netdesc;
 484 }
 485
 486 my $drivename_hash;
 487
 488 my %drivedesc_base = (
 489     volume => { alias => 'file' },
 490     file => {
 491         type => 'pve-volume-id',
 492         default_key => 1,
 493         format_description => 'volume',
 494         description => "The drive's backing volume.",
 495     },
 496     media => {
 497         type => 'string',
 498         format_description => 'cdrom|disk',
 499         enum => [qw(cdrom disk)],
 500         description => "The drive's media type.",
 501         default => 'disk',
 502         optional => 1
 503     },
 504     cyls => {
 505         type => 'integer',
 506         format_description => 'count',
 507         description => "Force the drive's physical geometry to have a specific cylinder count.",
 508         optional => 1
 509     },
 510     heads => {
 511         type => 'integer',
 512         format_description => 'count',
 513         description => "Force the drive's physical geometry to have a specific head count.",
 514         optional => 1
 515     },
 516     secs => {
 517         type => 'integer',
 518         format_description => 'count',
 519         description => "Force the drive's physical geometry to have a specific sector count.",
 520         optional => 1
 521     },
 522     trans => {
 523         type => 'string',
 524         format_description => 'none|lba|auto',
 525         enum => [qw(none lba auto)],
 526         description => "Force disk geometry bios translation mode.",
 527         optional => 1,
 528     },
 529     snapshot => {
 530         type => 'boolean',
 531         format_description => 'on|off',
 532         description => "Whether the drive should be included when making snapshots.",
 533         optional => 1,
 534     },
 535     cache => {
 536         type => 'string',
 537         format_description => 'none|writethrough|writeback|unsafe|directsync',
 538         enum => [qw(none writethrough writeback unsafe directsync)],
 539         description => "The drive's cache mode",
 540         optional => 1,
 541     },
 542     format => {
 543         type => 'string',
 544         format_description => 'drive format',
 545         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 546         description => "The drive's backing file's data format.",
 547         optional => 1,
 548     },
 549     size => {
 550         type => 'disk-size',
 551         description => "Disk size. This is purely informational and has no effect.",
 552         optional => 1,
 553     },
 554     backup => {
 555         type => 'boolean',
 556         format_description => 'on|off',
 557         description => "Whether the drive should be included when making backups.",
 558         optional => 1,
 559     },
 560     werror => {
 561         type => 'string',
 562         format_description => 'enospc|ignore|report|stop',
 563         enum => [qw(enospc ignore report stop)],
 564         description => 'Write error action.',
 565         optional => 1,
 566     },
 567     aio => {
 568         type => 'string',
 569         format_description => 'native|threads',
 570         enum => [qw(native threads)],
 571         description => 'AIO type to use.',
 572         optional => 1,
 573     },
 574     discard => {
 575         type => 'string',
 576         format_description => 'ignore|on',
 577         enum => [qw(ignore on)],
 578         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 579         optional => 1,
 580     },
 581     detect_zeroes => {
 582         type => 'boolean',
 583         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 584         optional => 1,
 585     },
 586     serial => {
 587         type => 'string',
 588         format_description => 'serial',
 589         description => "The drive's reported serial number.",
 590         optional => 1,
 591     }
 592 );
 593
 594 my %rerror_fmt = (
 595     rerror => {
 596         type => 'string',
 597         format_description => 'ignore|report|stop',
 598         enum => [qw(ignore report stop)],
 599         description => 'Read error action.',
 600         optional => 1,
 601     },
 602 );
 603
 604 my %iothread_fmt = ( iothread => {
 605         type => 'boolean',
 606         format_description => 'off|on',
 607         description => "Whether to use iothreads for this drive",
 608         optional => 1,
 609 });
 610
 611 my %model_fmt = (
 612     model => {
 613         type => 'string',
 614         format_description => 'model',
 615         description => "The drive's reported model name.",
 616         optional => 1,
 617     },
 618 );
 619
 620 my %queues_fmt = (
 621     queues => {
 622         type => 'integer',
 623         format_description => 'nbqueues',
 624         description => "Number of queues.",
 625         minimum => 2,
 626         optional => 1
 627     }
 628 );
 629
 630 my $add_throttle_desc = sub {
 631     my ($key, $type, $what, $size, $longsize) = @_;
 632     $drivedesc_base{$key} = {
 633         type => $type,
 634         format_description => $size,
 635         description => "Maximum $what speed in $longsize per second.",
 636         optional => 1,
 637     };
 638 };
 639 # throughput: (leaky bucket)
 640 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 641 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 642 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 643 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 644 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 645 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 646 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 647 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 648 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 649
 650 # pools: (pool of IO before throttling starts taking effect)
 651 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 652 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 653 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 654 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 655 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 656 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 657
 658 my $ide_fmt = {
 659     %drivedesc_base,
 660     %rerror_fmt,
 661     %model_fmt,
 662 };
 663
 664 my $idedesc = {
 665     optional => 1,
 666     type => 'string', format => $ide_fmt,
 667     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 668 };
 669 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 670
 671 my $scsi_fmt = {
 672     %drivedesc_base,
 673     %iothread_fmt,
 674     %queues_fmt,
 675 };
 676 my $scsidesc = {
 677     optional => 1,
 678     type => 'string', format => $scsi_fmt,
 679     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 680 };
 681 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 682
 683 my $sata_fmt = {
 684     %drivedesc_base,
 685     %rerror_fmt,
 686 };
 687 my $satadesc = {
 688     optional => 1,
 689     type => 'string', format => $sata_fmt,
 690     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 691 };
 692 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 693
 694 my $virtio_fmt = {
 695     %drivedesc_base,
 696     %iothread_fmt,
 697     %rerror_fmt,
 698 };
 699 my $virtiodesc = {
 700     optional => 1,
 701     type => 'string', format => $virtio_fmt,
 702     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 703 };
 704 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 705
 706 my $alldrive_fmt = {
 707     %drivedesc_base,
 708     %rerror_fmt,
 709     %iothread_fmt,
 710     %model_fmt,
 711     %queues_fmt,
 712 };
 713
 714 my $usbdesc = {
 715     optional => 1,
 716     type => 'string', format => 'pve-qm-usb-device',
 717     typetext => 'host=HOSTUSBDEVICE|spice',
 718     description => <<EODESCR,
 719 Configure an USB device (n is 0 to 4). This can be used to
 720 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 721
 722 'bus-port(.port)*' (decimal numbers) or
 723 'vendor_id:product_id' (hexadeciaml numbers)
 724
 725 You can use the 'lsusb -t' command to list existing usb devices.
 726
 727 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 728
 729 The value 'spice' can be used to add a usb redirection devices for spice.
 730
 731 EODESCR
 732 };
 733 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 734
 735 my $hostpcidesc = {
 736         optional => 1,
 737         type => 'string', format => 'pve-qm-hostpci',
 738         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 739         description => <<EODESCR,
 740 Map host pci devices. HOSTPCIDEVICE syntax is:
 741
 742 'bus:dev.func' (hexadecimal numbers)
 743
 744 You can us the 'lspci' command to list existing pci devices.
 745
 746 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 747
 748 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 749
 750 Experimental: user reported problems with this option.
 751 EODESCR
 752 };
 753 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 754
 755 my $serialdesc = {
 756         optional => 1,
 757         type => 'string',
 758         pattern => '(/dev/.+|socket)',
 759         description =>  <<EODESCR,
 760 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 761
 762 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 763
 764 Experimental: user reported problems with this option.
 765 EODESCR
 766 };
 767
 768 my $paralleldesc= {
 769         optional => 1,
 770         type => 'string',
 771         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 772         description =>  <<EODESCR,
 773 Map host parallel devices (n is 0 to 2).
 774
 775 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 776
 777 Experimental: user reported problems with this option.
 778 EODESCR
 779 };
 780
 781 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 782     $confdesc->{"parallel$i"} = $paralleldesc;
 783 }
 784
 785 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 786     $confdesc->{"serial$i"} = $serialdesc;
 787 }
 788
 789 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 790     $confdesc->{"hostpci$i"} = $hostpcidesc;
 791 }
 792
 793 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 794     $drivename_hash->{"ide$i"} = 1;
 795     $confdesc->{"ide$i"} = $idedesc;
 796 }
 797
 798 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 799     $drivename_hash->{"sata$i"} = 1;
 800     $confdesc->{"sata$i"} = $satadesc;
 801 }
 802
 803 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 804     $drivename_hash->{"scsi$i"} = 1;
 805     $confdesc->{"scsi$i"} = $scsidesc ;
 806 }
 807
 808 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 809     $drivename_hash->{"virtio$i"} = 1;
 810     $confdesc->{"virtio$i"} = $virtiodesc;
 811 }
 812
 813 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 814     $confdesc->{"usb$i"} = $usbdesc;
 815 }
 816
 817 my $unuseddesc = {
 818     optional => 1,
 819     type => 'string', format => 'pve-volume-id',
 820     description => "Reference to unused volumes.",
 821 };
 822
 823 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 824     $confdesc->{"unused$i"} = $unuseddesc;
 825 }
 826
 827 my $kvm_api_version = 0;
 828
 829 sub kvm_version {
 830
 831     return $kvm_api_version if $kvm_api_version;
 832
 833     my $fh = IO::File->new("</dev/kvm") ||
 834         return 0;
 835
 836     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 837         $kvm_api_version = $v;
 838     }
 839
 840     $fh->close();
 841
 842     return  $kvm_api_version;
 843 }
 844
 845 my $kvm_user_version;
 846
 847 sub kvm_user_version {
 848
 849     return $kvm_user_version if $kvm_user_version;
 850
 851     $kvm_user_version = 'unknown';
 852
 853     my $tmp = `kvm -help 2>/dev/null`;
 854
 855     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 856         $kvm_user_version = $2;
 857     }
 858
 859     return $kvm_user_version;
 860
 861 }
 862
 863 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 864
 865 sub disknames {
 866     # order is important - used to autoselect boot disk
 867     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 868             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 869             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 870             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 871 }
 872
 873 sub valid_drivename {
 874     my $dev = shift;
 875
 876     return defined($drivename_hash->{$dev});
 877 }
 878
 879 sub option_exists {
 880     my $key = shift;
 881     return defined($confdesc->{$key});
 882 }
 883
 884 sub nic_models {
 885     return $nic_model_list;
 886 }
 887
 888 sub os_list_description {
 889
 890     return {
 891         other => 'Other',
 892         wxp => 'Windows XP',
 893         w2k => 'Windows 2000',
 894         w2k3 =>, 'Windows 2003',
 895         w2k8 => 'Windows 2008',
 896         wvista => 'Windows Vista',
 897         win7 => 'Windows 7',
 898         win8 => 'Windows 8/2012',
 899         l24 => 'Linux 2.4',
 900         l26 => 'Linux 2.6',
 901     };
 902 }
 903
 904 my $cdrom_path;
 905
 906 sub get_cdrom_path {
 907
 908     return  $cdrom_path if $cdrom_path;
 909
 910     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 911     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 912     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 913 }
 914
 915 sub get_iso_path {
 916     my ($storecfg, $vmid, $cdrom) = @_;
 917
 918     if ($cdrom eq 'cdrom') {
 919         return get_cdrom_path();
 920     } elsif ($cdrom eq 'none') {
 921         return '';
 922     } elsif ($cdrom =~ m|^/|) {
 923         return $cdrom;
 924     } else {
 925         return PVE::Storage::path($storecfg, $cdrom);
 926     }
 927 }
 928
 929 # try to convert old style file names to volume IDs
 930 sub filename_to_volume_id {
 931     my ($vmid, $file, $media) = @_;
 932
 933     if (!($file eq 'none' || $file eq 'cdrom' ||
 934           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 935
 936         return undef if $file =~ m|/|;
 937
 938         if ($media && $media eq 'cdrom') {
 939             $file = "local:iso/$file";
 940         } else {
 941             $file = "local:$vmid/$file";
 942         }
 943     }
 944
 945     return $file;
 946 }
 947
 948 sub verify_media_type {
 949     my ($opt, $vtype, $media) = @_;
 950
 951     return if !$media;
 952
 953     my $etype;
 954     if ($media eq 'disk') {
 955         $etype = 'images';
 956     } elsif ($media eq 'cdrom') {
 957         $etype = 'iso';
 958     } else {
 959         die "internal error";
 960     }
 961
 962     return if ($vtype eq $etype);
 963
 964     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 965 }
 966
 967 sub cleanup_drive_path {
 968     my ($opt, $storecfg, $drive) = @_;
 969
 970     # try to convert filesystem paths to volume IDs
 971
 972     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 973         ($drive->{file} !~ m|^/dev/.+|) &&
 974         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 975         ($drive->{file} !~ m/^\d+$/)) {
 976         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 977         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 978         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 979         verify_media_type($opt, $vtype, $drive->{media});
 980         $drive->{file} = $volid;
 981     }
 982
 983     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 984 }
 985
 986 sub create_conf_nolock {
 987     my ($vmid, $settings) = @_;
 988
 989     my $filename = config_file($vmid);
 990
 991     die "configuration file '$filename' already exists\n" if -f $filename;
 992
 993     my $defaults = load_defaults();
 994
 995     $settings->{name} = "vm$vmid" if !$settings->{name};
 996     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 997
 998     my $data = '';
 999     foreach my $opt (keys %$settings) {
1000         next if !$confdesc->{$opt};
1001
1002         my $value = $settings->{$opt};
1003         next if !$value;
1004
1005         $data .= "$opt: $value\n";
1006     }
1007
1008     PVE::Tools::file_set_contents($filename, $data);
1009 }
1010
1011 sub parse_hotplug_features {
1012     my ($data) = @_;
1013
1014     my $res = {};
1015
1016     return $res if $data eq '0';
1017
1018     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1019
1020     foreach my $feature (PVE::Tools::split_list($data)) {
1021         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1022             $res->{$1} = 1;
1023         } else {
1024             warn "ignoring unknown hotplug feature '$feature'\n";
1025         }
1026     }
1027     return $res;
1028 }
1029
1030 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1031 sub pve_verify_hotplug_features {
1032     my ($value, $noerr) = @_;
1033
1034     return $value if parse_hotplug_features($value);
1035
1036     return undef if $noerr;
1037
1038     die "unable to parse hotplug option\n";
1039 }
1040
1041 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1042 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1043 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1044 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1045 #        [,iothread=on][,serial=serial][,model=model]
1046
1047 sub parse_drive {
1048     my ($key, $data) = @_;
1049
1050     my ($interface, $index);
1051
1052     if ($key =~ m/^([^\d]+)(\d+)$/) {
1053         $interface = $1;
1054         $index = $2;
1055     } else {
1056         return undef;
1057     }
1058
1059     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1060                                      : $confdesc->{$key}->{format};
1061     if (!$desc) {
1062         warn "invalid drive key: $key\n";
1063         return undef;
1064     }
1065     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1066     return undef if !$res;
1067     $res->{interface} = $interface;
1068     $res->{index} = $index;
1069
1070     my $error = 0;
1071     foreach my $opt (qw(bps bps_rd bps_wr)) {
1072         if (my $bps = defined(delete $res->{$opt})) {
1073             if (defined($res->{"m$opt"})) {
1074                 warn "both $opt and m$opt specified\n";
1075                 ++$error;
1076                 next;
1077             }
1078             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1079         }
1080     }
1081     return undef if $error;
1082
1083     return undef if $res->{mbps_rd} && $res->{mbps};
1084     return undef if $res->{mbps_wr} && $res->{mbps};
1085     return undef if $res->{iops_rd} && $res->{iops};
1086     return undef if $res->{iops_wr} && $res->{iops};
1087
1088     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1089         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1090         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1091         return undef if $res->{interface} eq 'virtio';
1092     }
1093
1094     if (my $size = $res->{size}) {
1095         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1096     }
1097
1098     return $res;
1099 }
1100
1101 sub print_drive {
1102     my ($vmid, $drive) = @_;
1103     my $data = { %$drive };
1104     delete $data->{$_} for qw(index interface);
1105     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1106 }
1107
1108 sub scsi_inquiry {
1109     my($fh, $noerr) = @_;
1110
1111     my $SG_IO = 0x2285;
1112     my $SG_GET_VERSION_NUM = 0x2282;
1113
1114     my $versionbuf = "\x00" x 8;
1115     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1116     if (!$ret) {
1117         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1118         return undef;
1119     }
1120     my $version = unpack("I", $versionbuf);
1121     if ($version < 30000) {
1122         die "scsi generic interface too old\n"  if !$noerr;
1123         return undef;
1124     }
1125
1126     my $buf = "\x00" x 36;
1127     my $sensebuf = "\x00" x 8;
1128     my $cmd = pack("C x3 C x1", 0x12, 36);
1129
1130     # see /usr/include/scsi/sg.h
1131     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1132
1133     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1134                       length($sensebuf), 0, length($buf), $buf,
1135                       $cmd, $sensebuf, 6000);
1136
1137     $ret = ioctl($fh, $SG_IO, $packet);
1138     if (!$ret) {
1139         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1140         return undef;
1141     }
1142
1143     my @res = unpack($sg_io_hdr_t, $packet);
1144     if ($res[17] || $res[18]) {
1145         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1146         return undef;
1147     }
1148
1149     my $res = {};
1150     (my $byte0, my $byte1, $res->{vendor},
1151      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1152
1153     $res->{removable} = $byte1 & 128 ? 1 : 0;
1154     $res->{type} = $byte0 & 31;
1155
1156     return $res;
1157 }
1158
1159 sub path_is_scsi {
1160     my ($path) = @_;
1161
1162     my $fh = IO::File->new("+<$path") || return undef;
1163     my $res = scsi_inquiry($fh, 1);
1164     close($fh);
1165
1166     return $res;
1167 }
1168
1169 sub machine_type_is_q35 {
1170     my ($conf) = @_;
1171
1172     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1173 }
1174
1175 sub print_tabletdevice_full {
1176     my ($conf) = @_;
1177
1178     my $q35 = machine_type_is_q35($conf);
1179
1180     # we use uhci for old VMs because tablet driver was buggy in older qemu
1181     my $usbbus = $q35 ? "ehci" : "uhci";
1182
1183     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1184 }
1185
1186 sub print_drivedevice_full {
1187     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1188
1189     my $device = '';
1190     my $maxdev = 0;
1191
1192     if ($drive->{interface} eq 'virtio') {
1193         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1194         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1195         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1196     } elsif ($drive->{interface} eq 'scsi') {
1197
1198         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1199         my $unit = $drive->{index} % $maxdev;
1200         my $devicetype = 'hd';
1201         my $path = '';
1202         if (drive_is_cdrom($drive)) {
1203               $devicetype = 'cd';
1204         } else {
1205               if ($drive->{file} =~ m|^/|) {
1206                   $path = $drive->{file};
1207               } else {
1208                   $path = PVE::Storage::path($storecfg, $drive->{file});
1209               }
1210
1211               if($path =~ m/^iscsi\:\/\//){
1212                   $devicetype = 'generic';
1213               } else {
1214                   if (my $info = path_is_scsi($path)) {
1215                       if ($info->{type} == 0) {
1216                           $devicetype = 'block';
1217                       } elsif ($info->{type} == 1) { # tape
1218                           $devicetype = 'generic';
1219                       }
1220                   }
1221               }
1222          }
1223
1224         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1225             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1226         } else {
1227             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1228         }
1229
1230     } elsif ($drive->{interface} eq 'ide'){
1231         $maxdev = 2;
1232         my $controller = int($drive->{index} / $maxdev);
1233         my $unit = $drive->{index} % $maxdev;
1234         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1235
1236         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1237         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1238             $device .= ",model=$model";
1239         }
1240     } elsif ($drive->{interface} eq 'sata'){
1241         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1242         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1243         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1244     } elsif ($drive->{interface} eq 'usb') {
1245         die "implement me";
1246         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1247     } else {
1248         die "unsupported interface type";
1249     }
1250
1251     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1252
1253     return $device;
1254 }
1255
1256 sub get_initiator_name {
1257     my $initiator;
1258
1259     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1260     while (defined(my $line = <$fh>)) {
1261         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1262         $initiator = $1;
1263         last;
1264     }
1265     $fh->close();
1266
1267     return $initiator;
1268 }
1269
1270 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1271 sub print_drive_full {
1272     my ($storecfg, $vmid, $drive) = @_;
1273
1274     my $path;
1275     my $volid = $drive->{file};
1276     my $format;
1277
1278     if (drive_is_cdrom($drive)) {
1279         $path = get_iso_path($storecfg, $vmid, $volid);
1280     } else {
1281         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1282         if ($storeid) {
1283             $path = PVE::Storage::path($storecfg, $volid);
1284             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1285             $format = qemu_img_format($scfg, $volname);
1286         } else {
1287             $path = $volid;
1288         }
1289    }
1290
1291     my $opts = '';
1292     foreach my $o (@qemu_drive_options) {
1293         next if $o eq 'bootindex';
1294         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1295     }
1296
1297     $opts .= ",format=$format" if $format && !$drive->{format};
1298
1299     foreach my $o (qw(bps bps_rd bps_wr)) {
1300         my $v = $drive->{"m$o"};
1301         $opts .= ",$o=" . int($v*1024*1024) if $v;
1302     }
1303
1304     my $cache_direct = 0;
1305
1306     if (my $cache = $drive->{cache}) {
1307         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1308     } elsif (!drive_is_cdrom($drive)) {
1309         $opts .= ",cache=none";
1310         $cache_direct = 1;
1311     }
1312
1313     # aio native works only with O_DIRECT
1314     if (!$drive->{aio}) {
1315         if($cache_direct) {
1316             $opts .= ",aio=native";
1317         } else {
1318             $opts .= ",aio=threads";
1319         }
1320     }
1321
1322     if (!drive_is_cdrom($drive)) {
1323         my $detectzeroes;
1324         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1325             $detectzeroes = 'off';
1326         } elsif ($drive->{discard}) {
1327             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1328         } else {
1329             # This used to be our default with discard not being specified:
1330             $detectzeroes = 'on';
1331         }
1332         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1333     }
1334
1335     my $pathinfo = $path ? "file=$path," : '';
1336
1337     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1338 }
1339
1340 sub print_netdevice_full {
1341     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1342
1343     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1344
1345     my $device = $net->{model};
1346     if ($net->{model} eq 'virtio') {
1347          $device = 'virtio-net-pci';
1348      };
1349
1350     my $pciaddr = print_pci_addr("$netid", $bridges);
1351     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1352     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1353         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1354         my $vectors = $net->{queues} * 2 + 2;
1355         $tmpstr .= ",vectors=$vectors,mq=on";
1356     }
1357     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1358
1359     if ($use_old_bios_files) {
1360         my $romfile;
1361         if ($device eq 'virtio-net-pci') {
1362             $romfile = 'pxe-virtio.rom';
1363         } elsif ($device eq 'e1000') {
1364             $romfile = 'pxe-e1000.rom';
1365         } elsif ($device eq 'ne2k') {
1366             $romfile = 'pxe-ne2k_pci.rom';
1367         } elsif ($device eq 'pcnet') {
1368             $romfile = 'pxe-pcnet.rom';
1369         } elsif ($device eq 'rtl8139') {
1370             $romfile = 'pxe-rtl8139.rom';
1371         }
1372         $tmpstr .= ",romfile=$romfile" if $romfile;
1373     }
1374
1375     return $tmpstr;
1376 }
1377
1378 sub print_netdev_full {
1379     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1380
1381     my $i = '';
1382     if ($netid =~ m/^net(\d+)$/) {
1383         $i = int($1);
1384     }
1385
1386     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1387
1388     my $ifname = "tap${vmid}i$i";
1389
1390     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1391     die "interface name '$ifname' is too long (max 15 character)\n"
1392         if length($ifname) >= 16;
1393
1394     my $vhostparam = '';
1395     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1396
1397     my $vmname = $conf->{name} || "vm$vmid";
1398
1399     my $netdev = "";
1400     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1401
1402     if ($net->{bridge}) {
1403         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1404     } else {
1405         $netdev = "type=user,id=$netid,hostname=$vmname";
1406     }
1407
1408     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1409
1410     return $netdev;
1411 }
1412
1413 sub drive_is_cdrom {
1414     my ($drive) = @_;
1415
1416     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1417
1418 }
1419
1420 sub parse_numa {
1421     my ($data) = @_;
1422
1423     my $res = {};
1424
1425     foreach my $kvp (split(/,/, $data)) {
1426
1427         if ($kvp =~ m/^memory=(\S+)$/) {
1428             $res->{memory} = $1;
1429         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1430             $res->{policy} = $1;
1431         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1432             $res->{cpus}->{start} = $1;
1433             $res->{cpus}->{end} = $3;
1434         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1435             $res->{hostnodes}->{start} = $1;
1436             $res->{hostnodes}->{end} = $3;
1437         } else {
1438             return undef;
1439         }
1440     }
1441
1442     return $res;
1443 }
1444
1445 sub parse_hostpci {
1446     my ($value) = @_;
1447
1448     return undef if !$value;
1449
1450
1451     my @list = split(/,/, $value);
1452     my $found;
1453
1454     my $res = {};
1455     foreach my $kv (@list) {
1456
1457         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1458             $found = 1;
1459             if(defined($4)){
1460                 push @{$res->{pciid}}, { id => $2 , function => $4};
1461
1462             }else{
1463                 my $pcidevices = lspci($2);
1464                 $res->{pciid} = $pcidevices->{$2};
1465             }
1466         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1467             $res->{rombar} = $1;
1468         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1469             $res->{'x-vga'} = $1;
1470         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1471             $res->{pcie} = 1 if $1 == 1;
1472         } else {
1473             warn "unknown hostpci setting '$kv'\n";
1474         }
1475     }
1476
1477     return undef if !$found;
1478
1479     return $res;
1480 }
1481
1482 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1483 sub parse_net {
1484     my ($data) = @_;
1485
1486     my $res = {};
1487
1488     foreach my $kvp (split(/,/, $data)) {
1489
1490         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1491             my $model = lc($1);
1492             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1493             $res->{model} = $model;
1494             $res->{macaddr} = $mac;
1495         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1496             $res->{bridge} = $1;
1497         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1498             $res->{queues} = $1;
1499         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1500             $res->{rate} = $1;
1501         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1502             $res->{tag} = $1;
1503         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1504             $res->{trunks} = $1;
1505         } elsif ($kvp =~ m/^firewall=([01])$/) {
1506             $res->{firewall} = $1;
1507         } elsif ($kvp =~ m/^link_down=([01])$/) {
1508             $res->{link_down} = $1;
1509         } else {
1510             return undef;
1511         }
1512
1513     }
1514
1515     return undef if !$res->{model};
1516
1517     return $res;
1518 }
1519
1520 sub print_net {
1521     my $net = shift;
1522
1523     my $res = "$net->{model}";
1524     $res .= "=$net->{macaddr}" if $net->{macaddr};
1525     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1526     $res .= ",rate=$net->{rate}" if $net->{rate};
1527     $res .= ",tag=$net->{tag}" if $net->{tag};
1528     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1529     $res .= ",firewall=1" if $net->{firewall};
1530     $res .= ",link_down=1" if $net->{link_down};
1531     $res .= ",queues=$net->{queues}" if $net->{queues};
1532
1533     return $res;
1534 }
1535
1536 sub add_random_macs {
1537     my ($settings) = @_;
1538
1539     foreach my $opt (keys %$settings) {
1540         next if $opt !~ m/^net(\d+)$/;
1541         my $net = parse_net($settings->{$opt});
1542         next if !$net;
1543         $settings->{$opt} = print_net($net);
1544     }
1545 }
1546
1547 sub add_unused_volume {
1548     my ($config, $volid) = @_;
1549
1550     my $key;
1551     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1552         my $test = "unused$ind";
1553         if (my $vid = $config->{$test}) {
1554             return if $vid eq $volid; # do not add duplicates
1555         } else {
1556             $key = $test;
1557         }
1558     }
1559
1560     die "To many unused volume - please delete them first.\n" if !$key;
1561
1562     $config->{$key} = $volid;
1563
1564     return $key;
1565 }
1566
1567 sub vm_is_volid_owner {
1568     my ($storecfg, $vmid, $volid) = @_;
1569
1570     if ($volid !~  m|^/|) {
1571         my ($path, $owner);
1572         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1573         if ($owner && ($owner == $vmid)) {
1574             return 1;
1575         }
1576     }
1577
1578     return undef;
1579 }
1580
1581 sub split_flagged_list {
1582     my $text = shift || '';
1583     $text =~ s/[,;]/ /g;
1584     $text =~ s/^\s+//;
1585     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1586 }
1587
1588 sub join_flagged_list {
1589     my ($how, $lst) = @_;
1590     join $how, map { $lst->{$_} . $_ } keys %$lst;
1591 }
1592
1593 sub vmconfig_delete_pending_option {
1594     my ($conf, $key, $force) = @_;
1595
1596     delete $conf->{pending}->{$key};
1597     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1598     $pending_delete_hash->{$key} = $force ? '!' : '';
1599     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1600 }
1601
1602 sub vmconfig_undelete_pending_option {
1603     my ($conf, $key) = @_;
1604
1605     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1606     delete $pending_delete_hash->{$key};
1607
1608     if (%$pending_delete_hash) {
1609         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1610     } else {
1611         delete $conf->{pending}->{delete};
1612     }
1613 }
1614
1615 sub vmconfig_register_unused_drive {
1616     my ($storecfg, $vmid, $conf, $drive) = @_;
1617
1618     if (!drive_is_cdrom($drive)) {
1619         my $volid = $drive->{file};
1620         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1621             add_unused_volume($conf, $volid, $vmid);
1622         }
1623     }
1624 }
1625
1626 sub vmconfig_cleanup_pending {
1627     my ($conf) = @_;
1628
1629     # remove pending changes when nothing changed
1630     my $changes;
1631     foreach my $opt (keys %{$conf->{pending}}) {
1632         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1633             $changes = 1;
1634             delete $conf->{pending}->{$opt};
1635         }
1636     }
1637
1638     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1639     my $pending_delete_hash = {};
1640     while (my ($opt, $force) = each %$current_delete_hash) {
1641         if (defined($conf->{$opt})) {
1642             $pending_delete_hash->{$opt} = $force;
1643         } else {
1644             $changes = 1;
1645         }
1646     }
1647
1648     if (%$pending_delete_hash) {
1649         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1650     } else {
1651         delete $conf->{pending}->{delete};
1652     }
1653
1654     return $changes;
1655 }
1656
1657 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1658 my $smbios1_desc = {
1659     uuid => {
1660         type => 'string',
1661         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1662         format_description => 'UUID',
1663         optional => 1,
1664     },
1665     version => {
1666         type => 'string',
1667         pattern => '\S+',
1668         format_description => 'str',
1669         optional => 1,
1670     },
1671     serial => {
1672         type => 'string',
1673         pattern => '\S+',
1674         format_description => 'str',
1675         optional => 1,
1676     },
1677     manufacturer => {
1678         type => 'string',
1679         pattern => '\S+',
1680         format_description => 'name',
1681         optional => 1,
1682     },
1683     product => {
1684         type => 'string',
1685         pattern => '\S+',
1686         format_description => 'name',
1687         optional => 1,
1688     },
1689     sku => {
1690         type => 'string',
1691         pattern => '\S+',
1692         format_description => 'str',
1693         optional => 1,
1694     },
1695     family => {
1696         type => 'string',
1697         pattern => '\S+',
1698         format_description => 'str',
1699         optional => 1,
1700     },
1701 };
1702
1703 sub parse_smbios1 {
1704     my ($data) = @_;
1705
1706     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1707     warn $@ if $@;
1708     return $res;
1709 }
1710
1711 sub print_smbios1 {
1712     my ($smbios1) = @_;
1713     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1714 }
1715
1716 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1717
1718 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1719 sub verify_bootdisk {
1720     my ($value, $noerr) = @_;
1721
1722     return $value if valid_drivename($value);
1723
1724     return undef if $noerr;
1725
1726     die "invalid boot disk '$value'\n";
1727 }
1728
1729 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1730 sub verify_numa {
1731     my ($value, $noerr) = @_;
1732
1733     return $value if parse_numa($value);
1734
1735     return undef if $noerr;
1736
1737     die "unable to parse numa options\n";
1738 }
1739
1740 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1741 sub verify_net {
1742     my ($value, $noerr) = @_;
1743
1744     return $value if parse_net($value);
1745
1746     return undef if $noerr;
1747
1748     die "unable to parse network options\n";
1749 }
1750
1751 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1752 sub verify_hostpci {
1753     my ($value, $noerr) = @_;
1754
1755     return $value if parse_hostpci($value);
1756
1757     return undef if $noerr;
1758
1759     die "unable to parse pci id\n";
1760 }
1761
1762 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1763 sub verify_watchdog {
1764     my ($value, $noerr) = @_;
1765
1766     return $value if parse_watchdog($value);
1767
1768     return undef if $noerr;
1769
1770     die "unable to parse watchdog options\n";
1771 }
1772
1773 sub parse_watchdog {
1774     my ($value) = @_;
1775
1776     return undef if !$value;
1777
1778     my $res = {};
1779
1780     foreach my $p (split(/,/, $value)) {
1781         next if $p =~ m/^\s*$/;
1782
1783         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1784             $res->{model} = $2;
1785         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1786             $res->{action} = $2;
1787         } else {
1788             return undef;
1789         }
1790     }
1791
1792     return $res;
1793 }
1794
1795 sub parse_usb_device {
1796     my ($value) = @_;
1797
1798     return undef if !$value;
1799
1800     my @dl = split(/,/, $value);
1801     my $found;
1802
1803     my $res = {};
1804     foreach my $v (@dl) {
1805         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1806             $found = 1;
1807             $res->{vendorid} = $2;
1808             $res->{productid} = $4;
1809         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1810             $found = 1;
1811             $res->{hostbus} = $1;
1812             $res->{hostport} = $2;
1813         } elsif ($v =~ m/^spice$/) {
1814             $found = 1;
1815             $res->{spice} = 1;
1816         } else {
1817             return undef;
1818         }
1819     }
1820     return undef if !$found;
1821
1822     return $res;
1823 }
1824
1825 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1826 sub verify_usb_device {
1827     my ($value, $noerr) = @_;
1828
1829     return $value if parse_usb_device($value);
1830
1831     return undef if $noerr;
1832
1833     die "unable to parse usb device\n";
1834 }
1835
1836 # add JSON properties for create and set function
1837 sub json_config_properties {
1838     my $prop = shift;
1839
1840     foreach my $opt (keys %$confdesc) {
1841         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1842         $prop->{$opt} = $confdesc->{$opt};
1843     }
1844
1845     return $prop;
1846 }
1847
1848 sub check_type {
1849     my ($key, $value) = @_;
1850
1851     die "unknown setting '$key'\n" if !$confdesc->{$key};
1852
1853     my $type = $confdesc->{$key}->{type};
1854
1855     if (!defined($value)) {
1856         die "got undefined value\n";
1857     }
1858
1859     if ($value =~ m/[\n\r]/) {
1860         die "property contains a line feed\n";
1861     }
1862
1863     if ($type eq 'boolean') {
1864         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1865         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1866         die "type check ('boolean') failed - got '$value'\n";
1867     } elsif ($type eq 'integer') {
1868         return int($1) if $value =~ m/^(\d+)$/;
1869         die "type check ('integer') failed - got '$value'\n";
1870     } elsif ($type eq 'number') {
1871         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1872         die "type check ('number') failed - got '$value'\n";
1873     } elsif ($type eq 'string') {
1874         if (my $fmt = $confdesc->{$key}->{format}) {
1875             if ($fmt eq 'pve-qm-drive') {
1876                 # special case - we need to pass $key to parse_drive()
1877                 my $drive = parse_drive($key, $value);
1878                 return $value if $drive;
1879                 die "unable to parse drive options\n";
1880             }
1881             PVE::JSONSchema::check_format($fmt, $value);
1882             return $value;
1883         }
1884         $value =~ s/^\"(.*)\"$/$1/;
1885         return $value;
1886     } else {
1887         die "internal error"
1888     }
1889 }
1890
1891 sub lock_config_full {
1892     my ($vmid, $timeout, $code, @param) = @_;
1893
1894     my $filename = config_file_lock($vmid);
1895
1896     my $res = lock_file($filename, $timeout, $code, @param);
1897
1898     die $@ if $@;
1899
1900     return $res;
1901 }
1902
1903 sub lock_config_mode {
1904     my ($vmid, $timeout, $shared, $code, @param) = @_;
1905
1906     my $filename = config_file_lock($vmid);
1907
1908     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1909
1910     die $@ if $@;
1911
1912     return $res;
1913 }
1914
1915 sub lock_config {
1916     my ($vmid, $code, @param) = @_;
1917
1918     return lock_config_full($vmid, 10, $code, @param);
1919 }
1920
1921 sub cfs_config_path {
1922     my ($vmid, $node) = @_;
1923
1924     $node = $nodename if !$node;
1925     return "nodes/$node/qemu-server/$vmid.conf";
1926 }
1927
1928 sub check_iommu_support{
1929     #fixme : need to check IOMMU support
1930     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1931
1932     my $iommu=1;
1933     return $iommu;
1934
1935 }
1936
1937 sub config_file {
1938     my ($vmid, $node) = @_;
1939
1940     my $cfspath = cfs_config_path($vmid, $node);
1941     return "/etc/pve/$cfspath";
1942 }
1943
1944 sub config_file_lock {
1945     my ($vmid) = @_;
1946
1947     return "$lock_dir/lock-$vmid.conf";
1948 }
1949
1950 sub touch_config {
1951     my ($vmid) = @_;
1952
1953     my $conf = config_file($vmid);
1954     utime undef, undef, $conf;
1955 }
1956
1957 sub destroy_vm {
1958     my ($storecfg, $vmid, $keep_empty_config) = @_;
1959
1960     my $conffile = config_file($vmid);
1961
1962     my $conf = load_config($vmid);
1963
1964     check_lock($conf);
1965
1966     # only remove disks owned by this VM
1967     foreach_drive($conf, sub {
1968         my ($ds, $drive) = @_;
1969
1970         return if drive_is_cdrom($drive);
1971
1972         my $volid = $drive->{file};
1973
1974         return if !$volid || $volid =~ m|^/|;
1975
1976         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1977         return if !$path || !$owner || ($owner != $vmid);
1978
1979         PVE::Storage::vdisk_free($storecfg, $volid);
1980     });
1981
1982     if ($keep_empty_config) {
1983         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1984     } else {
1985         unlink $conffile;
1986     }
1987
1988     # also remove unused disk
1989     eval {
1990         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1991
1992         eval {
1993             PVE::Storage::foreach_volid($dl, sub {
1994                 my ($volid, $sid, $volname, $d) = @_;
1995                 PVE::Storage::vdisk_free($storecfg, $volid);
1996             });
1997         };
1998         warn $@ if $@;
1999
2000     };
2001     warn $@ if $@;
2002 }
2003
2004 sub load_config {
2005     my ($vmid, $node) = @_;
2006
2007     my $cfspath = cfs_config_path($vmid, $node);
2008
2009     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2010
2011     die "no such VM ('$vmid')\n" if !defined($conf);
2012
2013     return $conf;
2014 }
2015
2016 sub parse_vm_config {
2017     my ($filename, $raw) = @_;
2018
2019     return undef if !defined($raw);
2020
2021     my $res = {
2022         digest => Digest::SHA::sha1_hex($raw),
2023         snapshots => {},
2024         pending => {},
2025     };
2026
2027     $filename =~ m|/qemu-server/(\d+)\.conf$|
2028         || die "got strange filename '$filename'";
2029
2030     my $vmid = $1;
2031
2032     my $conf = $res;
2033     my $descr;
2034     my $section = '';
2035
2036     my @lines = split(/\n/, $raw);
2037     foreach my $line (@lines) {
2038         next if $line =~ m/^\s*$/;
2039
2040         if ($line =~ m/^\[PENDING\]\s*$/i) {
2041             $section = 'pending';
2042             if (defined($descr)) {
2043                 $descr =~ s/\s+$//;
2044                 $conf->{description} = $descr;
2045             }
2046             $descr = undef;
2047             $conf = $res->{$section} = {};
2048             next;
2049
2050         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2051             $section = $1;
2052             if (defined($descr)) {
2053                 $descr =~ s/\s+$//;
2054                 $conf->{description} = $descr;
2055             }
2056             $descr = undef;
2057             $conf = $res->{snapshots}->{$section} = {};
2058             next;
2059         }
2060
2061         if ($line =~ m/^\#(.*)\s*$/) {
2062             $descr = '' if !defined($descr);
2063             $descr .= PVE::Tools::decode_text($1) . "\n";
2064             next;
2065         }
2066
2067         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2068             $descr = '' if !defined($descr);
2069             $descr .= PVE::Tools::decode_text($2);
2070         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2071             $conf->{snapstate} = $1;
2072         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2073             my $key = $1;
2074             my $value = $2;
2075             $conf->{$key} = $value;
2076         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2077             my $value = $1;
2078             if ($section eq 'pending') {
2079                 $conf->{delete} = $value; # we parse this later
2080             } else {
2081                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2082             }
2083         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2084             my $key = $1;
2085             my $value = $2;
2086             eval { $value = check_type($key, $value); };
2087             if ($@) {
2088                 warn "vm $vmid - unable to parse value of '$key' - $@";
2089             } else {
2090                 my $fmt = $confdesc->{$key}->{format};
2091                 if ($fmt && $fmt eq 'pve-qm-drive') {
2092                     my $v = parse_drive($key, $value);
2093                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2094                         $v->{file} = $volid;
2095                         $value = print_drive($vmid, $v);
2096                     } else {
2097                         warn "vm $vmid - unable to parse value of '$key'\n";
2098                         next;
2099                     }
2100                 }
2101
2102                 if ($key eq 'cdrom') {
2103                     $conf->{ide2} = $value;
2104                 } else {
2105                     $conf->{$key} = $value;
2106                 }
2107             }
2108         }
2109     }
2110
2111     if (defined($descr)) {
2112         $descr =~ s/\s+$//;
2113         $conf->{description} = $descr;
2114     }
2115     delete $res->{snapstate}; # just to be sure
2116
2117     return $res;
2118 }
2119
2120 sub write_vm_config {
2121     my ($filename, $conf) = @_;
2122
2123     delete $conf->{snapstate}; # just to be sure
2124
2125     if ($conf->{cdrom}) {
2126         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2127         $conf->{ide2} = $conf->{cdrom};
2128         delete $conf->{cdrom};
2129     }
2130
2131     # we do not use 'smp' any longer
2132     if ($conf->{sockets}) {
2133         delete $conf->{smp};
2134     } elsif ($conf->{smp}) {
2135         $conf->{sockets} = $conf->{smp};
2136         delete $conf->{cores};
2137         delete $conf->{smp};
2138     }
2139
2140     my $used_volids = {};
2141
2142     my $cleanup_config = sub {
2143         my ($cref, $pending, $snapname) = @_;
2144
2145         foreach my $key (keys %$cref) {
2146             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2147                 $key eq 'snapstate' || $key eq 'pending';
2148             my $value = $cref->{$key};
2149             if ($key eq 'delete') {
2150                 die "propertry 'delete' is only allowed in [PENDING]\n"
2151                     if !$pending;
2152                 # fixme: check syntax?
2153                 next;
2154             }
2155             eval { $value = check_type($key, $value); };
2156             die "unable to parse value of '$key' - $@" if $@;
2157
2158             $cref->{$key} = $value;
2159
2160             if (!$snapname && valid_drivename($key)) {
2161                 my $drive = parse_drive($key, $value);
2162                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2163             }
2164         }
2165     };
2166
2167     &$cleanup_config($conf);
2168
2169     &$cleanup_config($conf->{pending}, 1);
2170
2171     foreach my $snapname (keys %{$conf->{snapshots}}) {
2172         die "internal error" if $snapname eq 'pending';
2173         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2174     }
2175
2176     # remove 'unusedX' settings if we re-add a volume
2177     foreach my $key (keys %$conf) {
2178         my $value = $conf->{$key};
2179         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2180             delete $conf->{$key};
2181         }
2182     }
2183
2184     my $generate_raw_config = sub {
2185         my ($conf, $pending) = @_;
2186
2187         my $raw = '';
2188
2189         # add description as comment to top of file
2190         if (defined(my $descr = $conf->{description})) {
2191             if ($descr) {
2192                 foreach my $cl (split(/\n/, $descr)) {
2193                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2194                 }
2195             } else {
2196                 $raw .= "#\n" if $pending;
2197             }
2198         }
2199
2200         foreach my $key (sort keys %$conf) {
2201             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2202             $raw .= "$key: $conf->{$key}\n";
2203         }
2204         return $raw;
2205     };
2206
2207     my $raw = &$generate_raw_config($conf);
2208
2209     if (scalar(keys %{$conf->{pending}})){
2210         $raw .= "\n[PENDING]\n";
2211         $raw .= &$generate_raw_config($conf->{pending}, 1);
2212     }
2213
2214     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2215         $raw .= "\n[$snapname]\n";
2216         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2217     }
2218
2219     return $raw;
2220 }
2221
2222 sub update_config_nolock {
2223     my ($vmid, $conf, $skiplock) = @_;
2224
2225     check_lock($conf) if !$skiplock;
2226
2227     my $cfspath = cfs_config_path($vmid);
2228
2229     PVE::Cluster::cfs_write_file($cfspath, $conf);
2230 }
2231
2232 sub update_config {
2233     my ($vmid, $conf, $skiplock) = @_;
2234
2235     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2236 }
2237
2238 sub load_defaults {
2239
2240     my $res = {};
2241
2242     # we use static defaults from our JSON schema configuration
2243     foreach my $key (keys %$confdesc) {
2244         if (defined(my $default = $confdesc->{$key}->{default})) {
2245             $res->{$key} = $default;
2246         }
2247     }
2248
2249     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2250     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2251
2252     return $res;
2253 }
2254
2255 sub config_list {
2256     my $vmlist = PVE::Cluster::get_vmlist();
2257     my $res = {};
2258     return $res if !$vmlist || !$vmlist->{ids};
2259     my $ids = $vmlist->{ids};
2260
2261     foreach my $vmid (keys %$ids) {
2262         my $d = $ids->{$vmid};
2263         next if !$d->{node} || $d->{node} ne $nodename;
2264         next if !$d->{type} || $d->{type} ne 'qemu';
2265         $res->{$vmid}->{exists} = 1;
2266     }
2267     return $res;
2268 }
2269
2270 # test if VM uses local resources (to prevent migration)
2271 sub check_local_resources {
2272     my ($conf, $noerr) = @_;
2273
2274     my $loc_res = 0;
2275
2276     $loc_res = 1 if $conf->{hostusb}; # old syntax
2277     $loc_res = 1 if $conf->{hostpci}; # old syntax
2278
2279     foreach my $k (keys %$conf) {
2280         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2281         # sockets are safe: they will recreated be on the target side post-migrate
2282         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2283         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2284     }
2285
2286     die "VM uses local resources\n" if $loc_res && !$noerr;
2287
2288     return $loc_res;
2289 }
2290
2291 # check if used storages are available on all nodes (use by migrate)
2292 sub check_storage_availability {
2293     my ($storecfg, $conf, $node) = @_;
2294
2295     foreach_drive($conf, sub {
2296         my ($ds, $drive) = @_;
2297
2298         my $volid = $drive->{file};
2299         return if !$volid;
2300
2301         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2302         return if !$sid;
2303
2304         # check if storage is available on both nodes
2305         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2306         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2307    });
2308 }
2309
2310 # list nodes where all VM images are available (used by has_feature API)
2311 sub shared_nodes {
2312     my ($conf, $storecfg) = @_;
2313
2314     my $nodelist = PVE::Cluster::get_nodelist();
2315     my $nodehash = { map { $_ => 1 } @$nodelist };
2316     my $nodename = PVE::INotify::nodename();
2317
2318     foreach_drive($conf, sub {
2319         my ($ds, $drive) = @_;
2320
2321         my $volid = $drive->{file};
2322         return if !$volid;
2323
2324         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2325         if ($storeid) {
2326             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2327             if ($scfg->{disable}) {
2328                 $nodehash = {};
2329             } elsif (my $avail = $scfg->{nodes}) {
2330                 foreach my $node (keys %$nodehash) {
2331                     delete $nodehash->{$node} if !$avail->{$node};
2332                 }
2333             } elsif (!$scfg->{shared}) {
2334                 foreach my $node (keys %$nodehash) {
2335                     delete $nodehash->{$node} if $node ne $nodename
2336                 }
2337             }
2338         }
2339     });
2340
2341     return $nodehash
2342 }
2343
2344 sub check_lock {
2345     my ($conf) = @_;
2346
2347     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2348 }
2349
2350 sub check_cmdline {
2351     my ($pidfile, $pid) = @_;
2352
2353     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2354     if (defined($fh)) {
2355         my $line = <$fh>;
2356         $fh->close;
2357         return undef if !$line;
2358         my @param = split(/\0/, $line);
2359
2360         my $cmd = $param[0];
2361         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2362
2363         for (my $i = 0; $i < scalar (@param); $i++) {
2364             my $p = $param[$i];
2365             next if !$p;
2366             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2367                 my $p = $param[$i+1];
2368                 return 1 if $p && ($p eq $pidfile);
2369                 return undef;
2370             }
2371         }
2372     }
2373     return undef;
2374 }
2375
2376 sub check_running {
2377     my ($vmid, $nocheck, $node) = @_;
2378
2379     my $filename = config_file($vmid, $node);
2380
2381     die "unable to find configuration file for VM $vmid - no such machine\n"
2382         if !$nocheck && ! -f $filename;
2383
2384     my $pidfile = pidfile_name($vmid);
2385
2386     if (my $fd = IO::File->new("<$pidfile")) {
2387         my $st = stat($fd);
2388         my $line = <$fd>;
2389         close($fd);
2390
2391         my $mtime = $st->mtime;
2392         if ($mtime > time()) {
2393             warn "file '$filename' modified in future\n";
2394         }
2395
2396         if ($line =~ m/^(\d+)$/) {
2397             my $pid = $1;
2398             if (check_cmdline($pidfile, $pid)) {
2399                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2400                     return $pid;
2401                 }
2402             }
2403         }
2404     }
2405
2406     return undef;
2407 }
2408
2409 sub vzlist {
2410
2411     my $vzlist = config_list();
2412
2413     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2414
2415     while (defined(my $de = $fd->read)) {
2416         next if $de !~ m/^(\d+)\.pid$/;
2417         my $vmid = $1;
2418         next if !defined($vzlist->{$vmid});
2419         if (my $pid = check_running($vmid)) {
2420             $vzlist->{$vmid}->{pid} = $pid;
2421         }
2422     }
2423
2424     return $vzlist;
2425 }
2426
2427 sub disksize {
2428     my ($storecfg, $conf) = @_;
2429
2430     my $bootdisk = $conf->{bootdisk};
2431     return undef if !$bootdisk;
2432     return undef if !valid_drivename($bootdisk);
2433
2434     return undef if !$conf->{$bootdisk};
2435
2436     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2437     return undef if !defined($drive);
2438
2439     return undef if drive_is_cdrom($drive);
2440
2441     my $volid = $drive->{file};
2442     return undef if !$volid;
2443
2444     return $drive->{size};
2445 }
2446
2447 my $last_proc_pid_stat;
2448
2449 # get VM status information
2450 # This must be fast and should not block ($full == false)
2451 # We only query KVM using QMP if $full == true (this can be slow)
2452 sub vmstatus {
2453     my ($opt_vmid, $full) = @_;
2454
2455     my $res = {};
2456
2457     my $storecfg = PVE::Storage::config();
2458
2459     my $list = vzlist();
2460     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2461
2462     my $cpucount = $cpuinfo->{cpus} || 1;
2463
2464     foreach my $vmid (keys %$list) {
2465         next if $opt_vmid && ($vmid ne $opt_vmid);
2466
2467         my $cfspath = cfs_config_path($vmid);
2468         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2469
2470         my $d = {};
2471         $d->{pid} = $list->{$vmid}->{pid};
2472
2473         # fixme: better status?
2474         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2475
2476         my $size = disksize($storecfg, $conf);
2477         if (defined($size)) {
2478             $d->{disk} = 0; # no info available
2479             $d->{maxdisk} = $size;
2480         } else {
2481             $d->{disk} = 0;
2482             $d->{maxdisk} = 0;
2483         }
2484
2485         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2486         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2487         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2488
2489         $d->{name} = $conf->{name} || "VM $vmid";
2490         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2491
2492         if ($conf->{balloon}) {
2493             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2494             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2495         }
2496
2497         $d->{uptime} = 0;
2498         $d->{cpu} = 0;
2499         $d->{mem} = 0;
2500
2501         $d->{netout} = 0;
2502         $d->{netin} = 0;
2503
2504         $d->{diskread} = 0;
2505         $d->{diskwrite} = 0;
2506
2507         $d->{template} = is_template($conf);
2508
2509         $res->{$vmid} = $d;
2510     }
2511
2512     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2513     foreach my $dev (keys %$netdev) {
2514         next if $dev !~ m/^tap([1-9]\d*)i/;
2515         my $vmid = $1;
2516         my $d = $res->{$vmid};
2517         next if !$d;
2518
2519         $d->{netout} += $netdev->{$dev}->{receive};
2520         $d->{netin} += $netdev->{$dev}->{transmit};
2521
2522         if ($full) {
2523             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2524             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2525         }
2526
2527     }
2528
2529     my $ctime = gettimeofday;
2530
2531     foreach my $vmid (keys %$list) {
2532
2533         my $d = $res->{$vmid};
2534         my $pid = $d->{pid};
2535         next if !$pid;
2536
2537         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2538         next if !$pstat; # not running
2539
2540         my $used = $pstat->{utime} + $pstat->{stime};
2541
2542         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2543
2544         if ($pstat->{vsize}) {
2545             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2546         }
2547
2548         my $old = $last_proc_pid_stat->{$pid};
2549         if (!$old) {
2550             $last_proc_pid_stat->{$pid} = {
2551                 time => $ctime,
2552                 used => $used,
2553                 cpu => 0,
2554             };
2555             next;
2556         }
2557
2558         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2559
2560         if ($dtime > 1000) {
2561             my $dutime = $used -  $old->{used};
2562
2563             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2564             $last_proc_pid_stat->{$pid} = {
2565                 time => $ctime,
2566                 used => $used,
2567                 cpu => $d->{cpu},
2568             };
2569         } else {
2570             $d->{cpu} = $old->{cpu};
2571         }
2572     }
2573
2574     return $res if !$full;
2575
2576     my $qmpclient = PVE::QMPClient->new();
2577
2578     my $ballooncb = sub {
2579         my ($vmid, $resp) = @_;
2580
2581         my $info = $resp->{'return'};
2582         return if !$info->{max_mem};
2583
2584         my $d = $res->{$vmid};
2585
2586         # use memory assigned to VM
2587         $d->{maxmem} = $info->{max_mem};
2588         $d->{balloon} = $info->{actual};
2589
2590         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2591             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2592             $d->{freemem} = $info->{free_mem};
2593         }
2594
2595         $d->{ballooninfo} = $info;
2596     };
2597
2598     my $blockstatscb = sub {
2599         my ($vmid, $resp) = @_;
2600         my $data = $resp->{'return'} || [];
2601         my $totalrdbytes = 0;
2602         my $totalwrbytes = 0;
2603
2604         for my $blockstat (@$data) {
2605             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2606             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2607
2608             $blockstat->{device} =~ s/drive-//;
2609             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2610         }
2611         $res->{$vmid}->{diskread} = $totalrdbytes;
2612         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2613     };
2614
2615     my $statuscb = sub {
2616         my ($vmid, $resp) = @_;
2617
2618         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2619         # this fails if ballon driver is not loaded, so this must be
2620         # the last commnand (following command are aborted if this fails).
2621         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2622
2623         my $status = 'unknown';
2624         if (!defined($status = $resp->{'return'}->{status})) {
2625             warn "unable to get VM status\n";
2626             return;
2627         }
2628
2629         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2630     };
2631
2632     foreach my $vmid (keys %$list) {
2633         next if $opt_vmid && ($vmid ne $opt_vmid);
2634         next if !$res->{$vmid}->{pid}; # not running
2635         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2636     }
2637
2638     $qmpclient->queue_execute(undef, 1);
2639
2640     foreach my $vmid (keys %$list) {
2641         next if $opt_vmid && ($vmid ne $opt_vmid);
2642         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2643     }
2644
2645     return $res;
2646 }
2647
2648 sub foreach_dimm {
2649     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2650
2651     my $dimm_id = 0;
2652     my $current_size = 1024;
2653     my $dimm_size = 512;
2654     return if $current_size == $memory;
2655
2656     for (my $j = 0; $j < 8; $j++) {
2657         for (my $i = 0; $i < 32; $i++) {
2658             my $name = "dimm${dimm_id}";
2659             $dimm_id++;
2660             my $numanode = $i % $sockets;
2661             $current_size += $dimm_size;
2662             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2663             return  $current_size if $current_size >= $memory;
2664         }
2665         $dimm_size *= 2;
2666     }
2667 }
2668
2669 sub foreach_reverse_dimm {
2670     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2671
2672     my $dimm_id = 253;
2673     my $current_size = 4177920;
2674     my $dimm_size = 65536;
2675     return if $current_size == $memory;
2676
2677     for (my $j = 0; $j < 8; $j++) {
2678         for (my $i = 0; $i < 32; $i++) {
2679             my $name = "dimm${dimm_id}";
2680             $dimm_id--;
2681             my $numanode = $i % $sockets;
2682             $current_size -= $dimm_size;
2683             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2684             return  $current_size if $current_size <= $memory;
2685         }
2686         $dimm_size /= 2;
2687     }
2688 }
2689
2690 sub foreach_drive {
2691     my ($conf, $func) = @_;
2692
2693     foreach my $ds (keys %$conf) {
2694         next if !valid_drivename($ds);
2695
2696         my $drive = parse_drive($ds, $conf->{$ds});
2697         next if !$drive;
2698
2699         &$func($ds, $drive);
2700     }
2701 }
2702
2703 sub foreach_volid {
2704     my ($conf, $func) = @_;
2705
2706     my $volhash = {};
2707
2708     my $test_volid = sub {
2709         my ($volid, $is_cdrom) = @_;
2710
2711         return if !$volid;
2712
2713         $volhash->{$volid} = $is_cdrom || 0;
2714     };
2715
2716     foreach_drive($conf, sub {
2717         my ($ds, $drive) = @_;
2718         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2719     });
2720
2721     foreach my $snapname (keys %{$conf->{snapshots}}) {
2722         my $snap = $conf->{snapshots}->{$snapname};
2723         &$test_volid($snap->{vmstate}, 0);
2724         foreach_drive($snap, sub {
2725             my ($ds, $drive) = @_;
2726             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2727         });
2728     }
2729
2730     foreach my $volid (keys %$volhash) {
2731         &$func($volid, $volhash->{$volid});
2732     }
2733 }
2734
2735 sub vga_conf_has_spice {
2736     my ($vga) = @_;
2737
2738     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2739
2740     return $1 || 1;
2741 }
2742
2743 sub config_to_command {
2744     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2745
2746     my $cmd = [];
2747     my $globalFlags = [];
2748     my $machineFlags = [];
2749     my $rtcFlags = [];
2750     my $cpuFlags = [];
2751     my $devices = [];
2752     my $pciaddr = '';
2753     my $bridges = {};
2754     my $kvmver = kvm_user_version();
2755     my $vernum = 0; # unknown
2756     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2757         $vernum = $1*1000000+$2*1000;
2758     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2759         $vernum = $1*1000000+$2*1000+$3;
2760     }
2761
2762     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2763
2764     my $have_ovz = -f '/proc/vz/vestat';
2765
2766     my $q35 = machine_type_is_q35($conf);
2767     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2768     my $machine_type = $forcemachine || $conf->{machine};
2769     my $use_old_bios_files = undef;
2770     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2771
2772     my $cpuunits = defined($conf->{cpuunits}) ?
2773             $conf->{cpuunits} : $defaults->{cpuunits};
2774
2775     push @$cmd, '/usr/bin/systemd-run';
2776     push @$cmd, '--scope';
2777     push @$cmd, '--slice', "qemu";
2778     push @$cmd, '--unit', $vmid;
2779     # set KillMode=none, so that systemd don't kill those scopes
2780     # at shutdown (pve-manager service should stop the VMs instead)
2781     push @$cmd, '-p', "KillMode=none";
2782     push @$cmd, '-p', "CPUShares=$cpuunits";
2783     if ($conf->{cpulimit}) {
2784         my $cpulimit = int($conf->{cpulimit} * 100);
2785         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2786     }
2787
2788     push @$cmd, '/usr/bin/kvm';
2789
2790     push @$cmd, '-id', $vmid;
2791
2792     my $use_virtio = 0;
2793
2794     my $qmpsocket = qmp_socket($vmid);
2795     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2796     push @$cmd, '-mon', "chardev=qmp,mode=control";
2797
2798     my $socket = vnc_socket($vmid);
2799     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2800
2801     push @$cmd, '-pidfile' , pidfile_name($vmid);
2802
2803     push @$cmd, '-daemonize';
2804
2805     if ($conf->{smbios1}) {
2806         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2807     }
2808
2809     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2810         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2811         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2812         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2813         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2814         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2815         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2816     }
2817
2818     if ($q35) {
2819         # the q35 chipset support native usb2, so we enable usb controller
2820         # by default for this machine type
2821         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2822     } else {
2823         $pciaddr = print_pci_addr("piix3", $bridges);
2824         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2825
2826         my $use_usb2 = 0;
2827         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2828             next if !$conf->{"usb$i"};
2829             $use_usb2 = 1;
2830         }
2831         # include usb device config
2832         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2833     }
2834
2835     my $vga = $conf->{vga};
2836
2837     my $qxlnum = vga_conf_has_spice($vga);
2838     $vga = 'qxl' if $qxlnum;
2839
2840     if (!$vga) {
2841         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2842                                 $conf->{ostype} eq 'win7' ||
2843                                 $conf->{ostype} eq 'w2k8')) {
2844             $vga = 'std';
2845         } else {
2846             $vga = 'cirrus';
2847         }
2848     }
2849
2850     # enable absolute mouse coordinates (needed by vnc)
2851     my $tablet;
2852     if (defined($conf->{tablet})) {
2853         $tablet = $conf->{tablet};
2854     } else {
2855         $tablet = $defaults->{tablet};
2856         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2857         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2858     }
2859
2860     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2861
2862     my $nohyperv;
2863     # host pci devices
2864     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2865         my $d = parse_hostpci($conf->{"hostpci$i"});
2866         next if !$d;
2867
2868         my $pcie = $d->{pcie};
2869         if($pcie){
2870             die "q35 machine model is not enabled" if !$q35;
2871             $pciaddr = print_pcie_addr("hostpci$i");
2872         }else{
2873             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2874         }
2875
2876         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2877         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2878         if ($xvga && $xvga ne '') {
2879             push @$cpuFlags, 'kvm=off';
2880             $vga = 'none';
2881             $nohyperv = 1;
2882             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2883                 $xvga = "";
2884             }
2885         }
2886         my $pcidevices = $d->{pciid};
2887         my $multifunction = 1 if @$pcidevices > 1;
2888
2889         my $j=0;
2890         foreach my $pcidevice (@$pcidevices) {
2891
2892             my $id = "hostpci$i";
2893             $id .= ".$j" if $multifunction;
2894             my $addr = $pciaddr;
2895             $addr .= ".$j" if $multifunction;
2896             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2897
2898             if($j == 0){
2899                 $devicestr .= "$rombar$xvga";
2900                 $devicestr .= ",multifunction=on" if $multifunction;
2901             }
2902
2903             push @$devices, '-device', $devicestr;
2904             $j++;
2905         }
2906     }
2907
2908     # usb devices
2909     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2910         my $d = parse_usb_device($conf->{"usb$i"});
2911         next if !$d;
2912         if ($d->{vendorid} && $d->{productid}) {
2913             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2914         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2915             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2916         } elsif ($d->{spice}) {
2917             # usb redir support for spice
2918             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2919             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2920         }
2921     }
2922
2923     # serial devices
2924     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2925         if (my $path = $conf->{"serial$i"}) {
2926             if ($path eq 'socket') {
2927                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2928                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2929                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2930             } else {
2931                 die "no such serial device\n" if ! -c $path;
2932                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2933                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2934             }
2935         }
2936     }
2937
2938     # parallel devices
2939     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2940         if (my $path = $conf->{"parallel$i"}) {
2941             die "no such parallel device\n" if ! -c $path;
2942             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2943             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2944             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2945         }
2946     }
2947
2948     my $vmname = $conf->{name} || "vm$vmid";
2949
2950     push @$cmd, '-name', $vmname;
2951
2952     my $sockets = 1;
2953     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2954     $sockets = $conf->{sockets} if  $conf->{sockets};
2955
2956     my $cores = $conf->{cores} || 1;
2957
2958     my $maxcpus = $sockets * $cores;
2959
2960     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2961
2962     my $allowed_vcpus = $cpuinfo->{cpus};
2963
2964     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2965         if ($allowed_vcpus < $maxcpus);
2966
2967     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2968
2969     push @$cmd, '-nodefaults';
2970
2971     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2972
2973     my $bootindex_hash = {};
2974     my $i = 1;
2975     foreach my $o (split(//, $bootorder)) {
2976         $bootindex_hash->{$o} = $i*100;
2977         $i++;
2978     }
2979
2980     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2981
2982     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2983
2984     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2985
2986     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2987
2988     # time drift fix
2989     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2990
2991     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2992     my $useLocaltime = $conf->{localtime};
2993
2994     if (my $ost = $conf->{ostype}) {
2995         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2996
2997         if ($ost =~ m/^w/) { # windows
2998             $useLocaltime = 1 if !defined($conf->{localtime});
2999
3000             # use time drift fix when acpi is enabled
3001             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3002                 $tdf = 1 if !defined($conf->{tdf});
3003             }
3004         }
3005
3006         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
3007             $ost eq 'wvista') {
3008             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3009             push @$cmd, '-no-hpet';
3010             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3011                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
3012                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
3013                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
3014
3015             } else {
3016                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
3017             }
3018         }
3019
3020         if ($ost eq 'win7' || $ost eq 'win8') {
3021             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
3022         }
3023     }
3024
3025     push @$rtcFlags, 'driftfix=slew' if $tdf;
3026
3027     if ($nokvm) {
3028         push @$machineFlags, 'accel=tcg';
3029     } else {
3030         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3031     }
3032
3033     if ($machine_type) {
3034         push @$machineFlags, "type=${machine_type}";
3035     }
3036
3037     if ($conf->{startdate}) {
3038         push @$rtcFlags, "base=$conf->{startdate}";
3039     } elsif ($useLocaltime) {
3040         push @$rtcFlags, 'base=localtime';
3041     }
3042
3043     my $cpu = $nokvm ? "qemu64" : "kvm64";
3044     $cpu = $conf->{cpu} if $conf->{cpu};
3045
3046     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3047
3048     push @$cpuFlags , '-x2apic'
3049         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3050
3051     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3052
3053     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3054
3055     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3056
3057         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3058         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3059     }
3060
3061     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3062
3063     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3064
3065     push @$cmd, '-cpu', $cpu;
3066
3067     my $memory = $conf->{memory} || $defaults->{memory};
3068     my $static_memory = 0;
3069     my $dimm_memory = 0;
3070
3071     if ($hotplug_features->{memory}) {
3072         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3073         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3074         $static_memory = $STATICMEM;
3075         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3076         $dimm_memory = $memory - $static_memory;
3077         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3078
3079     } else {
3080
3081         $static_memory = $memory;
3082         push @$cmd, '-m', $static_memory;
3083     }
3084
3085     if ($conf->{numa}) {
3086
3087         my $numa_totalmemory = undef;
3088         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3089             next if !$conf->{"numa$i"};
3090             my $numa = parse_numa($conf->{"numa$i"});
3091             next if !$numa;
3092             # memory
3093             die "missing numa node$i memory value\n" if !$numa->{memory};
3094             my $numa_memory = $numa->{memory};
3095             $numa_totalmemory += $numa_memory;
3096             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3097
3098             # cpus
3099             my $cpus_start = $numa->{cpus}->{start};
3100             die "missing numa node$i cpus\n" if !defined($cpus_start);
3101             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3102             my $cpus = $cpus_start;
3103             if (defined($cpus_end)) {
3104                 $cpus .= "-$cpus_end";
3105                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3106             }
3107
3108             # hostnodes
3109             my $hostnodes_start = $numa->{hostnodes}->{start};
3110             if (defined($hostnodes_start)) {
3111                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3112                 my $hostnodes = $hostnodes_start;
3113                 if (defined($hostnodes_end)) {
3114                     $hostnodes .= "-$hostnodes_end";
3115                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3116                 }
3117
3118                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3119                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3120                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3121                 }
3122
3123                 # policy
3124                 my $policy = $numa->{policy};
3125                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3126                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3127             }
3128
3129             push @$cmd, '-object', $numa_object;
3130             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3131         }
3132
3133         die "total memory for NUMA nodes must be equal to vm static memory\n"
3134             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3135
3136         #if no custom tology, we split memory and cores across numa nodes
3137         if(!$numa_totalmemory) {
3138
3139             my $numa_memory = ($static_memory / $sockets) . "M";
3140
3141             for (my $i = 0; $i < $sockets; $i++)  {
3142
3143                 my $cpustart = ($cores * $i);
3144                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3145                 my $cpus = $cpustart;
3146                 $cpus .= "-$cpuend" if $cpuend;
3147
3148                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3149                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3150             }
3151         }
3152     }
3153
3154     if ($hotplug_features->{memory}) {
3155         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3156             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3157             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3158             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3159
3160             #if dimm_memory is not aligned to dimm map
3161             if($current_size > $memory) {
3162                  $conf->{memory} = $current_size;
3163                  update_config_nolock($vmid, $conf, 1);
3164             }
3165         });
3166     }
3167
3168     push @$cmd, '-S' if $conf->{freeze};
3169
3170     # set keyboard layout
3171     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3172     push @$cmd, '-k', $kb if $kb;
3173
3174     # enable sound
3175     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3176     #push @$cmd, '-soundhw', 'es1370';
3177     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3178
3179     if($conf->{agent}) {
3180         my $qgasocket = qmp_socket($vmid, 1);
3181         my $pciaddr = print_pci_addr("qga0", $bridges);
3182         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3183         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3184         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3185     }
3186
3187     my $spice_port;
3188
3189     if ($qxlnum) {
3190         if ($qxlnum > 1) {
3191             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3192                 for(my $i = 1; $i < $qxlnum; $i++){
3193                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3194                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3195                 }
3196             } else {
3197                 # assume other OS works like Linux
3198                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3199                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3200             }
3201         }
3202
3203         my $pciaddr = print_pci_addr("spice", $bridges);
3204
3205         my $nodename = PVE::INotify::nodename();
3206         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3207         $spice_port = PVE::Tools::next_spice_port($pfamily);
3208
3209         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3210
3211         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3212         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3213         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3214     }
3215
3216     # enable balloon by default, unless explicitly disabled
3217     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3218         $pciaddr = print_pci_addr("balloon0", $bridges);
3219         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3220     }
3221
3222     if ($conf->{watchdog}) {
3223         my $wdopts = parse_watchdog($conf->{watchdog});
3224         $pciaddr = print_pci_addr("watchdog", $bridges);
3225         my $watchdog = $wdopts->{model} || 'i6300esb';
3226         push @$devices, '-device', "$watchdog$pciaddr";
3227         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3228     }
3229
3230     my $vollist = [];
3231     my $scsicontroller = {};
3232     my $ahcicontroller = {};
3233     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3234
3235     # Add iscsi initiator name if available
3236     if (my $initiator = get_initiator_name()) {
3237         push @$devices, '-iscsi', "initiator-name=$initiator";
3238     }
3239
3240     foreach_drive($conf, sub {
3241         my ($ds, $drive) = @_;
3242
3243         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3244             push @$vollist, $drive->{file};
3245         }
3246
3247         $use_virtio = 1 if $ds =~ m/^virtio/;
3248
3249         if (drive_is_cdrom ($drive)) {
3250             if ($bootindex_hash->{d}) {
3251                 $drive->{bootindex} = $bootindex_hash->{d};
3252                 $bootindex_hash->{d} += 1;
3253             }
3254         } else {
3255             if ($bootindex_hash->{c}) {
3256                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3257                 $bootindex_hash->{c} += 1;
3258             }
3259         }
3260
3261         if($drive->{interface} eq 'virtio'){
3262            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3263         }
3264
3265         if ($drive->{interface} eq 'scsi') {
3266
3267             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3268
3269             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3270             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3271
3272             my $iothread = '';
3273             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3274                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3275                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3276             }
3277
3278             my $queues = '';
3279             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3280                 $queues = ",num_queues=$drive->{queues}";
3281             }
3282
3283             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3284             $scsicontroller->{$controller}=1;
3285         }
3286
3287         if ($drive->{interface} eq 'sata') {
3288            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3289            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3290            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3291            $ahcicontroller->{$controller}=1;
3292         }
3293
3294         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3295         push @$devices, '-drive',$drive_cmd;
3296         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3297     });
3298
3299     for (my $i = 0; $i < $MAX_NETS; $i++) {
3300          next if !$conf->{"net$i"};
3301          my $d = parse_net($conf->{"net$i"});
3302          next if !$d;
3303
3304          $use_virtio = 1 if $d->{model} eq 'virtio';
3305
3306          if ($bootindex_hash->{n}) {
3307             $d->{bootindex} = $bootindex_hash->{n};
3308             $bootindex_hash->{n} += 1;
3309          }
3310
3311          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3312          push @$devices, '-netdev', $netdevfull;
3313
3314          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3315          push @$devices, '-device', $netdevicefull;
3316     }
3317
3318     if (!$q35) {
3319         # add pci bridges
3320         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3321            $bridges->{1} = 1;
3322            $bridges->{2} = 1;
3323         }
3324
3325         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3326
3327         while (my ($k, $v) = each %$bridges) {
3328             $pciaddr = print_pci_addr("pci.$k");
3329             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3330         }
3331     }
3332
3333     # add custom args
3334     if ($conf->{args}) {
3335         my $aa = PVE::Tools::split_args($conf->{args});
3336         push @$cmd, @$aa;
3337     }
3338
3339     push @$cmd, @$devices;
3340     push @$cmd, '-rtc', join(',', @$rtcFlags)
3341         if scalar(@$rtcFlags);
3342     push @$cmd, '-machine', join(',', @$machineFlags)
3343         if scalar(@$machineFlags);
3344     push @$cmd, '-global', join(',', @$globalFlags)
3345         if scalar(@$globalFlags);
3346
3347     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3348 }
3349
3350 sub vnc_socket {
3351     my ($vmid) = @_;
3352     return "${var_run_tmpdir}/$vmid.vnc";
3353 }
3354
3355 sub spice_port {
3356     my ($vmid) = @_;
3357
3358     my $res = vm_mon_cmd($vmid, 'query-spice');
3359
3360     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3361 }
3362
3363 sub qmp_socket {
3364     my ($vmid, $qga) = @_;
3365     my $sockettype = $qga ? 'qga' : 'qmp';
3366     return "${var_run_tmpdir}/$vmid.$sockettype";
3367 }
3368
3369 sub pidfile_name {
3370     my ($vmid) = @_;
3371     return "${var_run_tmpdir}/$vmid.pid";
3372 }
3373
3374 sub vm_devices_list {
3375     my ($vmid) = @_;
3376
3377     my $res = vm_mon_cmd($vmid, 'query-pci');
3378     my $devices = {};
3379     foreach my $pcibus (@$res) {
3380         foreach my $device (@{$pcibus->{devices}}) {
3381             next if !$device->{'qdev_id'};
3382             if ($device->{'pci_bridge'}) {
3383                 $devices->{$device->{'qdev_id'}} = 1;
3384                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3385                     next if !$bridge_device->{'qdev_id'};
3386                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3387                     $devices->{$device->{'qdev_id'}}++;
3388                 }
3389             } else {
3390                 $devices->{$device->{'qdev_id'}} = 1;
3391             }
3392         }
3393     }
3394
3395     my $resblock = vm_mon_cmd($vmid, 'query-block');
3396     foreach my $block (@$resblock) {
3397         if($block->{device} =~ m/^drive-(\S+)/){
3398                 $devices->{$1} = 1;
3399         }
3400     }
3401
3402     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3403     foreach my $mice (@$resmice) {
3404         if ($mice->{name} eq 'QEMU HID Tablet') {
3405             $devices->{tablet} = 1;
3406             last;
3407         }
3408     }
3409
3410     return $devices;
3411 }
3412
3413 sub vm_deviceplug {
3414     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3415
3416     my $q35 = machine_type_is_q35($conf);
3417
3418     my $devices_list = vm_devices_list($vmid);
3419     return 1 if defined($devices_list->{$deviceid});
3420
3421     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3422
3423     if ($deviceid eq 'tablet') {
3424
3425         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3426
3427     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3428
3429         qemu_iothread_add($vmid, $deviceid, $device);
3430
3431         qemu_driveadd($storecfg, $vmid, $device);
3432         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3433
3434         qemu_deviceadd($vmid, $devicefull);
3435         eval { qemu_deviceaddverify($vmid, $deviceid); };
3436         if (my $err = $@) {
3437             eval { qemu_drivedel($vmid, $deviceid); };
3438             warn $@ if $@;
3439             die $err;
3440         }
3441
3442     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3443
3444
3445         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3446         my $pciaddr = print_pci_addr($deviceid);
3447         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3448
3449         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3450
3451         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3452             qemu_iothread_add($vmid, $deviceid, $device);
3453             $devicefull .= ",iothread=iothread-$deviceid";
3454         }
3455
3456         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3457             $devicefull .= ",num_queues=$device->{queues}";
3458         }
3459
3460         qemu_deviceadd($vmid, $devicefull);
3461         qemu_deviceaddverify($vmid, $deviceid);
3462
3463     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3464
3465         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3466         qemu_driveadd($storecfg, $vmid, $device);
3467
3468         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3469         eval { qemu_deviceadd($vmid, $devicefull); };
3470         if (my $err = $@) {
3471             eval { qemu_drivedel($vmid, $deviceid); };
3472             warn $@ if $@;
3473             die $err;
3474         }
3475
3476     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3477
3478         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3479
3480         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3481         my $use_old_bios_files = undef;
3482         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3483
3484         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3485         qemu_deviceadd($vmid, $netdevicefull);
3486         eval { qemu_deviceaddverify($vmid, $deviceid); };
3487         if (my $err = $@) {
3488             eval { qemu_netdevdel($vmid, $deviceid); };
3489             warn $@ if $@;
3490             die $err;
3491         }
3492
3493     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3494
3495         my $bridgeid = $2;
3496         my $pciaddr = print_pci_addr($deviceid);
3497         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3498
3499         qemu_deviceadd($vmid, $devicefull);
3500         qemu_deviceaddverify($vmid, $deviceid);
3501
3502     } else {
3503         die "can't hotplug device '$deviceid'\n";
3504     }
3505
3506     return 1;
3507 }
3508
3509 # fixme: this should raise exceptions on error!
3510 sub vm_deviceunplug {
3511     my ($vmid, $conf, $deviceid) = @_;
3512
3513     my $devices_list = vm_devices_list($vmid);
3514     return 1 if !defined($devices_list->{$deviceid});
3515
3516     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3517
3518     if ($deviceid eq 'tablet') {
3519
3520         qemu_devicedel($vmid, $deviceid);
3521
3522     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3523
3524         qemu_devicedel($vmid, $deviceid);
3525         qemu_devicedelverify($vmid, $deviceid);
3526         qemu_drivedel($vmid, $deviceid);
3527         qemu_iothread_del($conf, $vmid, $deviceid);
3528
3529     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3530
3531         qemu_devicedel($vmid, $deviceid);
3532         qemu_devicedelverify($vmid, $deviceid);
3533         qemu_iothread_del($conf, $vmid, $deviceid);
3534
3535     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3536
3537         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3538         my $device = parse_drive($deviceid, $conf->{$deviceid});
3539         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3540
3541         qemu_devicedel($vmid, $deviceid);
3542         qemu_drivedel($vmid, $deviceid);
3543         qemu_deletescsihw($conf, $vmid, $deviceid);
3544
3545     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3546
3547         qemu_devicedel($vmid, $deviceid);
3548         qemu_devicedelverify($vmid, $deviceid);
3549         qemu_netdevdel($vmid, $deviceid);
3550
3551     } else {
3552         die "can't unplug device '$deviceid'\n";
3553     }
3554
3555     return 1;
3556 }
3557
3558 sub qemu_deviceadd {
3559     my ($vmid, $devicefull) = @_;
3560
3561     $devicefull = "driver=".$devicefull;
3562     my %options =  split(/[=,]/, $devicefull);
3563
3564     vm_mon_cmd($vmid, "device_add" , %options);
3565 }
3566
3567 sub qemu_devicedel {
3568     my ($vmid, $deviceid) = @_;
3569
3570     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3571 }
3572
3573 sub qemu_iothread_add {
3574     my($vmid, $deviceid, $device) = @_;
3575
3576     if ($device->{iothread}) {
3577         my $iothreads = vm_iothreads_list($vmid);
3578         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3579     }
3580 }
3581
3582 sub qemu_iothread_del {
3583     my($conf, $vmid, $deviceid) = @_;
3584
3585     my $device = parse_drive($deviceid, $conf->{$deviceid});
3586     if ($device->{iothread}) {
3587         my $iothreads = vm_iothreads_list($vmid);
3588         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3589     }
3590 }
3591
3592 sub qemu_objectadd {
3593     my($vmid, $objectid, $qomtype) = @_;
3594
3595     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3596
3597     return 1;
3598 }
3599
3600 sub qemu_objectdel {
3601     my($vmid, $objectid) = @_;
3602
3603     vm_mon_cmd($vmid, "object-del", id => $objectid);
3604
3605     return 1;
3606 }
3607
3608 sub qemu_driveadd {
3609     my ($storecfg, $vmid, $device) = @_;
3610
3611     my $drive = print_drive_full($storecfg, $vmid, $device);
3612     $drive =~ s/\\/\\\\/g;
3613     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3614
3615     # If the command succeeds qemu prints: "OK"
3616     return 1 if $ret =~ m/OK/s;
3617
3618     die "adding drive failed: $ret\n";
3619 }
3620
3621 sub qemu_drivedel {
3622     my($vmid, $deviceid) = @_;
3623
3624     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3625     $ret =~ s/^\s+//;
3626
3627     return 1 if $ret eq "";
3628
3629     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3630     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3631
3632     die "deleting drive $deviceid failed : $ret\n";
3633 }
3634
3635 sub qemu_deviceaddverify {
3636     my ($vmid, $deviceid) = @_;
3637
3638     for (my $i = 0; $i <= 5; $i++) {
3639          my $devices_list = vm_devices_list($vmid);
3640          return 1 if defined($devices_list->{$deviceid});
3641          sleep 1;
3642     }
3643
3644     die "error on hotplug device '$deviceid'\n";
3645 }
3646
3647
3648 sub qemu_devicedelverify {
3649     my ($vmid, $deviceid) = @_;
3650
3651     # need to verify that the device is correctly removed as device_del
3652     # is async and empty return is not reliable
3653
3654     for (my $i = 0; $i <= 5; $i++) {
3655          my $devices_list = vm_devices_list($vmid);
3656          return 1 if !defined($devices_list->{$deviceid});
3657          sleep 1;
3658     }
3659
3660     die "error on hot-unplugging device '$deviceid'\n";
3661 }
3662
3663 sub qemu_findorcreatescsihw {
3664     my ($storecfg, $conf, $vmid, $device) = @_;
3665
3666     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3667
3668     my $scsihwid="$controller_prefix$controller";
3669     my $devices_list = vm_devices_list($vmid);
3670
3671     if(!defined($devices_list->{$scsihwid})) {
3672         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3673     }
3674
3675     return 1;
3676 }
3677
3678 sub qemu_deletescsihw {
3679     my ($conf, $vmid, $opt) = @_;
3680
3681     my $device = parse_drive($opt, $conf->{$opt});
3682
3683     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3684         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3685         return 1;
3686     }
3687
3688     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3689
3690     my $devices_list = vm_devices_list($vmid);
3691     foreach my $opt (keys %{$devices_list}) {
3692         if (PVE::QemuServer::valid_drivename($opt)) {
3693             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3694             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3695                 return 1;
3696             }
3697         }
3698     }
3699
3700     my $scsihwid="scsihw$controller";
3701
3702     vm_deviceunplug($vmid, $conf, $scsihwid);
3703
3704     return 1;
3705 }
3706
3707 sub qemu_add_pci_bridge {
3708     my ($storecfg, $conf, $vmid, $device) = @_;
3709
3710     my $bridges = {};
3711
3712     my $bridgeid;
3713
3714     print_pci_addr($device, $bridges);
3715
3716     while (my ($k, $v) = each %$bridges) {
3717         $bridgeid = $k;
3718     }
3719     return 1 if !defined($bridgeid) || $bridgeid < 1;
3720
3721     my $bridge = "pci.$bridgeid";
3722     my $devices_list = vm_devices_list($vmid);
3723
3724     if (!defined($devices_list->{$bridge})) {
3725         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3726     }
3727
3728     return 1;
3729 }
3730
3731 sub qemu_set_link_status {
3732     my ($vmid, $device, $up) = @_;
3733
3734     vm_mon_cmd($vmid, "set_link", name => $device,
3735                up => $up ? JSON::true : JSON::false);
3736 }
3737
3738 sub qemu_netdevadd {
3739     my ($vmid, $conf, $device, $deviceid) = @_;
3740
3741     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3742     my %options =  split(/[=,]/, $netdev);
3743
3744     vm_mon_cmd($vmid, "netdev_add",  %options);
3745     return 1;
3746 }
3747
3748 sub qemu_netdevdel {
3749     my ($vmid, $deviceid) = @_;
3750
3751     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3752 }
3753
3754 sub qemu_cpu_hotplug {
3755     my ($vmid, $conf, $vcpus) = @_;
3756
3757     my $sockets = 1;
3758     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3759     $sockets = $conf->{sockets} if  $conf->{sockets};
3760     my $cores = $conf->{cores} || 1;
3761     my $maxcpus = $sockets * $cores;
3762
3763     $vcpus = $maxcpus if !$vcpus;
3764
3765     die "you can't add more vcpus than maxcpus\n"
3766         if $vcpus > $maxcpus;
3767
3768     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3769     die "online cpu unplug is not yet possible\n"
3770         if $vcpus < $currentvcpus;
3771
3772     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3773     die "vcpus in running vm is different than configuration\n"
3774         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3775
3776     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3777         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3778     }
3779 }
3780
3781 sub qemu_memory_hotplug {
3782     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3783
3784     return $value if !check_running($vmid);
3785
3786     my $memory = $conf->{memory} || $defaults->{memory};
3787     $value = $defaults->{memory} if !$value;
3788     return $value if $value == $memory;
3789
3790     my $static_memory = $STATICMEM;
3791     my $dimm_memory = $memory - $static_memory;
3792
3793     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3794     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3795
3796
3797     my $sockets = 1;
3798     $sockets = $conf->{sockets} if $conf->{sockets};
3799
3800     if($value > $memory) {
3801
3802         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3803             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3804
3805                 return if $current_size <= $conf->{memory};
3806
3807                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3808                 if (my $err = $@) {
3809                     eval { qemu_objectdel($vmid, "mem-$name"); };
3810                     die $err;
3811                 }
3812
3813                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3814                 if (my $err = $@) {
3815                     eval { qemu_objectdel($vmid, "mem-$name"); };
3816                     die $err;
3817                 }
3818                 #update conf after each succesful module hotplug
3819                 $conf->{memory} = $current_size;
3820                 update_config_nolock($vmid, $conf, 1);
3821         });
3822
3823     } else {
3824
3825         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3826             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3827
3828                 return if $current_size >= $conf->{memory};
3829                 print "try to unplug memory dimm $name\n";
3830
3831                 my $retry = 0;
3832                 while (1) {
3833                     eval { qemu_devicedel($vmid, $name) };
3834                     sleep 3;
3835                     my $dimm_list = qemu_dimm_list($vmid);
3836                     last if !$dimm_list->{$name};
3837                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3838                     $retry++;
3839                 }
3840
3841                 #update conf after each succesful module unplug
3842                 $conf->{memory} = $current_size;
3843
3844                 eval { qemu_objectdel($vmid, "mem-$name"); };
3845                 update_config_nolock($vmid, $conf, 1);
3846         });
3847     }
3848 }
3849
3850 sub qemu_dimm_list {
3851     my ($vmid) = @_;
3852
3853     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3854     my $dimms = {};
3855
3856     foreach my $dimm (@$dimmarray) {
3857
3858         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3859         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3860         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3861         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3862         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3863     }
3864     return $dimms;
3865 }
3866
3867 sub qemu_block_set_io_throttle {
3868     my ($vmid, $deviceid,
3869         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3870         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3871
3872     return if !check_running($vmid) ;
3873
3874     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3875         bps => int($bps),
3876         bps_rd => int($bps_rd),
3877         bps_wr => int($bps_wr),
3878         iops => int($iops),
3879         iops_rd => int($iops_rd),
3880         iops_wr => int($iops_wr),
3881         bps_max => int($bps_max),
3882         bps_rd_max => int($bps_rd_max),
3883         bps_wr_max => int($bps_wr_max),
3884         iops_max => int($iops_max),
3885         iops_rd_max => int($iops_rd_max),
3886         iops_wr_max => int($iops_wr_max)
3887     );
3888
3889 }
3890
3891 # old code, only used to shutdown old VM after update
3892 sub __read_avail {
3893     my ($fh, $timeout) = @_;
3894
3895     my $sel = new IO::Select;
3896     $sel->add($fh);
3897
3898     my $res = '';
3899     my $buf;
3900
3901     my @ready;
3902     while (scalar (@ready = $sel->can_read($timeout))) {
3903         my $count;
3904         if ($count = $fh->sysread($buf, 8192)) {
3905             if ($buf =~ /^(.*)\(qemu\) $/s) {
3906                 $res .= $1;
3907                 last;
3908             } else {
3909                 $res .= $buf;
3910             }
3911         } else {
3912             if (!defined($count)) {
3913                 die "$!\n";
3914             }
3915             last;
3916         }
3917     }
3918
3919     die "monitor read timeout\n" if !scalar(@ready);
3920
3921     return $res;
3922 }
3923
3924 # old code, only used to shutdown old VM after update
3925 sub vm_monitor_command {
3926     my ($vmid, $cmdstr, $nocheck) = @_;
3927
3928     my $res;
3929
3930     eval {
3931         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3932
3933         my $sname = "${var_run_tmpdir}/$vmid.mon";
3934
3935         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3936             die "unable to connect to VM $vmid socket - $!\n";
3937
3938         my $timeout = 3;
3939
3940         # hack: migrate sometime blocks the monitor (when migrate_downtime
3941         # is set)
3942         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3943             $timeout = 60*60; # 1 hour
3944         }
3945
3946         # read banner;
3947         my $data = __read_avail($sock, $timeout);
3948
3949         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3950             die "got unexpected qemu monitor banner\n";
3951         }
3952
3953         my $sel = new IO::Select;
3954         $sel->add($sock);
3955
3956         if (!scalar(my @ready = $sel->can_write($timeout))) {
3957             die "monitor write error - timeout";
3958         }
3959
3960         my $fullcmd = "$cmdstr\r";
3961
3962         # syslog('info', "VM $vmid monitor command: $cmdstr");
3963
3964         my $b;
3965         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3966             die "monitor write error - $!";
3967         }
3968
3969         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3970
3971         $timeout = 20;
3972
3973         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3974             $timeout = 60*60; # 1 hour
3975         } elsif ($cmdstr =~ m/^(eject|change)/) {
3976             $timeout = 60; # note: cdrom mount command is slow
3977         }
3978         if ($res = __read_avail($sock, $timeout)) {
3979
3980             my @lines = split("\r?\n", $res);
3981
3982             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3983
3984             $res = join("\n", @lines);
3985             $res .= "\n";
3986         }
3987     };
3988
3989     my $err = $@;
3990
3991     if ($err) {
3992         syslog("err", "VM $vmid monitor command failed - $err");
3993         die $err;
3994     }
3995
3996     return $res;
3997 }
3998
3999 sub qemu_block_resize {
4000     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4001
4002     my $running = check_running($vmid);
4003
4004     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4005
4006     return if !$running;
4007
4008     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4009
4010 }
4011
4012 sub qemu_volume_snapshot {
4013     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4014
4015     my $running = check_running($vmid);
4016
4017     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4018         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4019     } else {
4020         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4021     }
4022 }
4023
4024 sub qemu_volume_snapshot_delete {
4025     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4026
4027     my $running = check_running($vmid);
4028
4029     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4030
4031     return if !$running;
4032
4033     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4034 }
4035
4036 sub set_migration_caps {
4037     my ($vmid) = @_;
4038
4039     my $cap_ref = [];
4040
4041     my $enabled_cap = {
4042         "auto-converge" => 1,
4043         "xbzrle" => 1,
4044         "x-rdma-pin-all" => 0,
4045         "zero-blocks" => 0,
4046         "compress" => 0
4047     };
4048
4049     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4050
4051     for my $supported_capability (@$supported_capabilities) {
4052         push @$cap_ref, {
4053             capability => $supported_capability->{capability},
4054             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4055         };
4056     }
4057
4058     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4059 }
4060
4061 my $fast_plug_option = {
4062     'lock' => 1,
4063     'name' => 1,
4064     'onboot' => 1,
4065     'shares' => 1,
4066     'startup' => 1,
4067     'description' => 1,
4068 };
4069
4070 # hotplug changes in [PENDING]
4071 # $selection hash can be used to only apply specified options, for
4072 # example: { cores => 1 } (only apply changed 'cores')
4073 # $errors ref is used to return error messages
4074 sub vmconfig_hotplug_pending {
4075     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4076
4077     my $defaults = load_defaults();
4078
4079     # commit values which do not have any impact on running VM first
4080     # Note: those option cannot raise errors, we we do not care about
4081     # $selection and always apply them.
4082
4083     my $add_error = sub {
4084         my ($opt, $msg) = @_;
4085         $errors->{$opt} = "hotplug problem - $msg";
4086     };
4087
4088     my $changes = 0;
4089     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4090         if ($fast_plug_option->{$opt}) {
4091             $conf->{$opt} = $conf->{pending}->{$opt};
4092             delete $conf->{pending}->{$opt};
4093             $changes = 1;
4094         }
4095     }
4096
4097     if ($changes) {
4098         update_config_nolock($vmid, $conf, 1);
4099         $conf = load_config($vmid); # update/reload
4100     }
4101
4102     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4103
4104     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4105     while (my ($opt, $force) = each %$pending_delete_hash) {
4106         next if $selection && !$selection->{$opt};
4107         eval {
4108             if ($opt eq 'hotplug') {
4109                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4110             } elsif ($opt eq 'tablet') {
4111                 die "skip\n" if !$hotplug_features->{usb};
4112                 if ($defaults->{tablet}) {
4113                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4114                 } else {
4115                     vm_deviceunplug($vmid, $conf, $opt);
4116                 }
4117             } elsif ($opt eq 'vcpus') {
4118                 die "skip\n" if !$hotplug_features->{cpu};
4119                 qemu_cpu_hotplug($vmid, $conf, undef);
4120             } elsif ($opt eq 'balloon') {
4121                 # enable balloon device is not hotpluggable
4122                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4123             } elsif ($fast_plug_option->{$opt}) {
4124                 # do nothing
4125             } elsif ($opt =~ m/^net(\d+)$/) {
4126                 die "skip\n" if !$hotplug_features->{network};
4127                 vm_deviceunplug($vmid, $conf, $opt);
4128             } elsif (valid_drivename($opt)) {
4129                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4130                 vm_deviceunplug($vmid, $conf, $opt);
4131                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4132             } elsif ($opt =~ m/^memory$/) {
4133                 die "skip\n" if !$hotplug_features->{memory};
4134                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4135             } elsif ($opt eq 'cpuunits') {
4136                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4137             } elsif ($opt eq 'cpulimit') {
4138                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4139             } else {
4140                 die "skip\n";
4141             }
4142         };
4143         if (my $err = $@) {
4144             &$add_error($opt, $err) if $err ne "skip\n";
4145         } else {
4146             # save new config if hotplug was successful
4147             delete $conf->{$opt};
4148             vmconfig_undelete_pending_option($conf, $opt);
4149             update_config_nolock($vmid, $conf, 1);
4150             $conf = load_config($vmid); # update/reload
4151         }
4152     }
4153
4154     foreach my $opt (keys %{$conf->{pending}}) {
4155         next if $selection && !$selection->{$opt};
4156         my $value = $conf->{pending}->{$opt};
4157         eval {
4158             if ($opt eq 'hotplug') {
4159                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4160             } elsif ($opt eq 'tablet') {
4161                 die "skip\n" if !$hotplug_features->{usb};
4162                 if ($value == 1) {
4163                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4164                 } elsif ($value == 0) {
4165                     vm_deviceunplug($vmid, $conf, $opt);
4166                 }
4167             } elsif ($opt eq 'vcpus') {
4168                 die "skip\n" if !$hotplug_features->{cpu};
4169                 qemu_cpu_hotplug($vmid, $conf, $value);
4170             } elsif ($opt eq 'balloon') {
4171                 # enable/disable balloning device is not hotpluggable
4172                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4173                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4174                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4175
4176                 # allow manual ballooning if shares is set to zero
4177                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4178                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4179                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4180                 }
4181             } elsif ($opt =~ m/^net(\d+)$/) {
4182                 # some changes can be done without hotplug
4183                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4184                                     $vmid, $opt, $value);
4185             } elsif (valid_drivename($opt)) {
4186                 # some changes can be done without hotplug
4187                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4188                                      $vmid, $opt, $value, 1);
4189             } elsif ($opt =~ m/^memory$/) { #dimms
4190                 die "skip\n" if !$hotplug_features->{memory};
4191                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4192             } elsif ($opt eq 'cpuunits') {
4193                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4194             } elsif ($opt eq 'cpulimit') {
4195                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4196                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4197             } else {
4198                 die "skip\n";  # skip non-hot-pluggable options
4199             }
4200         };
4201         if (my $err = $@) {
4202             &$add_error($opt, $err) if $err ne "skip\n";
4203         } else {
4204             # save new config if hotplug was successful
4205             $conf->{$opt} = $value;
4206             delete $conf->{pending}->{$opt};
4207             update_config_nolock($vmid, $conf, 1);
4208             $conf = load_config($vmid); # update/reload
4209         }
4210     }
4211 }
4212
4213 sub try_deallocate_drive {
4214     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4215
4216     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4217         my $volid = $drive->{file};
4218         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4219             my $sid = PVE::Storage::parse_volume_id($volid);
4220             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4221
4222             # check if the disk is really unused
4223             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4224                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4225             PVE::Storage::vdisk_free($storecfg, $volid);
4226             return 1;
4227         } else {
4228             # If vm is not owner of this disk remove from config
4229             return 1;
4230         }
4231     }
4232
4233     return undef;
4234 }
4235
4236 sub vmconfig_delete_or_detach_drive {
4237     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4238
4239     my $drive = parse_drive($opt, $conf->{$opt});
4240
4241     my $rpcenv = PVE::RPCEnvironment::get();
4242     my $authuser = $rpcenv->get_user();
4243
4244     if ($force) {
4245         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4246         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4247     } else {
4248         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4249     }
4250 }
4251
4252 sub vmconfig_apply_pending {
4253     my ($vmid, $conf, $storecfg) = @_;
4254
4255     # cold plug
4256
4257     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4258     while (my ($opt, $force) = each %$pending_delete_hash) {
4259         die "internal error" if $opt =~ m/^unused/;
4260         $conf = load_config($vmid); # update/reload
4261         if (!defined($conf->{$opt})) {
4262             vmconfig_undelete_pending_option($conf, $opt);
4263             update_config_nolock($vmid, $conf, 1);
4264         } elsif (valid_drivename($opt)) {
4265             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4266             vmconfig_undelete_pending_option($conf, $opt);
4267             delete $conf->{$opt};
4268             update_config_nolock($vmid, $conf, 1);
4269         } else {
4270             vmconfig_undelete_pending_option($conf, $opt);
4271             delete $conf->{$opt};
4272             update_config_nolock($vmid, $conf, 1);
4273         }
4274     }
4275
4276     $conf = load_config($vmid); # update/reload
4277
4278     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4279         $conf = load_config($vmid); # update/reload
4280
4281         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4282             # skip if nothing changed
4283         } elsif (valid_drivename($opt)) {
4284             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4285                 if defined($conf->{$opt});
4286             $conf->{$opt} = $conf->{pending}->{$opt};
4287         } else {
4288             $conf->{$opt} = $conf->{pending}->{$opt};
4289         }
4290
4291         delete $conf->{pending}->{$opt};
4292         update_config_nolock($vmid, $conf, 1);
4293     }
4294 }
4295
4296 my $safe_num_ne = sub {
4297     my ($a, $b) = @_;
4298
4299     return 0 if !defined($a) && !defined($b);
4300     return 1 if !defined($a);
4301     return 1 if !defined($b);
4302
4303     return $a != $b;
4304 };
4305
4306 my $safe_string_ne = sub {
4307     my ($a, $b) = @_;
4308
4309     return 0 if !defined($a) && !defined($b);
4310     return 1 if !defined($a);
4311     return 1 if !defined($b);
4312
4313     return $a ne $b;
4314 };
4315
4316 sub vmconfig_update_net {
4317     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4318
4319     my $newnet = parse_net($value);
4320
4321     if ($conf->{$opt}) {
4322         my $oldnet = parse_net($conf->{$opt});
4323
4324         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4325             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4326             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4327             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4328
4329             # for non online change, we try to hot-unplug
4330             die "skip\n" if !$hotplug;
4331             vm_deviceunplug($vmid, $conf, $opt);
4332         } else {
4333
4334             die "internal error" if $opt !~ m/net(\d+)/;
4335             my $iface = "tap${vmid}i$1";
4336
4337             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4338                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4339             }
4340
4341             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4342                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4343                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4344                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4345                 PVE::Network::tap_unplug($iface);
4346                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4347             }
4348
4349             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4350                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4351             }
4352
4353             return 1;
4354         }
4355     }
4356
4357     if ($hotplug) {
4358         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4359     } else {
4360         die "skip\n";
4361     }
4362 }
4363
4364 sub vmconfig_update_disk {
4365     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4366
4367     # fixme: do we need force?
4368
4369     my $drive = parse_drive($opt, $value);
4370
4371     if ($conf->{$opt}) {
4372
4373         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4374
4375             my $media = $drive->{media} || 'disk';
4376             my $oldmedia = $old_drive->{media} || 'disk';
4377             die "unable to change media type\n" if $media ne $oldmedia;
4378
4379             if (!drive_is_cdrom($old_drive)) {
4380
4381                 if ($drive->{file} ne $old_drive->{file}) {
4382
4383                     die "skip\n" if !$hotplug;
4384
4385                     # unplug and register as unused
4386                     vm_deviceunplug($vmid, $conf, $opt);
4387                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4388
4389                 } else {
4390                     # update existing disk
4391
4392                     # skip non hotpluggable value
4393                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4394                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4395                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4396                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4397                         die "skip\n";
4398                     }
4399
4400                     # apply throttle
4401                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4402                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4403                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4404                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4405                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4406                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4407                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4408                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4409                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4410                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4411                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4412                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4413
4414                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4415                                                    ($drive->{mbps} || 0)*1024*1024,
4416                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4417                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4418                                                    $drive->{iops} || 0,
4419                                                    $drive->{iops_rd} || 0,
4420                                                    $drive->{iops_wr} || 0,
4421                                                    ($drive->{mbps_max} || 0)*1024*1024,
4422                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4423                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4424                                                    $drive->{iops_max} || 0,
4425                                                    $drive->{iops_rd_max} || 0,
4426                                                    $drive->{iops_wr_max} || 0);
4427
4428                     }
4429
4430                     return 1;
4431                 }
4432
4433             } else { # cdrom
4434
4435                 if ($drive->{file} eq 'none') {
4436                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4437                 } else {
4438                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4439                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4440                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4441                 }
4442
4443                 return 1;
4444             }
4445         }
4446     }
4447
4448     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4449     # hotplug new disks
4450     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4451     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4452 }
4453
4454 sub vm_start {
4455     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4456         $forcemachine, $spice_ticket) = @_;
4457
4458     lock_config($vmid, sub {
4459         my $conf = load_config($vmid, $migratedfrom);
4460
4461         die "you can't start a vm if it's a template\n" if is_template($conf);
4462
4463         check_lock($conf) if !$skiplock;
4464
4465         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4466
4467         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4468             vmconfig_apply_pending($vmid, $conf, $storecfg);
4469             $conf = load_config($vmid); # update/reload
4470         }
4471
4472         my $defaults = load_defaults();
4473
4474         # set environment variable useful inside network script
4475         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4476
4477         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4478
4479         my $migrate_port = 0;
4480         my $migrate_uri;
4481         if ($statefile) {
4482             if ($statefile eq 'tcp') {
4483                 my $localip = "localhost";
4484                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4485                 my $nodename = PVE::INotify::nodename();
4486                 if ($datacenterconf->{migration_unsecure}) {
4487                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4488                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4489                 }
4490                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4491                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4492                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4493                 push @$cmd, '-incoming', $migrate_uri;
4494                 push @$cmd, '-S';
4495             } else {
4496                 push @$cmd, '-loadstate', $statefile;
4497             }
4498         } elsif ($paused) {
4499             push @$cmd, '-S';
4500         }
4501
4502         # host pci devices
4503         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4504           my $d = parse_hostpci($conf->{"hostpci$i"});
4505           next if !$d;
4506           my $pcidevices = $d->{pciid};
4507           foreach my $pcidevice (@$pcidevices) {
4508                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4509
4510                 my $info = pci_device_info("0000:$pciid");
4511                 die "IOMMU not present\n" if !check_iommu_support();
4512                 die "no pci device info for device '$pciid'\n" if !$info;
4513                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4514                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4515           }
4516         }
4517
4518         PVE::Storage::activate_volumes($storecfg, $vollist);
4519
4520         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4521                     umask => 0077); };
4522         my $err = $@;
4523         die "start failed: $err" if $err;
4524
4525         print "migration listens on $migrate_uri\n" if $migrate_uri;
4526
4527         if ($statefile && $statefile ne 'tcp')  {
4528             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4529             warn $@ if $@;
4530         }
4531
4532         if ($migratedfrom) {
4533
4534             eval {
4535                 set_migration_caps($vmid);
4536             };
4537             warn $@ if $@;
4538
4539             if ($spice_port) {
4540                 print "spice listens on port $spice_port\n";
4541                 if ($spice_ticket) {
4542                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4543                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4544                 }
4545             }
4546
4547         } else {
4548
4549             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4550                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4551                     if $conf->{balloon};
4552             }
4553
4554             foreach my $opt (keys %$conf) {
4555                 next if $opt !~  m/^net\d+$/;
4556                 my $nicconf = parse_net($conf->{$opt});
4557                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4558             }
4559         }
4560
4561         vm_mon_cmd_nocheck($vmid, 'qom-set',
4562                     path => "machine/peripheral/balloon0",
4563                     property => "guest-stats-polling-interval",
4564                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4565
4566     });
4567 }
4568
4569 sub vm_mon_cmd {
4570     my ($vmid, $execute, %params) = @_;
4571
4572     my $cmd = { execute => $execute, arguments => \%params };
4573     vm_qmp_command($vmid, $cmd);
4574 }
4575
4576 sub vm_mon_cmd_nocheck {
4577     my ($vmid, $execute, %params) = @_;
4578
4579     my $cmd = { execute => $execute, arguments => \%params };
4580     vm_qmp_command($vmid, $cmd, 1);
4581 }
4582
4583 sub vm_qmp_command {
4584     my ($vmid, $cmd, $nocheck) = @_;
4585
4586     my $res;
4587
4588     my $timeout;
4589     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4590         $timeout = $cmd->{arguments}->{timeout};
4591         delete $cmd->{arguments}->{timeout};
4592     }
4593
4594     eval {
4595         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4596         my $sname = qmp_socket($vmid);
4597         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4598             my $qmpclient = PVE::QMPClient->new();
4599
4600             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4601         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4602             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4603                 if scalar(%{$cmd->{arguments}});
4604             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4605         } else {
4606             die "unable to open monitor socket\n";
4607         }
4608     };
4609     if (my $err = $@) {
4610         syslog("err", "VM $vmid qmp command failed - $err");
4611         die $err;
4612     }
4613
4614     return $res;
4615 }
4616
4617 sub vm_human_monitor_command {
4618     my ($vmid, $cmdline) = @_;
4619
4620     my $res;
4621
4622     my $cmd = {
4623         execute => 'human-monitor-command',
4624         arguments => { 'command-line' => $cmdline},
4625     };
4626
4627     return vm_qmp_command($vmid, $cmd);
4628 }
4629
4630 sub vm_commandline {
4631     my ($storecfg, $vmid) = @_;
4632
4633     my $conf = load_config($vmid);
4634
4635     my $defaults = load_defaults();
4636
4637     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4638
4639     return join(' ', @$cmd);
4640 }
4641
4642 sub vm_reset {
4643     my ($vmid, $skiplock) = @_;
4644
4645     lock_config($vmid, sub {
4646
4647         my $conf = load_config($vmid);
4648
4649         check_lock($conf) if !$skiplock;
4650
4651         vm_mon_cmd($vmid, "system_reset");
4652     });
4653 }
4654
4655 sub get_vm_volumes {
4656     my ($conf) = @_;
4657
4658     my $vollist = [];
4659     foreach_volid($conf, sub {
4660         my ($volid, $is_cdrom) = @_;
4661
4662         return if $volid =~ m|^/|;
4663
4664         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4665         return if !$sid;
4666
4667         push @$vollist, $volid;
4668     });
4669
4670     return $vollist;
4671 }
4672
4673 sub vm_stop_cleanup {
4674     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4675
4676     eval {
4677
4678         if (!$keepActive) {
4679             my $vollist = get_vm_volumes($conf);
4680             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4681         }
4682
4683         foreach my $ext (qw(mon qmp pid vnc qga)) {
4684             unlink "/var/run/qemu-server/${vmid}.$ext";
4685         }
4686
4687         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4688     };
4689     warn $@ if $@; # avoid errors - just warn
4690 }
4691
4692 # Note: use $nockeck to skip tests if VM configuration file exists.
4693 # We need that when migration VMs to other nodes (files already moved)
4694 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4695 sub vm_stop {
4696     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4697
4698     $force = 1 if !defined($force) && !$shutdown;
4699
4700     if ($migratedfrom){
4701         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4702         kill 15, $pid if $pid;
4703         my $conf = load_config($vmid, $migratedfrom);
4704         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4705         return;
4706     }
4707
4708     lock_config($vmid, sub {
4709
4710         my $pid = check_running($vmid, $nocheck);
4711         return if !$pid;
4712
4713         my $conf;
4714         if (!$nocheck) {
4715             $conf = load_config($vmid);
4716             check_lock($conf) if !$skiplock;
4717             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4718                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4719                 $timeout = $opts->{down} if $opts->{down};
4720             }
4721         }
4722
4723         $timeout = 60 if !defined($timeout);
4724
4725         eval {
4726             if ($shutdown) {
4727                 if (defined($conf) && $conf->{agent}) {
4728                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4729                 } else {
4730                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4731                 }
4732             } else {
4733                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4734             }
4735         };
4736         my $err = $@;
4737
4738         if (!$err) {
4739             my $count = 0;
4740             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4741                 $count++;
4742                 sleep 1;
4743             }
4744
4745             if ($count >= $timeout) {
4746                 if ($force) {
4747                     warn "VM still running - terminating now with SIGTERM\n";
4748                     kill 15, $pid;
4749                 } else {
4750                     die "VM quit/powerdown failed - got timeout\n";
4751                 }
4752             } else {
4753                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4754                 return;
4755             }
4756         } else {
4757             if ($force) {
4758                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4759                 kill 15, $pid;
4760             } else {
4761                 die "VM quit/powerdown failed\n";
4762             }
4763         }
4764
4765         # wait again
4766         $timeout = 10;
4767
4768         my $count = 0;
4769         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4770             $count++;
4771             sleep 1;
4772         }
4773
4774         if ($count >= $timeout) {
4775             warn "VM still running - terminating now with SIGKILL\n";
4776             kill 9, $pid;
4777             sleep 1;
4778         }
4779
4780         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4781    });
4782 }
4783
4784 sub vm_suspend {
4785     my ($vmid, $skiplock) = @_;
4786
4787     lock_config($vmid, sub {
4788
4789         my $conf = load_config($vmid);
4790
4791         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4792
4793         vm_mon_cmd($vmid, "stop");
4794     });
4795 }
4796
4797 sub vm_resume {
4798     my ($vmid, $skiplock, $nocheck) = @_;
4799
4800     lock_config($vmid, sub {
4801
4802         if (!$nocheck) {
4803
4804             my $conf = load_config($vmid);
4805
4806             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4807
4808             vm_mon_cmd($vmid, "cont");
4809
4810         } else {
4811             vm_mon_cmd_nocheck($vmid, "cont");
4812         }
4813     });
4814 }
4815
4816 sub vm_sendkey {
4817     my ($vmid, $skiplock, $key) = @_;
4818
4819     lock_config($vmid, sub {
4820
4821         my $conf = load_config($vmid);
4822
4823         # there is no qmp command, so we use the human monitor command
4824         vm_human_monitor_command($vmid, "sendkey $key");
4825     });
4826 }
4827
4828 sub vm_destroy {
4829     my ($storecfg, $vmid, $skiplock) = @_;
4830
4831     lock_config($vmid, sub {
4832
4833         my $conf = load_config($vmid);
4834
4835         check_lock($conf) if !$skiplock;
4836
4837         if (!check_running($vmid)) {
4838             destroy_vm($storecfg, $vmid);
4839         } else {
4840             die "VM $vmid is running - destroy failed\n";
4841         }
4842     });
4843 }
4844
4845 # pci helpers
4846
4847 sub file_write {
4848     my ($filename, $buf) = @_;
4849
4850     my $fh = IO::File->new($filename, "w");
4851     return undef if !$fh;
4852
4853     my $res = print $fh $buf;
4854
4855     $fh->close();
4856
4857     return $res;
4858 }
4859
4860 sub pci_device_info {
4861     my ($name) = @_;
4862
4863     my $res;
4864
4865     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4866     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4867
4868     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4869     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4870
4871     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4872     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4873
4874     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4875     return undef if !defined($product) || $product !~ s/^0x//;
4876
4877     $res = {
4878         name => $name,
4879         vendor => $vendor,
4880         product => $product,
4881         domain => $domain,
4882         bus => $bus,
4883         slot => $slot,
4884         func => $func,
4885         irq => $irq,
4886         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4887     };
4888
4889     return $res;
4890 }
4891
4892 sub pci_dev_reset {
4893     my ($dev) = @_;
4894
4895     my $name = $dev->{name};
4896
4897     my $fn = "$pcisysfs/devices/$name/reset";
4898
4899     return file_write($fn, "1");
4900 }
4901
4902 sub pci_dev_bind_to_vfio {
4903     my ($dev) = @_;
4904
4905     my $name = $dev->{name};
4906
4907     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4908
4909     if (!-d $vfio_basedir) {
4910         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4911     }
4912     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4913
4914     my $testdir = "$vfio_basedir/$name";
4915     return 1 if -d $testdir;
4916
4917     my $data = "$dev->{vendor} $dev->{product}";
4918     return undef if !file_write("$vfio_basedir/new_id", $data);
4919
4920     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4921     if (!file_write($fn, $name)) {
4922         return undef if -f $fn;
4923     }
4924
4925     $fn = "$vfio_basedir/bind";
4926     if (! -d $testdir) {
4927         return undef if !file_write($fn, $name);
4928     }
4929
4930     return -d $testdir;
4931 }
4932
4933 sub pci_dev_group_bind_to_vfio {
4934     my ($pciid) = @_;
4935
4936     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4937
4938     if (!-d $vfio_basedir) {
4939         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4940     }
4941     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4942
4943     # get IOMMU group devices
4944     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4945       my @devs = grep /^0000:/, readdir($D);
4946     closedir($D);
4947
4948     foreach my $pciid (@devs) {
4949         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4950
4951         # pci bridges, switches or root ports are not supported
4952         # they have a pci_bus subdirectory so skip them
4953         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4954
4955         my $info = pci_device_info($1);
4956         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4957     }
4958
4959     return 1;
4960 }
4961
4962 sub print_pci_addr {
4963     my ($id, $bridges) = @_;
4964
4965     my $res = '';
4966     my $devices = {
4967         piix3 => { bus => 0, addr => 1 },
4968         #addr2 : first videocard
4969         balloon0 => { bus => 0, addr => 3 },
4970         watchdog => { bus => 0, addr => 4 },
4971         scsihw0 => { bus => 0, addr => 5 },
4972         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4973         scsihw1 => { bus => 0, addr => 6 },
4974         ahci0 => { bus => 0, addr => 7 },
4975         qga0 => { bus => 0, addr => 8 },
4976         spice => { bus => 0, addr => 9 },
4977         virtio0 => { bus => 0, addr => 10 },
4978         virtio1 => { bus => 0, addr => 11 },
4979         virtio2 => { bus => 0, addr => 12 },
4980         virtio3 => { bus => 0, addr => 13 },
4981         virtio4 => { bus => 0, addr => 14 },
4982         virtio5 => { bus => 0, addr => 15 },
4983         hostpci0 => { bus => 0, addr => 16 },
4984         hostpci1 => { bus => 0, addr => 17 },
4985         net0 => { bus => 0, addr => 18 },
4986         net1 => { bus => 0, addr => 19 },
4987         net2 => { bus => 0, addr => 20 },
4988         net3 => { bus => 0, addr => 21 },
4989         net4 => { bus => 0, addr => 22 },
4990         net5 => { bus => 0, addr => 23 },
4991         vga1 => { bus => 0, addr => 24 },
4992         vga2 => { bus => 0, addr => 25 },
4993         vga3 => { bus => 0, addr => 26 },
4994         hostpci2 => { bus => 0, addr => 27 },
4995         hostpci3 => { bus => 0, addr => 28 },
4996         #addr29 : usb-host (pve-usb.cfg)
4997         'pci.1' => { bus => 0, addr => 30 },
4998         'pci.2' => { bus => 0, addr => 31 },
4999         'net6' => { bus => 1, addr => 1 },
5000         'net7' => { bus => 1, addr => 2 },
5001         'net8' => { bus => 1, addr => 3 },
5002         'net9' => { bus => 1, addr => 4 },
5003         'net10' => { bus => 1, addr => 5 },
5004         'net11' => { bus => 1, addr => 6 },
5005         'net12' => { bus => 1, addr => 7 },
5006         'net13' => { bus => 1, addr => 8 },
5007         'net14' => { bus => 1, addr => 9 },
5008         'net15' => { bus => 1, addr => 10 },
5009         'net16' => { bus => 1, addr => 11 },
5010         'net17' => { bus => 1, addr => 12 },
5011         'net18' => { bus => 1, addr => 13 },
5012         'net19' => { bus => 1, addr => 14 },
5013         'net20' => { bus => 1, addr => 15 },
5014         'net21' => { bus => 1, addr => 16 },
5015         'net22' => { bus => 1, addr => 17 },
5016         'net23' => { bus => 1, addr => 18 },
5017         'net24' => { bus => 1, addr => 19 },
5018         'net25' => { bus => 1, addr => 20 },
5019         'net26' => { bus => 1, addr => 21 },
5020         'net27' => { bus => 1, addr => 22 },
5021         'net28' => { bus => 1, addr => 23 },
5022         'net29' => { bus => 1, addr => 24 },
5023         'net30' => { bus => 1, addr => 25 },
5024         'net31' => { bus => 1, addr => 26 },
5025         'virtio6' => { bus => 2, addr => 1 },
5026         'virtio7' => { bus => 2, addr => 2 },
5027         'virtio8' => { bus => 2, addr => 3 },
5028         'virtio9' => { bus => 2, addr => 4 },
5029         'virtio10' => { bus => 2, addr => 5 },
5030         'virtio11' => { bus => 2, addr => 6 },
5031         'virtio12' => { bus => 2, addr => 7 },
5032         'virtio13' => { bus => 2, addr => 8 },
5033         'virtio14' => { bus => 2, addr => 9 },
5034         'virtio15' => { bus => 2, addr => 10 },
5035         'virtioscsi0' => { bus => 3, addr => 1 },
5036         'virtioscsi1' => { bus => 3, addr => 2 },
5037         'virtioscsi2' => { bus => 3, addr => 3 },
5038         'virtioscsi3' => { bus => 3, addr => 4 },
5039         'virtioscsi4' => { bus => 3, addr => 5 },
5040         'virtioscsi5' => { bus => 3, addr => 6 },
5041         'virtioscsi6' => { bus => 3, addr => 7 },
5042         'virtioscsi7' => { bus => 3, addr => 8 },
5043         'virtioscsi8' => { bus => 3, addr => 9 },
5044         'virtioscsi9' => { bus => 3, addr => 10 },
5045         'virtioscsi10' => { bus => 3, addr => 11 },
5046         'virtioscsi11' => { bus => 3, addr => 12 },
5047         'virtioscsi12' => { bus => 3, addr => 13 },
5048         'virtioscsi13' => { bus => 3, addr => 14 },
5049         'virtioscsi14' => { bus => 3, addr => 15 },
5050         'virtioscsi15' => { bus => 3, addr => 16 },
5051         'virtioscsi16' => { bus => 3, addr => 17 },
5052         'virtioscsi17' => { bus => 3, addr => 18 },
5053         'virtioscsi18' => { bus => 3, addr => 19 },
5054         'virtioscsi19' => { bus => 3, addr => 20 },
5055         'virtioscsi20' => { bus => 3, addr => 21 },
5056         'virtioscsi21' => { bus => 3, addr => 22 },
5057         'virtioscsi22' => { bus => 3, addr => 23 },
5058         'virtioscsi23' => { bus => 3, addr => 24 },
5059         'virtioscsi24' => { bus => 3, addr => 25 },
5060         'virtioscsi25' => { bus => 3, addr => 26 },
5061         'virtioscsi26' => { bus => 3, addr => 27 },
5062         'virtioscsi27' => { bus => 3, addr => 28 },
5063         'virtioscsi28' => { bus => 3, addr => 29 },
5064         'virtioscsi29' => { bus => 3, addr => 30 },
5065         'virtioscsi30' => { bus => 3, addr => 31 },
5066
5067     };
5068
5069     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5070            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5071            my $bus = $devices->{$id}->{bus};
5072            $res = ",bus=pci.$bus,addr=$addr";
5073            $bridges->{$bus} = 1 if $bridges;
5074     }
5075     return $res;
5076
5077 }
5078
5079 sub print_pcie_addr {
5080     my ($id) = @_;
5081
5082     my $res = '';
5083     my $devices = {
5084         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5085         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5086         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5087         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5088     };
5089
5090     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5091            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5092            my $bus = $devices->{$id}->{bus};
5093            $res = ",bus=$bus,addr=$addr";
5094     }
5095     return $res;
5096
5097 }
5098
5099 # vzdump restore implementaion
5100
5101 sub tar_archive_read_firstfile {
5102     my $archive = shift;
5103
5104     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5105
5106     # try to detect archive type first
5107     my $pid = open (TMP, "tar tf '$archive'|") ||
5108         die "unable to open file '$archive'\n";
5109     my $firstfile = <TMP>;
5110     kill 15, $pid;
5111     close TMP;
5112
5113     die "ERROR: archive contaions no data\n" if !$firstfile;
5114     chomp $firstfile;
5115
5116     return $firstfile;
5117 }
5118
5119 sub tar_restore_cleanup {
5120     my ($storecfg, $statfile) = @_;
5121
5122     print STDERR "starting cleanup\n";
5123
5124     if (my $fd = IO::File->new($statfile, "r")) {
5125         while (defined(my $line = <$fd>)) {
5126             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5127                 my $volid = $2;
5128                 eval {
5129                     if ($volid =~ m|^/|) {
5130                         unlink $volid || die 'unlink failed\n';
5131                     } else {
5132                         PVE::Storage::vdisk_free($storecfg, $volid);
5133                     }
5134                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5135                 };
5136                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5137             } else {
5138                 print STDERR "unable to parse line in statfile - $line";
5139             }
5140         }
5141         $fd->close();
5142     }
5143 }
5144
5145 sub restore_archive {
5146     my ($archive, $vmid, $user, $opts) = @_;
5147
5148     my $format = $opts->{format};
5149     my $comp;
5150
5151     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5152         $format = 'tar' if !$format;
5153         $comp = 'gzip';
5154     } elsif ($archive =~ m/\.tar$/) {
5155         $format = 'tar' if !$format;
5156     } elsif ($archive =~ m/.tar.lzo$/) {
5157         $format = 'tar' if !$format;
5158         $comp = 'lzop';
5159     } elsif ($archive =~ m/\.vma$/) {
5160         $format = 'vma' if !$format;
5161     } elsif ($archive =~ m/\.vma\.gz$/) {
5162         $format = 'vma' if !$format;
5163         $comp = 'gzip';
5164     } elsif ($archive =~ m/\.vma\.lzo$/) {
5165         $format = 'vma' if !$format;
5166         $comp = 'lzop';
5167     } else {
5168         $format = 'vma' if !$format; # default
5169     }
5170
5171     # try to detect archive format
5172     if ($format eq 'tar') {
5173         return restore_tar_archive($archive, $vmid, $user, $opts);
5174     } else {
5175         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5176     }
5177 }
5178
5179 sub restore_update_config_line {
5180     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5181
5182     return if $line =~ m/^\#qmdump\#/;
5183     return if $line =~ m/^\#vzdump\#/;
5184     return if $line =~ m/^lock:/;
5185     return if $line =~ m/^unused\d+:/;
5186     return if $line =~ m/^parent:/;
5187     return if $line =~ m/^template:/; # restored VM is never a template
5188
5189     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5190         # try to convert old 1.X settings
5191         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5192         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5193             my ($model, $macaddr) = split(/\=/, $devconfig);
5194             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5195             my $net = {
5196                 model => $model,
5197                 bridge => "vmbr$ind",
5198                 macaddr => $macaddr,
5199             };
5200             my $netstr = print_net($net);
5201
5202             print $outfd "net$cookie->{netcount}: $netstr\n";
5203             $cookie->{netcount}++;
5204         }
5205     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5206         my ($id, $netstr) = ($1, $2);
5207         my $net = parse_net($netstr);
5208         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5209         $netstr = print_net($net);
5210         print $outfd "$id: $netstr\n";
5211     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5212         my $virtdev = $1;
5213         my $value = $3;
5214         if ($line =~ m/backup=no/) {
5215             print $outfd "#$line";
5216         } elsif ($virtdev && $map->{$virtdev}) {
5217             my $di = parse_drive($virtdev, $value);
5218             delete $di->{format}; # format can change on restore
5219             $di->{file} = $map->{$virtdev};
5220             $value = print_drive($vmid, $di);
5221             print $outfd "$virtdev: $value\n";
5222         } else {
5223             print $outfd $line;
5224         }
5225     } else {
5226         print $outfd $line;
5227     }
5228 }
5229
5230 sub scan_volids {
5231     my ($cfg, $vmid) = @_;
5232
5233     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5234
5235     my $volid_hash = {};
5236     foreach my $storeid (keys %$info) {
5237         foreach my $item (@{$info->{$storeid}}) {
5238             next if !($item->{volid} && $item->{size});
5239             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5240             $volid_hash->{$item->{volid}} = $item;
5241         }
5242     }
5243
5244     return $volid_hash;
5245 }
5246
5247 sub is_volume_in_use {
5248     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5249
5250     my $path = PVE::Storage::path($storecfg, $volid);
5251
5252     my $scan_config = sub {
5253         my ($cref, $snapname) = @_;
5254
5255         foreach my $key (keys %$cref) {
5256             my $value = $cref->{$key};
5257             if (valid_drivename($key)) {
5258                 next if $skip_drive && $key eq $skip_drive;
5259                 my $drive = parse_drive($key, $value);
5260                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5261                 return 1 if $volid eq $drive->{file};
5262                 if ($drive->{file} =~ m!^/!) {
5263                     return 1 if $drive->{file} eq $path;
5264                 } else {
5265                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5266                     next if !$storeid;
5267                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5268                     next if !$scfg;
5269                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5270                 }
5271             }
5272         }
5273
5274         return 0;
5275     };
5276
5277     return 1 if &$scan_config($conf);
5278
5279     undef $skip_drive;
5280
5281     foreach my $snapname (keys %{$conf->{snapshots}}) {
5282         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5283     }
5284
5285     return 0;
5286 }
5287
5288 sub update_disksize {
5289     my ($vmid, $conf, $volid_hash) = @_;
5290
5291     my $changes;
5292
5293     my $used = {};
5294
5295     # Note: it is allowed to define multiple storages with same path (alias), so
5296     # we need to check both 'volid' and real 'path' (two different volid can point
5297     # to the same path).
5298
5299     my $usedpath = {};
5300
5301     # update size info
5302     foreach my $opt (keys %$conf) {
5303         if (valid_drivename($opt)) {
5304             my $drive = parse_drive($opt, $conf->{$opt});
5305             my $volid = $drive->{file};
5306             next if !$volid;
5307
5308             $used->{$volid} = 1;
5309             if ($volid_hash->{$volid} &&
5310                 (my $path = $volid_hash->{$volid}->{path})) {
5311                 $usedpath->{$path} = 1;
5312             }
5313
5314             next if drive_is_cdrom($drive);
5315             next if !$volid_hash->{$volid};
5316
5317             $drive->{size} = $volid_hash->{$volid}->{size};
5318             my $new = print_drive($vmid, $drive);
5319             if ($new ne $conf->{$opt}) {
5320                 $changes = 1;
5321                 $conf->{$opt} = $new;
5322             }
5323         }
5324     }
5325
5326     # remove 'unusedX' entry if volume is used
5327     foreach my $opt (keys %$conf) {
5328         next if $opt !~ m/^unused\d+$/;
5329         my $volid = $conf->{$opt};
5330         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5331         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5332             $changes = 1;
5333             delete $conf->{$opt};
5334         }
5335     }
5336
5337     foreach my $volid (sort keys %$volid_hash) {
5338         next if $volid =~ m/vm-$vmid-state-/;
5339         next if $used->{$volid};
5340         my $path = $volid_hash->{$volid}->{path};
5341         next if !$path; # just to be sure
5342         next if $usedpath->{$path};
5343         $changes = 1;
5344         add_unused_volume($conf, $volid);
5345         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5346     }
5347
5348     return $changes;
5349 }
5350
5351 sub rescan {
5352     my ($vmid, $nolock) = @_;
5353
5354     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5355
5356     my $volid_hash = scan_volids($cfg, $vmid);
5357
5358     my $updatefn =  sub {
5359         my ($vmid) = @_;
5360
5361         my $conf = load_config($vmid);
5362
5363         check_lock($conf);
5364
5365         my $vm_volids = {};
5366         foreach my $volid (keys %$volid_hash) {
5367             my $info = $volid_hash->{$volid};
5368             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5369         }
5370
5371         my $changes = update_disksize($vmid, $conf, $vm_volids);
5372
5373         update_config_nolock($vmid, $conf, 1) if $changes;
5374     };
5375
5376     if (defined($vmid)) {
5377         if ($nolock) {
5378             &$updatefn($vmid);
5379         } else {
5380             lock_config($vmid, $updatefn, $vmid);
5381         }
5382     } else {
5383         my $vmlist = config_list();
5384         foreach my $vmid (keys %$vmlist) {
5385             if ($nolock) {
5386                 &$updatefn($vmid);
5387             } else {
5388                 lock_config($vmid, $updatefn, $vmid);
5389             }
5390         }
5391     }
5392 }
5393
5394 sub restore_vma_archive {
5395     my ($archive, $vmid, $user, $opts, $comp) = @_;
5396
5397     my $input = $archive eq '-' ? "<&STDIN" : undef;
5398     my $readfrom = $archive;
5399
5400     my $uncomp = '';
5401     if ($comp) {
5402         $readfrom = '-';
5403         my $qarchive = PVE::Tools::shellquote($archive);
5404         if ($comp eq 'gzip') {
5405             $uncomp = "zcat $qarchive|";
5406         } elsif ($comp eq 'lzop') {
5407             $uncomp = "lzop -d -c $qarchive|";
5408         } else {
5409             die "unknown compression method '$comp'\n";
5410         }
5411
5412     }
5413
5414     my $tmpdir = "/var/tmp/vzdumptmp$$";
5415     rmtree $tmpdir;
5416
5417     # disable interrupts (always do cleanups)
5418     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5419         warn "got interrupt - ignored\n";
5420     };
5421
5422     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5423     POSIX::mkfifo($mapfifo, 0600);
5424     my $fifofh;
5425
5426     my $openfifo = sub {
5427         open($fifofh, '>', $mapfifo) || die $!;
5428     };
5429
5430     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5431
5432     my $oldtimeout;
5433     my $timeout = 5;
5434
5435     my $devinfo = {};
5436
5437     my $rpcenv = PVE::RPCEnvironment::get();
5438
5439     my $conffile = config_file($vmid);
5440     my $tmpfn = "$conffile.$$.tmp";
5441
5442     # Note: $oldconf is undef if VM does not exists
5443     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5444
5445     my $print_devmap = sub {
5446         my $virtdev_hash = {};
5447
5448         my $cfgfn = "$tmpdir/qemu-server.conf";
5449
5450         # we can read the config - that is already extracted
5451         my $fh = IO::File->new($cfgfn, "r") ||
5452             "unable to read qemu-server.conf - $!\n";
5453
5454         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5455         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5456             if -f $fwcfgfn;
5457
5458         while (defined(my $line = <$fh>)) {
5459             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5460                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5461                 die "archive does not contain data for drive '$virtdev'\n"
5462                     if !$devinfo->{$devname};
5463                 if (defined($opts->{storage})) {
5464                     $storeid = $opts->{storage} || 'local';
5465                 } elsif (!$storeid) {
5466                     $storeid = 'local';
5467                 }
5468                 $format = 'raw' if !$format;
5469                 $devinfo->{$devname}->{devname} = $devname;
5470                 $devinfo->{$devname}->{virtdev} = $virtdev;
5471                 $devinfo->{$devname}->{format} = $format;
5472                 $devinfo->{$devname}->{storeid} = $storeid;
5473
5474                 # check permission on storage
5475                 my $pool = $opts->{pool}; # todo: do we need that?
5476                 if ($user ne 'root@pam') {
5477                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5478                 }
5479
5480                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5481             }
5482         }
5483
5484         foreach my $devname (keys %$devinfo) {
5485             die "found no device mapping information for device '$devname'\n"
5486                 if !$devinfo->{$devname}->{virtdev};
5487         }
5488
5489         my $cfg = cfs_read_file('storage.cfg');
5490
5491         # create empty/temp config
5492         if ($oldconf) {
5493             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5494             foreach_drive($oldconf, sub {
5495                 my ($ds, $drive) = @_;
5496
5497                 return if drive_is_cdrom($drive);
5498
5499                 my $volid = $drive->{file};
5500
5501                 return if !$volid || $volid =~ m|^/|;
5502
5503                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5504                 return if !$path || !$owner || ($owner != $vmid);
5505
5506                 # Note: only delete disk we want to restore
5507                 # other volumes will become unused
5508                 if ($virtdev_hash->{$ds}) {
5509                     PVE::Storage::vdisk_free($cfg, $volid);
5510                 }
5511             });
5512         }
5513
5514         my $map = {};
5515         foreach my $virtdev (sort keys %$virtdev_hash) {
5516             my $d = $virtdev_hash->{$virtdev};
5517             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5518             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5519
5520             # test if requested format is supported
5521             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5522             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5523             $d->{format} = $defFormat if !$supported;
5524
5525             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5526                                                   $d->{format}, undef, $alloc_size);
5527             print STDERR "new volume ID is '$volid'\n";
5528             $d->{volid} = $volid;
5529             my $path = PVE::Storage::path($cfg, $volid);
5530
5531             PVE::Storage::activate_volumes($cfg,[$volid]);
5532
5533             my $write_zeros = 1;
5534             # fixme: what other storages types initialize volumes with zero?
5535             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5536                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5537                 $write_zeros = 0;
5538             }
5539
5540             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5541
5542             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5543             $map->{$virtdev} = $volid;
5544         }
5545
5546         $fh->seek(0, 0) || die "seek failed - $!\n";
5547
5548         my $outfd = new IO::File ($tmpfn, "w") ||
5549             die "unable to write config for VM $vmid\n";
5550
5551         my $cookie = { netcount => 0 };
5552         while (defined(my $line = <$fh>)) {
5553             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5554         }
5555
5556         $fh->close();
5557         $outfd->close();
5558     };
5559
5560     eval {
5561         # enable interrupts
5562         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5563             die "interrupted by signal\n";
5564         };
5565         local $SIG{ALRM} = sub { die "got timeout\n"; };
5566
5567         $oldtimeout = alarm($timeout);
5568
5569         my $parser = sub {
5570             my $line = shift;
5571
5572             print "$line\n";
5573
5574             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5575                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5576                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5577             } elsif ($line =~ m/^CTIME: /) {
5578                 # we correctly received the vma config, so we can disable
5579                 # the timeout now for disk allocation (set to 10 minutes, so
5580                 # that we always timeout if something goes wrong)
5581                 alarm(600);
5582                 &$print_devmap();
5583                 print $fifofh "done\n";
5584                 my $tmp = $oldtimeout || 0;
5585                 $oldtimeout = undef;
5586                 alarm($tmp);
5587                 close($fifofh);
5588             }
5589         };
5590
5591         print "restore vma archive: $cmd\n";
5592         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5593     };
5594     my $err = $@;
5595
5596     alarm($oldtimeout) if $oldtimeout;
5597
5598     my $vollist = [];
5599     foreach my $devname (keys %$devinfo) {
5600         my $volid = $devinfo->{$devname}->{volid};
5601         push @$vollist, $volid if $volid;
5602     }
5603
5604     my $cfg = cfs_read_file('storage.cfg');
5605     PVE::Storage::deactivate_volumes($cfg, $vollist);
5606
5607     unlink $mapfifo;
5608
5609     if ($err) {
5610         rmtree $tmpdir;
5611         unlink $tmpfn;
5612
5613         foreach my $devname (keys %$devinfo) {
5614             my $volid = $devinfo->{$devname}->{volid};
5615             next if !$volid;
5616             eval {
5617                 if ($volid =~ m|^/|) {
5618                     unlink $volid || die 'unlink failed\n';
5619                 } else {
5620                     PVE::Storage::vdisk_free($cfg, $volid);
5621                 }
5622                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5623             };
5624             print STDERR "unable to cleanup '$volid' - $@" if $@;
5625         }
5626         die $err;
5627     }
5628
5629     rmtree $tmpdir;
5630
5631     rename($tmpfn, $conffile) ||
5632         die "unable to commit configuration file '$conffile'\n";
5633
5634     PVE::Cluster::cfs_update(); # make sure we read new file
5635
5636     eval { rescan($vmid, 1); };
5637     warn $@ if $@;
5638 }
5639
5640 sub restore_tar_archive {
5641     my ($archive, $vmid, $user, $opts) = @_;
5642
5643     if ($archive ne '-') {
5644         my $firstfile = tar_archive_read_firstfile($archive);
5645         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5646             if $firstfile ne 'qemu-server.conf';
5647     }
5648
5649     my $storecfg = cfs_read_file('storage.cfg');
5650
5651     # destroy existing data - keep empty config
5652     my $vmcfgfn = config_file($vmid);
5653     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5654
5655     my $tocmd = "/usr/lib/qemu-server/qmextract";
5656
5657     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5658     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5659     $tocmd .= ' --prealloc' if $opts->{prealloc};
5660     $tocmd .= ' --info' if $opts->{info};
5661
5662     # tar option "xf" does not autodetect compression when read from STDIN,
5663     # so we pipe to zcat
5664     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5665         PVE::Tools::shellquote("--to-command=$tocmd");
5666
5667     my $tmpdir = "/var/tmp/vzdumptmp$$";
5668     mkpath $tmpdir;
5669
5670     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5671     local $ENV{VZDUMP_VMID} = $vmid;
5672     local $ENV{VZDUMP_USER} = $user;
5673
5674     my $conffile = config_file($vmid);
5675     my $tmpfn = "$conffile.$$.tmp";
5676
5677     # disable interrupts (always do cleanups)
5678     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5679         print STDERR "got interrupt - ignored\n";
5680     };
5681
5682     eval {
5683         # enable interrupts
5684         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5685             die "interrupted by signal\n";
5686         };
5687
5688         if ($archive eq '-') {
5689             print "extracting archive from STDIN\n";
5690             run_command($cmd, input => "<&STDIN");
5691         } else {
5692             print "extracting archive '$archive'\n";
5693             run_command($cmd);
5694         }
5695
5696         return if $opts->{info};
5697
5698         # read new mapping
5699         my $map = {};
5700         my $statfile = "$tmpdir/qmrestore.stat";
5701         if (my $fd = IO::File->new($statfile, "r")) {
5702             while (defined (my $line = <$fd>)) {
5703                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5704                     $map->{$1} = $2 if $1;
5705                 } else {
5706                     print STDERR "unable to parse line in statfile - $line\n";
5707                 }
5708             }
5709             $fd->close();
5710         }
5711
5712         my $confsrc = "$tmpdir/qemu-server.conf";
5713
5714         my $srcfd = new IO::File($confsrc, "r") ||
5715             die "unable to open file '$confsrc'\n";
5716
5717         my $outfd = new IO::File ($tmpfn, "w") ||
5718             die "unable to write config for VM $vmid\n";
5719
5720         my $cookie = { netcount => 0 };
5721         while (defined (my $line = <$srcfd>)) {
5722             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5723         }
5724
5725         $srcfd->close();
5726         $outfd->close();
5727     };
5728     my $err = $@;
5729
5730     if ($err) {
5731
5732         unlink $tmpfn;
5733
5734         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5735
5736         die $err;
5737     }
5738
5739     rmtree $tmpdir;
5740
5741     rename $tmpfn, $conffile ||
5742         die "unable to commit configuration file '$conffile'\n";
5743
5744     PVE::Cluster::cfs_update(); # make sure we read new file
5745
5746     eval { rescan($vmid, 1); };
5747     warn $@ if $@;
5748 };
5749
5750
5751 # Internal snapshots
5752
5753 # NOTE: Snapshot create/delete involves several non-atomic
5754 # action, and can take a long time.
5755 # So we try to avoid locking the file and use 'lock' variable
5756 # inside the config file instead.
5757
5758 my $snapshot_copy_config = sub {
5759     my ($source, $dest) = @_;
5760
5761     foreach my $k (keys %$source) {
5762         next if $k eq 'snapshots';
5763         next if $k eq 'snapstate';
5764         next if $k eq 'snaptime';
5765         next if $k eq 'vmstate';
5766         next if $k eq 'lock';
5767         next if $k eq 'digest';
5768         next if $k eq 'description';
5769         next if $k =~ m/^unused\d+$/;
5770
5771         $dest->{$k} = $source->{$k};
5772     }
5773 };
5774
5775 my $snapshot_apply_config = sub {
5776     my ($conf, $snap) = @_;
5777
5778     # copy snapshot list
5779     my $newconf = {
5780         snapshots => $conf->{snapshots},
5781     };
5782
5783     # keep description and list of unused disks
5784     foreach my $k (keys %$conf) {
5785         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5786         $newconf->{$k} = $conf->{$k};
5787     }
5788
5789     &$snapshot_copy_config($snap, $newconf);
5790
5791     return $newconf;
5792 };
5793
5794 sub foreach_writable_storage {
5795     my ($conf, $func) = @_;
5796
5797     my $sidhash = {};
5798
5799     foreach my $ds (keys %$conf) {
5800         next if !valid_drivename($ds);
5801
5802         my $drive = parse_drive($ds, $conf->{$ds});
5803         next if !$drive;
5804         next if drive_is_cdrom($drive);
5805
5806         my $volid = $drive->{file};
5807
5808         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5809         $sidhash->{$sid} = $sid if $sid;
5810     }
5811
5812     foreach my $sid (sort keys %$sidhash) {
5813         &$func($sid);
5814     }
5815 }
5816
5817 my $alloc_vmstate_volid = sub {
5818     my ($storecfg, $vmid, $conf, $snapname) = @_;
5819
5820     # Note: we try to be smart when selecting a $target storage
5821
5822     my $target;
5823
5824     # search shared storage first
5825     foreach_writable_storage($conf, sub {
5826         my ($sid) = @_;
5827         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5828         return if !$scfg->{shared};
5829
5830         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5831     });
5832
5833     if (!$target) {
5834         # now search local storage
5835         foreach_writable_storage($conf, sub {
5836             my ($sid) = @_;
5837             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5838             return if $scfg->{shared};
5839
5840             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5841         });
5842     }
5843
5844     $target = 'local' if !$target;
5845
5846     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5847     # we abort live save after $conf->{memory}, so we need at max twice that space
5848     my $size = $conf->{memory}*2 + $driver_state_size;
5849
5850     my $name = "vm-$vmid-state-$snapname";
5851     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5852     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5853     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5854
5855     return $volid;
5856 };
5857
5858 my $snapshot_prepare = sub {
5859     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5860
5861     my $snap;
5862
5863     my $updatefn =  sub {
5864
5865         my $conf = load_config($vmid);
5866
5867         die "you can't take a snapshot if it's a template\n"
5868             if is_template($conf);
5869
5870         check_lock($conf);
5871
5872         $conf->{lock} = 'snapshot';
5873
5874         die "snapshot name '$snapname' already used\n"
5875             if defined($conf->{snapshots}->{$snapname});
5876
5877         my $storecfg = PVE::Storage::config();
5878         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5879
5880         $snap = $conf->{snapshots}->{$snapname} = {};
5881
5882         if ($save_vmstate && check_running($vmid)) {
5883             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5884         }
5885
5886         &$snapshot_copy_config($conf, $snap);
5887
5888         $snap->{snapstate} = "prepare";
5889         $snap->{snaptime} = time();
5890         $snap->{description} = $comment if $comment;
5891
5892         # always overwrite machine if we save vmstate. This makes sure we
5893         # can restore it later using correct machine type
5894         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5895
5896         update_config_nolock($vmid, $conf, 1);
5897     };
5898
5899     lock_config($vmid, $updatefn);
5900
5901     return $snap;
5902 };
5903
5904 my $snapshot_commit = sub {
5905     my ($vmid, $snapname) = @_;
5906
5907     my $updatefn = sub {
5908
5909         my $conf = load_config($vmid);
5910
5911         die "missing snapshot lock\n"
5912             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5913
5914         my $has_machine_config = defined($conf->{machine});
5915
5916         my $snap = $conf->{snapshots}->{$snapname};
5917
5918         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5919
5920         die "wrong snapshot state\n"
5921             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5922
5923         delete $snap->{snapstate};
5924         delete $conf->{lock};
5925
5926         my $newconf = &$snapshot_apply_config($conf, $snap);
5927
5928         delete $newconf->{machine} if !$has_machine_config;
5929
5930         $newconf->{parent} = $snapname;
5931
5932         update_config_nolock($vmid, $newconf, 1);
5933     };
5934
5935     lock_config($vmid, $updatefn);
5936 };
5937
5938 sub snapshot_rollback {
5939     my ($vmid, $snapname) = @_;
5940
5941     my $prepare = 1;
5942
5943     my $storecfg = PVE::Storage::config();
5944
5945     my $conf = load_config($vmid);
5946
5947     my $get_snapshot_config = sub {
5948
5949         die "you can't rollback if vm is a template\n" if is_template($conf);
5950
5951         my $res = $conf->{snapshots}->{$snapname};
5952
5953         die "snapshot '$snapname' does not exist\n" if !defined($res);
5954
5955         return $res;
5956     };
5957
5958     my $snap = &$get_snapshot_config();
5959
5960     foreach_drive($snap, sub {
5961         my ($ds, $drive) = @_;
5962
5963         return if drive_is_cdrom($drive);
5964
5965         my $volid = $drive->{file};
5966
5967         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5968     });
5969
5970     my $updatefn = sub {
5971
5972         $conf = load_config($vmid);
5973
5974         $snap = &$get_snapshot_config();
5975
5976         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5977             if $snap->{snapstate};
5978
5979         if ($prepare) {
5980             check_lock($conf);
5981             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5982         }
5983
5984         die "unable to rollback vm $vmid: vm is running\n"
5985             if check_running($vmid);
5986
5987         if ($prepare) {
5988             $conf->{lock} = 'rollback';
5989         } else {
5990             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5991             delete $conf->{lock};
5992         }
5993
5994         my $forcemachine;
5995
5996         if (!$prepare) {
5997             my $has_machine_config = defined($conf->{machine});
5998
5999             # copy snapshot config to current config
6000             $conf = &$snapshot_apply_config($conf, $snap);
6001             $conf->{parent} = $snapname;
6002
6003             # Note: old code did not store 'machine', so we try to be smart
6004             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6005             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6006             # we remove the 'machine' configuration if not explicitly specified
6007             # in the original config.
6008             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6009         }
6010
6011         update_config_nolock($vmid, $conf, 1);
6012
6013         if (!$prepare && $snap->{vmstate}) {
6014             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6015             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6016         }
6017     };
6018
6019     lock_config($vmid, $updatefn);
6020
6021     foreach_drive($snap, sub {
6022         my ($ds, $drive) = @_;
6023
6024         return if drive_is_cdrom($drive);
6025
6026         my $volid = $drive->{file};
6027         my $device = "drive-$ds";
6028
6029         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6030     });
6031
6032     $prepare = 0;
6033     lock_config($vmid, $updatefn);
6034 }
6035
6036 my $savevm_wait = sub {
6037     my ($vmid) = @_;
6038
6039     for(;;) {
6040         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6041         if (!$stat->{status}) {
6042             die "savevm not active\n";
6043         } elsif ($stat->{status} eq 'active') {
6044             sleep(1);
6045             next;
6046         } elsif ($stat->{status} eq 'completed') {
6047             last;
6048         } else {
6049             die "query-savevm returned status '$stat->{status}'\n";
6050         }
6051     }
6052 };
6053
6054 sub do_snapshots_with_qemu {
6055     my ($storecfg, $volid) = @_;
6056
6057     my $storage_name = PVE::Storage::parse_volume_id($volid);
6058
6059     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6060         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6061         return 1;
6062     }
6063
6064     if ($volid =~ m/\.(qcow2|qed)$/){
6065         return 1;
6066     }
6067
6068     return undef;
6069 }
6070
6071 sub snapshot_create {
6072     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6073
6074     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6075
6076     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6077
6078     my $config = load_config($vmid);
6079
6080     my $running = check_running($vmid);
6081
6082     my $freezefs = $running && $config->{agent};
6083     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6084
6085     my $drivehash = {};
6086
6087     if ($freezefs) {
6088         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6089         warn "guest-fsfreeze-freeze problems - $@" if $@;
6090     }
6091
6092     eval {
6093         # create internal snapshots of all drives
6094
6095         my $storecfg = PVE::Storage::config();
6096
6097         if ($running) {
6098             if ($snap->{vmstate}) {
6099                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6100                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6101                 &$savevm_wait($vmid);
6102             } else {
6103                 vm_mon_cmd($vmid, "savevm-start");
6104             }
6105         };
6106
6107         foreach_drive($snap, sub {
6108             my ($ds, $drive) = @_;
6109
6110             return if drive_is_cdrom($drive);
6111
6112             my $volid = $drive->{file};
6113             my $device = "drive-$ds";
6114
6115             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6116             $drivehash->{$ds} = 1;
6117        });
6118     };
6119     my $err = $@;
6120
6121     if ($running) {
6122         eval { vm_mon_cmd($vmid, "savevm-end")  };
6123         warn $@ if $@;
6124
6125         if ($freezefs) {
6126             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6127             warn "guest-fsfreeze-thaw problems - $@" if $@;
6128         }
6129
6130         # savevm-end is async, we need to wait
6131         for (;;) {
6132             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6133             if (!$stat->{bytes}) {
6134                 last;
6135             } else {
6136                 print "savevm not yet finished\n";
6137                 sleep(1);
6138                 next;
6139             }
6140         }
6141     }
6142
6143     if ($err) {
6144         warn "snapshot create failed: starting cleanup\n";
6145         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6146         warn $@ if $@;
6147         die $err;
6148     }
6149
6150     &$snapshot_commit($vmid, $snapname);
6151 }
6152
6153 # Note: $drivehash is only set when called from snapshot_create.
6154 sub snapshot_delete {
6155     my ($vmid, $snapname, $force, $drivehash) = @_;
6156
6157     my $prepare = 1;
6158
6159     my $snap;
6160     my $unused = [];
6161
6162     my $unlink_parent = sub {
6163         my ($confref, $new_parent) = @_;
6164
6165         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6166             if ($new_parent) {
6167                 $confref->{parent} = $new_parent;
6168             } else {
6169                 delete $confref->{parent};
6170             }
6171         }
6172     };
6173
6174     my $updatefn =  sub {
6175         my ($remove_drive) = @_;
6176
6177         my $conf = load_config($vmid);
6178
6179         if (!$drivehash) {
6180             check_lock($conf);
6181             die "you can't delete a snapshot if vm is a template\n"
6182                 if is_template($conf);
6183         }
6184
6185         $snap = $conf->{snapshots}->{$snapname};
6186
6187         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6188
6189         # remove parent refs
6190         if (!$prepare) {
6191             &$unlink_parent($conf, $snap->{parent});
6192             foreach my $sn (keys %{$conf->{snapshots}}) {
6193                 next if $sn eq $snapname;
6194                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6195             }
6196         }
6197
6198         if ($remove_drive) {
6199             if ($remove_drive eq 'vmstate') {
6200                 delete $snap->{$remove_drive};
6201             } else {
6202                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6203                 my $volid = $drive->{file};
6204                 delete $snap->{$remove_drive};
6205                 add_unused_volume($conf, $volid);
6206             }
6207         }
6208
6209         if ($prepare) {
6210             $snap->{snapstate} = 'delete';
6211         } else {
6212             delete $conf->{snapshots}->{$snapname};
6213             delete $conf->{lock} if $drivehash;
6214             foreach my $volid (@$unused) {
6215                 add_unused_volume($conf, $volid);
6216             }
6217         }
6218
6219         update_config_nolock($vmid, $conf, 1);
6220     };
6221
6222     lock_config($vmid, $updatefn);
6223
6224     # now remove vmstate file
6225
6226     my $storecfg = PVE::Storage::config();
6227
6228     if ($snap->{vmstate}) {
6229         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6230         if (my $err = $@) {
6231             die $err if !$force;
6232             warn $err;
6233         }
6234         # save changes (remove vmstate from snapshot)
6235         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6236     };
6237
6238     # now remove all internal snapshots
6239     foreach_drive($snap, sub {
6240         my ($ds, $drive) = @_;
6241
6242         return if drive_is_cdrom($drive);
6243
6244         my $volid = $drive->{file};
6245         my $device = "drive-$ds";
6246
6247         if (!$drivehash || $drivehash->{$ds}) {
6248             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6249             if (my $err = $@) {
6250                 die $err if !$force;
6251                 warn $err;
6252             }
6253         }
6254
6255         # save changes (remove drive fron snapshot)
6256         lock_config($vmid, $updatefn, $ds) if !$force;
6257         push @$unused, $volid;
6258     });
6259
6260     # now cleanup config
6261     $prepare = 0;
6262     lock_config($vmid, $updatefn);
6263 }
6264
6265 sub has_feature {
6266     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6267
6268     my $err;
6269     foreach_drive($conf, sub {
6270         my ($ds, $drive) = @_;
6271
6272         return if drive_is_cdrom($drive);
6273         my $volid = $drive->{file};
6274         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6275     });
6276
6277     return $err ? 0 : 1;
6278 }
6279
6280 sub template_create {
6281     my ($vmid, $conf, $disk) = @_;
6282
6283     my $storecfg = PVE::Storage::config();
6284
6285     foreach_drive($conf, sub {
6286         my ($ds, $drive) = @_;
6287
6288         return if drive_is_cdrom($drive);
6289         return if $disk && $ds ne $disk;
6290
6291         my $volid = $drive->{file};
6292         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6293
6294         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6295         $drive->{file} = $voliddst;
6296         $conf->{$ds} = print_drive($vmid, $drive);
6297         update_config_nolock($vmid, $conf, 1);
6298     });
6299 }
6300
6301 sub is_template {
6302     my ($conf) = @_;
6303
6304     return 1 if defined $conf->{template} && $conf->{template} == 1;
6305 }
6306
6307 sub qemu_img_convert {
6308     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6309
6310     my $storecfg = PVE::Storage::config();
6311     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6312     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6313
6314     if ($src_storeid && $dst_storeid) {
6315
6316         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6317
6318         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6319         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6320
6321         my $src_format = qemu_img_format($src_scfg, $src_volname);
6322         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6323
6324         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6325         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6326
6327         my $cmd = [];
6328         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6329         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6330         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6331
6332         my $parser = sub {
6333             my $line = shift;
6334             if($line =~ m/\((\S+)\/100\%\)/){
6335                 my $percent = $1;
6336                 my $transferred = int($size * $percent / 100);
6337                 my $remaining = $size - $transferred;
6338
6339                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6340             }
6341
6342         };
6343
6344         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6345         my $err = $@;
6346         die "copy failed: $err" if $err;
6347     }
6348 }
6349
6350 sub qemu_img_format {
6351     my ($scfg, $volname) = @_;
6352
6353     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6354         return $1;
6355     } else {
6356         return "raw";
6357     }
6358 }
6359
6360 sub qemu_drive_mirror {
6361     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6362
6363     my $storecfg = PVE::Storage::config();
6364     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6365
6366     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6367
6368     my $format = qemu_img_format($dst_scfg, $dst_volname);
6369
6370     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6371
6372     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6373     $opts->{format} = $format if $format;
6374
6375     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6376
6377     eval {
6378     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6379         while (1) {
6380             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6381             my $stat = @$stats[0];
6382             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6383             die "error job is not mirroring" if $stat->{type} ne "mirror";
6384
6385             my $busy = $stat->{busy};
6386             my $ready = $stat->{ready};
6387
6388             if (my $total = $stat->{len}) {
6389                 my $transferred = $stat->{offset} || 0;
6390                 my $remaining = $total - $transferred;
6391                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6392
6393                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6394             }
6395
6396
6397             if ($stat->{ready} eq 'true') {
6398
6399                 last if $vmiddst != $vmid;
6400
6401                 # try to switch the disk if source and destination are on the same guest
6402                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6403                 last if !$@;
6404                 die $@ if $@ !~ m/cannot be completed/;
6405             }
6406             sleep 1;
6407         }
6408
6409
6410     };
6411     my $err = $@;
6412
6413     my $cancel_job = sub {
6414         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6415         while (1) {
6416             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6417             my $stat = @$stats[0];
6418             last if !$stat;
6419             sleep 1;
6420         }
6421     };
6422
6423     if ($err) {
6424         eval { &$cancel_job(); };
6425         die "mirroring error: $err";
6426     }
6427
6428     if ($vmiddst != $vmid) {
6429         # if we clone a disk for a new target vm, we don't switch the disk
6430         &$cancel_job(); # so we call block-job-cancel
6431     }
6432 }
6433
6434 sub clone_disk {
6435     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6436         $newvmid, $storage, $format, $full, $newvollist) = @_;
6437
6438     my $newvolid;
6439
6440     if (!$full) {
6441         print "create linked clone of drive $drivename ($drive->{file})\n";
6442         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6443         push @$newvollist, $newvolid;
6444     } else {
6445         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6446         $storeid = $storage if $storage;
6447
6448         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6449         if (!$format) {
6450             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6451             $format = qemu_img_format($scfg, $volname);
6452         }
6453
6454         # test if requested format is supported - else use default
6455         my $supported = grep { $_ eq $format } @$validFormats;
6456         $format = $defFormat if !$supported;
6457
6458         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6459
6460         print "create full clone of drive $drivename ($drive->{file})\n";
6461         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6462         push @$newvollist, $newvolid;
6463
6464         PVE::Storage::activate_volumes($storecfg, $newvollist);
6465
6466         if (!$running || $snapname) {
6467             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6468         } else {
6469             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6470         }
6471     }
6472
6473     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6474
6475     my $disk = $drive;
6476     $disk->{format} = undef;
6477     $disk->{file} = $newvolid;
6478     $disk->{size} = $size;
6479
6480     return $disk;
6481 }
6482
6483 # this only works if VM is running
6484 sub get_current_qemu_machine {
6485     my ($vmid) = @_;
6486
6487     my $cmd = { execute => 'query-machines', arguments => {} };
6488     my $res = vm_qmp_command($vmid, $cmd);
6489
6490     my ($current, $default);
6491     foreach my $e (@$res) {
6492         $default = $e->{name} if $e->{'is-default'};
6493         $current = $e->{name} if $e->{'is-current'};
6494     }
6495
6496     # fallback to the default machine if current is not supported by qemu
6497     return $current || $default || 'pc';
6498 }
6499
6500 sub qemu_machine_feature_enabled {
6501     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6502
6503     my $current_major;
6504     my $current_minor;
6505
6506     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6507
6508         $current_major = $3;
6509         $current_minor = $4;
6510
6511     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6512
6513         $current_major = $1;
6514         $current_minor = $2;
6515     }
6516
6517     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6518
6519
6520 }
6521
6522 sub qemu_machine_pxe {
6523     my ($vmid, $conf, $machine) = @_;
6524
6525     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6526
6527     foreach my $opt (keys %$conf) {
6528         next if $opt !~ m/^net(\d+)$/;
6529         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6530         next if !$net;
6531         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6532         return $machine.".pxe" if $romfile =~ m/pxe/;
6533         last;
6534     }
6535
6536 }
6537
6538 sub qemu_use_old_bios_files {
6539     my ($machine_type) = @_;
6540
6541     return if !$machine_type;
6542
6543     my $use_old_bios_files = undef;
6544
6545     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6546         $machine_type = $1;
6547         $use_old_bios_files = 1;
6548     } else {
6549         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6550         # load new efi bios files on migration. So this hack is required to allow
6551         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6552         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6553         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6554     }
6555
6556     return ($use_old_bios_files, $machine_type);
6557 }
6558
6559 sub lspci {
6560
6561     my $devices = {};
6562
6563     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6564             my (undef, $id, $function) = @_;
6565             my $res = { id => $id, function => $function};
6566             push @{$devices->{$id}}, $res;
6567     });
6568
6569     return $devices;
6570 }
6571
6572 sub vm_iothreads_list {
6573     my ($vmid) = @_;
6574
6575     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6576
6577     my $iothreads = {};
6578     foreach my $iothread (@$res) {
6579         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6580     }
6581
6582     return $iothreads;
6583 }
6584
6585 sub scsihw_infos {
6586     my ($conf, $drive) = @_;
6587
6588     my $maxdev = 0;
6589
6590     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6591         $maxdev = 7;
6592     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6593         $maxdev = 1;
6594     } else {
6595         $maxdev = 256;
6596     }
6597
6598     my $controller = int($drive->{index} / $maxdev);
6599     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6600
6601     return ($maxdev, $controller, $controller_prefix);
6602 }
6603
6604 # bash completion helper
6605
6606 sub complete_backup_archives {
6607     my ($cmdname, $pname, $cvalue) = @_;
6608
6609     my $cfg = PVE::Storage::config();
6610
6611     my $storeid;
6612
6613     if ($cvalue =~ m/^([^:]+):/) {
6614         $storeid = $1;
6615     }
6616
6617     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6618
6619     my $res = [];
6620     foreach my $id (keys %$data) {
6621         foreach my $item (@{$data->{$id}}) {
6622             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6623             push @$res, $item->{volid} if defined($item->{volid});
6624         }
6625     }
6626
6627     return $res;
6628 }
6629
6630 my $complete_vmid_full = sub {
6631     my ($running) = @_;
6632
6633     my $idlist = vmstatus();
6634
6635     my $res = [];
6636
6637     foreach my $id (keys %$idlist) {
6638         my $d = $idlist->{$id};
6639         if (defined($running)) {
6640             next if $d->{template};
6641             next if $running && $d->{status} ne 'running';
6642             next if !$running && $d->{status} eq 'running';
6643         }
6644         push @$res, $id;
6645
6646     }
6647     return $res;
6648 };
6649
6650 sub complete_vmid {
6651     return &$complete_vmid_full();
6652 }
6653
6654 sub complete_vmid_stopped {
6655     return &$complete_vmid_full(0);
6656 }
6657
6658 sub complete_vmid_running {
6659     return &$complete_vmid_full(1);
6660 }
6661
6662 sub complete_storage {
6663
6664     my $cfg = PVE::Storage::config();
6665     my $ids = $cfg->{ids};
6666
6667     my $res = [];
6668     foreach my $sid (keys %$ids) {
6669         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6670         next if !$ids->{$sid}->{content}->{images};
6671         push @$res, $sid;
6672     }
6673
6674     return $res;
6675 }
6676
6677 1;