PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393     bios => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [ qw(seabios ovmf) ],
 397         description => "Select BIOS implementation.",
 398         default => 'seabios',
 399     },
 400 };
 401
 402 # what about other qemu settings ?
 403 #cpu => 'string',
 404 #machine => 'string',
 405 #fda => 'file',
 406 #fdb => 'file',
 407 #mtdblock => 'file',
 408 #sd => 'file',
 409 #pflash => 'file',
 410 #snapshot => 'bool',
 411 #bootp => 'file',
 412 ##tftp => 'dir',
 413 ##smb => 'dir',
 414 #kernel => 'file',
 415 #append => 'string',
 416 #initrd => 'file',
 417 ##soundhw => 'string',
 418
 419 while (my ($k, $v) = each %$confdesc) {
 420     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 421 }
 422
 423 my $MAX_IDE_DISKS = 4;
 424 my $MAX_SCSI_DISKS = 14;
 425 my $MAX_VIRTIO_DISKS = 16;
 426 my $MAX_SATA_DISKS = 6;
 427 my $MAX_USB_DEVICES = 5;
 428 my $MAX_NETS = 32;
 429 my $MAX_UNUSED_DISKS = 8;
 430 my $MAX_HOSTPCI_DEVICES = 4;
 431 my $MAX_SERIAL_PORTS = 4;
 432 my $MAX_PARALLEL_PORTS = 3;
 433 my $MAX_NUMA = 8;
 434 my $MAX_MEM = 4194304;
 435 my $STATICMEM = 1024;
 436
 437 my $numadesc = {
 438     optional => 1,
 439     type => 'string', format => 'pve-qm-numanode',
 440     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 441     description => "numa topology",
 442 };
 443 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 444
 445 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 446     $confdesc->{"numa$i"} = $numadesc;
 447 }
 448
 449 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 450                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 451                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 452 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 453
 454 my $netdesc = {
 455     optional => 1,
 456     type => 'string', format => 'pve-qm-net',
 457     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 458     description => <<EODESCR,
 459 Specify network devices.
 460
 461 MODEL is one of: $nic_model_list_txt
 462
 463 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 464 automatically generated if not specified.
 465
 466 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 467
 468 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 469
 470 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 471
 472 10.0.2.2   Gateway
 473 10.0.2.3   DNS Server
 474 10.0.2.4   SMB Server
 475
 476 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 477
 478 EODESCR
 479 };
 480 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 481
 482 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 483     $confdesc->{"net$i"} = $netdesc;
 484 }
 485
 486 my $drivename_hash;
 487
 488 my %drivedesc_base = (
 489     volume => { alias => 'file' },
 490     file => {
 491         type => 'pve-volume-id',
 492         default_key => 1,
 493         format_description => 'volume',
 494         description => "The drive's backing volume.",
 495     },
 496     media => {
 497         type => 'string',
 498         format_description => 'cdrom|disk',
 499         enum => [qw(cdrom disk)],
 500         description => "The drive's media type.",
 501         default => 'disk',
 502         optional => 1
 503     },
 504     cyls => {
 505         type => 'integer',
 506         format_description => 'count',
 507         description => "Force the drive's physical geometry to have a specific cylinder count.",
 508         optional => 1
 509     },
 510     heads => {
 511         type => 'integer',
 512         format_description => 'count',
 513         description => "Force the drive's physical geometry to have a specific head count.",
 514         optional => 1
 515     },
 516     secs => {
 517         type => 'integer',
 518         format_description => 'count',
 519         description => "Force the drive's physical geometry to have a specific sector count.",
 520         optional => 1
 521     },
 522     trans => {
 523         type => 'string',
 524         format_description => 'none|lba|auto',
 525         enum => [qw(none lba auto)],
 526         description => "Force disk geometry bios translation mode.",
 527         optional => 1,
 528     },
 529     snapshot => {
 530         type => 'boolean',
 531         format_description => 'on|off',
 532         description => "Whether the drive should be included when making snapshots.",
 533         optional => 1,
 534     },
 535     cache => {
 536         type => 'string',
 537         format_description => 'none|writethrough|writeback|unsafe|directsync',
 538         enum => [qw(none writethrough writeback unsafe directsync)],
 539         description => "The drive's cache mode",
 540         optional => 1,
 541     },
 542     format => {
 543         type => 'string',
 544         format_description => 'drive format',
 545         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 546         description => "The drive's backing file's data format.",
 547         optional => 1,
 548     },
 549     size => {
 550         type => 'disk-size',
 551         description => "Disk size. This is purely informational and has no effect.",
 552         optional => 1,
 553     },
 554     backup => {
 555         type => 'boolean',
 556         format_description => 'on|off',
 557         description => "Whether the drive should be included when making backups.",
 558         optional => 1,
 559     },
 560     werror => {
 561         type => 'string',
 562         format_description => 'enospc|ignore|report|stop',
 563         enum => [qw(enospc ignore report stop)],
 564         description => 'Write error action.',
 565         optional => 1,
 566     },
 567     aio => {
 568         type => 'string',
 569         format_description => 'native|threads',
 570         enum => [qw(native threads)],
 571         description => 'AIO type to use.',
 572         optional => 1,
 573     },
 574     discard => {
 575         type => 'string',
 576         format_description => 'ignore|on',
 577         enum => [qw(ignore on)],
 578         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 579         optional => 1,
 580     },
 581     detect_zeroes => {
 582         type => 'boolean',
 583         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 584         optional => 1,
 585     },
 586     serial => {
 587         type => 'string',
 588         format_description => 'serial',
 589         description => "The drive's reported serial number.",
 590         optional => 1,
 591     }
 592 );
 593
 594 my %rerror_fmt = (
 595     rerror => {
 596         type => 'string',
 597         format_description => 'ignore|report|stop',
 598         enum => [qw(ignore report stop)],
 599         description => 'Read error action.',
 600         optional => 1,
 601     },
 602 );
 603
 604 my %iothread_fmt = ( iothread => {
 605         type => 'boolean',
 606         format_description => 'off|on',
 607         description => "Whether to use iothreads for this drive",
 608         optional => 1,
 609 });
 610
 611 my %model_fmt = (
 612     model => {
 613         type => 'string',
 614         format_description => 'model',
 615         description => "The drive's reported model name.",
 616         optional => 1,
 617     },
 618 );
 619
 620 my %queues_fmt = (
 621     queues => {
 622         type => 'integer',
 623         format_description => 'nbqueues',
 624         description => "Number of queues.",
 625         minimum => 2,
 626         optional => 1
 627     }
 628 );
 629
 630 my $add_throttle_desc = sub {
 631     my ($key, $type, $what, $size, $longsize) = @_;
 632     $drivedesc_base{$key} = {
 633         type => $type,
 634         format_description => $size,
 635         description => "Maximum $what speed in $longsize per second.",
 636         optional => 1,
 637     };
 638 };
 639 # throughput: (leaky bucket)
 640 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 641 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 642 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 643 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 644 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 645 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 646 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 647 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 648 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 649
 650 # pools: (pool of IO before throttling starts taking effect)
 651 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 652 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 653 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 654 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 655 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 656 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 657
 658 my $ide_fmt = {
 659     %drivedesc_base,
 660     %rerror_fmt,
 661     %model_fmt,
 662 };
 663
 664 my $idedesc = {
 665     optional => 1,
 666     type => 'string', format => $ide_fmt,
 667     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 668 };
 669 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 670
 671 my $scsi_fmt = {
 672     %drivedesc_base,
 673     %iothread_fmt,
 674     %queues_fmt,
 675 };
 676 my $scsidesc = {
 677     optional => 1,
 678     type => 'string', format => $scsi_fmt,
 679     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 680 };
 681 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 682
 683 my $sata_fmt = {
 684     %drivedesc_base,
 685     %rerror_fmt,
 686 };
 687 my $satadesc = {
 688     optional => 1,
 689     type => 'string', format => $sata_fmt,
 690     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 691 };
 692 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 693
 694 my $virtio_fmt = {
 695     %drivedesc_base,
 696     %iothread_fmt,
 697     %rerror_fmt,
 698 };
 699 my $virtiodesc = {
 700     optional => 1,
 701     type => 'string', format => $virtio_fmt,
 702     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 703 };
 704 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 705
 706 my $alldrive_fmt = {
 707     %drivedesc_base,
 708     %rerror_fmt,
 709     %iothread_fmt,
 710     %model_fmt,
 711     %queues_fmt,
 712 };
 713
 714 my $usbdesc = {
 715     optional => 1,
 716     type => 'string', format => 'pve-qm-usb-device',
 717     typetext => 'host=HOSTUSBDEVICE|spice',
 718     description => <<EODESCR,
 719 Configure an USB device (n is 0 to 4). This can be used to
 720 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 721
 722 'bus-port(.port)*' (decimal numbers) or
 723 'vendor_id:product_id' (hexadeciaml numbers)
 724
 725 You can use the 'lsusb -t' command to list existing usb devices.
 726
 727 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 728
 729 The value 'spice' can be used to add a usb redirection devices for spice.
 730
 731 EODESCR
 732 };
 733 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 734
 735 my $hostpcidesc = {
 736         optional => 1,
 737         type => 'string', format => 'pve-qm-hostpci',
 738         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 739         description => <<EODESCR,
 740 Map host pci devices. HOSTPCIDEVICE syntax is:
 741
 742 'bus:dev.func' (hexadecimal numbers)
 743
 744 You can us the 'lspci' command to list existing pci devices.
 745
 746 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 747
 748 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 749
 750 Experimental: user reported problems with this option.
 751 EODESCR
 752 };
 753 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 754
 755 my $serialdesc = {
 756         optional => 1,
 757         type => 'string',
 758         pattern => '(/dev/.+|socket)',
 759         description =>  <<EODESCR,
 760 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 761
 762 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 763
 764 Experimental: user reported problems with this option.
 765 EODESCR
 766 };
 767
 768 my $paralleldesc= {
 769         optional => 1,
 770         type => 'string',
 771         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 772         description =>  <<EODESCR,
 773 Map host parallel devices (n is 0 to 2).
 774
 775 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 776
 777 Experimental: user reported problems with this option.
 778 EODESCR
 779 };
 780
 781 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 782     $confdesc->{"parallel$i"} = $paralleldesc;
 783 }
 784
 785 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 786     $confdesc->{"serial$i"} = $serialdesc;
 787 }
 788
 789 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 790     $confdesc->{"hostpci$i"} = $hostpcidesc;
 791 }
 792
 793 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 794     $drivename_hash->{"ide$i"} = 1;
 795     $confdesc->{"ide$i"} = $idedesc;
 796 }
 797
 798 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 799     $drivename_hash->{"sata$i"} = 1;
 800     $confdesc->{"sata$i"} = $satadesc;
 801 }
 802
 803 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 804     $drivename_hash->{"scsi$i"} = 1;
 805     $confdesc->{"scsi$i"} = $scsidesc ;
 806 }
 807
 808 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 809     $drivename_hash->{"virtio$i"} = 1;
 810     $confdesc->{"virtio$i"} = $virtiodesc;
 811 }
 812
 813 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 814     $confdesc->{"usb$i"} = $usbdesc;
 815 }
 816
 817 my $unuseddesc = {
 818     optional => 1,
 819     type => 'string', format => 'pve-volume-id',
 820     description => "Reference to unused volumes.",
 821 };
 822
 823 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 824     $confdesc->{"unused$i"} = $unuseddesc;
 825 }
 826
 827 my $kvm_api_version = 0;
 828
 829 sub kvm_version {
 830
 831     return $kvm_api_version if $kvm_api_version;
 832
 833     my $fh = IO::File->new("</dev/kvm") ||
 834         return 0;
 835
 836     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 837         $kvm_api_version = $v;
 838     }
 839
 840     $fh->close();
 841
 842     return  $kvm_api_version;
 843 }
 844
 845 my $kvm_user_version;
 846
 847 sub kvm_user_version {
 848
 849     return $kvm_user_version if $kvm_user_version;
 850
 851     $kvm_user_version = 'unknown';
 852
 853     my $tmp = `kvm -help 2>/dev/null`;
 854
 855     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 856         $kvm_user_version = $2;
 857     }
 858
 859     return $kvm_user_version;
 860
 861 }
 862
 863 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 864
 865 sub disknames {
 866     # order is important - used to autoselect boot disk
 867     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 868             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 869             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 870             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 871 }
 872
 873 sub valid_drivename {
 874     my $dev = shift;
 875
 876     return defined($drivename_hash->{$dev});
 877 }
 878
 879 sub option_exists {
 880     my $key = shift;
 881     return defined($confdesc->{$key});
 882 }
 883
 884 sub nic_models {
 885     return $nic_model_list;
 886 }
 887
 888 sub os_list_description {
 889
 890     return {
 891         other => 'Other',
 892         wxp => 'Windows XP',
 893         w2k => 'Windows 2000',
 894         w2k3 =>, 'Windows 2003',
 895         w2k8 => 'Windows 2008',
 896         wvista => 'Windows Vista',
 897         win7 => 'Windows 7',
 898         win8 => 'Windows 8/2012',
 899         l24 => 'Linux 2.4',
 900         l26 => 'Linux 2.6',
 901     };
 902 }
 903
 904 my $cdrom_path;
 905
 906 sub get_cdrom_path {
 907
 908     return  $cdrom_path if $cdrom_path;
 909
 910     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 911     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 912     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 913 }
 914
 915 sub get_iso_path {
 916     my ($storecfg, $vmid, $cdrom) = @_;
 917
 918     if ($cdrom eq 'cdrom') {
 919         return get_cdrom_path();
 920     } elsif ($cdrom eq 'none') {
 921         return '';
 922     } elsif ($cdrom =~ m|^/|) {
 923         return $cdrom;
 924     } else {
 925         return PVE::Storage::path($storecfg, $cdrom);
 926     }
 927 }
 928
 929 # try to convert old style file names to volume IDs
 930 sub filename_to_volume_id {
 931     my ($vmid, $file, $media) = @_;
 932
 933     if (!($file eq 'none' || $file eq 'cdrom' ||
 934           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 935
 936         return undef if $file =~ m|/|;
 937
 938         if ($media && $media eq 'cdrom') {
 939             $file = "local:iso/$file";
 940         } else {
 941             $file = "local:$vmid/$file";
 942         }
 943     }
 944
 945     return $file;
 946 }
 947
 948 sub verify_media_type {
 949     my ($opt, $vtype, $media) = @_;
 950
 951     return if !$media;
 952
 953     my $etype;
 954     if ($media eq 'disk') {
 955         $etype = 'images';
 956     } elsif ($media eq 'cdrom') {
 957         $etype = 'iso';
 958     } else {
 959         die "internal error";
 960     }
 961
 962     return if ($vtype eq $etype);
 963
 964     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 965 }
 966
 967 sub cleanup_drive_path {
 968     my ($opt, $storecfg, $drive) = @_;
 969
 970     # try to convert filesystem paths to volume IDs
 971
 972     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 973         ($drive->{file} !~ m|^/dev/.+|) &&
 974         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 975         ($drive->{file} !~ m/^\d+$/)) {
 976         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 977         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 978         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 979         verify_media_type($opt, $vtype, $drive->{media});
 980         $drive->{file} = $volid;
 981     }
 982
 983     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 984 }
 985
 986 sub create_conf_nolock {
 987     my ($vmid, $settings) = @_;
 988
 989     my $filename = config_file($vmid);
 990
 991     die "configuration file '$filename' already exists\n" if -f $filename;
 992
 993     my $defaults = load_defaults();
 994
 995     $settings->{name} = "vm$vmid" if !$settings->{name};
 996     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 997
 998     my $data = '';
 999     foreach my $opt (keys %$settings) {
1000         next if !$confdesc->{$opt};
1001
1002         my $value = $settings->{$opt};
1003         next if !$value;
1004
1005         $data .= "$opt: $value\n";
1006     }
1007
1008     PVE::Tools::file_set_contents($filename, $data);
1009 }
1010
1011 sub parse_hotplug_features {
1012     my ($data) = @_;
1013
1014     my $res = {};
1015
1016     return $res if $data eq '0';
1017
1018     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1019
1020     foreach my $feature (PVE::Tools::split_list($data)) {
1021         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1022             $res->{$1} = 1;
1023         } else {
1024             warn "ignoring unknown hotplug feature '$feature'\n";
1025         }
1026     }
1027     return $res;
1028 }
1029
1030 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1031 sub pve_verify_hotplug_features {
1032     my ($value, $noerr) = @_;
1033
1034     return $value if parse_hotplug_features($value);
1035
1036     return undef if $noerr;
1037
1038     die "unable to parse hotplug option\n";
1039 }
1040
1041 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1042 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1043 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1044 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1045 #        [,iothread=on][,serial=serial][,model=model]
1046
1047 sub parse_drive {
1048     my ($key, $data) = @_;
1049
1050     my ($interface, $index);
1051
1052     if ($key =~ m/^([^\d]+)(\d+)$/) {
1053         $interface = $1;
1054         $index = $2;
1055     } else {
1056         return undef;
1057     }
1058
1059     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1060                                      : $confdesc->{$key}->{format};
1061     if (!$desc) {
1062         warn "invalid drive key: $key\n";
1063         return undef;
1064     }
1065     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1066     return undef if !$res;
1067     $res->{interface} = $interface;
1068     $res->{index} = $index;
1069
1070     my $error = 0;
1071     foreach my $opt (qw(bps bps_rd bps_wr)) {
1072         if (my $bps = defined(delete $res->{$opt})) {
1073             if (defined($res->{"m$opt"})) {
1074                 warn "both $opt and m$opt specified\n";
1075                 ++$error;
1076                 next;
1077             }
1078             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1079         }
1080     }
1081     return undef if $error;
1082
1083     return undef if $res->{mbps_rd} && $res->{mbps};
1084     return undef if $res->{mbps_wr} && $res->{mbps};
1085     return undef if $res->{iops_rd} && $res->{iops};
1086     return undef if $res->{iops_wr} && $res->{iops};
1087
1088     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1089         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1090         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1091         return undef if $res->{interface} eq 'virtio';
1092     }
1093
1094     if (my $size = $res->{size}) {
1095         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1096     }
1097
1098     return $res;
1099 }
1100
1101 sub print_drive {
1102     my ($vmid, $drive) = @_;
1103     my $data = { %$drive };
1104     delete $data->{$_} for qw(index interface);
1105     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1106 }
1107
1108 sub scsi_inquiry {
1109     my($fh, $noerr) = @_;
1110
1111     my $SG_IO = 0x2285;
1112     my $SG_GET_VERSION_NUM = 0x2282;
1113
1114     my $versionbuf = "\x00" x 8;
1115     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1116     if (!$ret) {
1117         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1118         return undef;
1119     }
1120     my $version = unpack("I", $versionbuf);
1121     if ($version < 30000) {
1122         die "scsi generic interface too old\n"  if !$noerr;
1123         return undef;
1124     }
1125
1126     my $buf = "\x00" x 36;
1127     my $sensebuf = "\x00" x 8;
1128     my $cmd = pack("C x3 C x1", 0x12, 36);
1129
1130     # see /usr/include/scsi/sg.h
1131     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1132
1133     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1134                       length($sensebuf), 0, length($buf), $buf,
1135                       $cmd, $sensebuf, 6000);
1136
1137     $ret = ioctl($fh, $SG_IO, $packet);
1138     if (!$ret) {
1139         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1140         return undef;
1141     }
1142
1143     my @res = unpack($sg_io_hdr_t, $packet);
1144     if ($res[17] || $res[18]) {
1145         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1146         return undef;
1147     }
1148
1149     my $res = {};
1150     (my $byte0, my $byte1, $res->{vendor},
1151      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1152
1153     $res->{removable} = $byte1 & 128 ? 1 : 0;
1154     $res->{type} = $byte0 & 31;
1155
1156     return $res;
1157 }
1158
1159 sub path_is_scsi {
1160     my ($path) = @_;
1161
1162     my $fh = IO::File->new("+<$path") || return undef;
1163     my $res = scsi_inquiry($fh, 1);
1164     close($fh);
1165
1166     return $res;
1167 }
1168
1169 sub machine_type_is_q35 {
1170     my ($conf) = @_;
1171
1172     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1173 }
1174
1175 sub print_tabletdevice_full {
1176     my ($conf) = @_;
1177
1178     my $q35 = machine_type_is_q35($conf);
1179
1180     # we use uhci for old VMs because tablet driver was buggy in older qemu
1181     my $usbbus = $q35 ? "ehci" : "uhci";
1182
1183     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1184 }
1185
1186 sub print_drivedevice_full {
1187     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1188
1189     my $device = '';
1190     my $maxdev = 0;
1191
1192     if ($drive->{interface} eq 'virtio') {
1193         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1194         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1195         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1196     } elsif ($drive->{interface} eq 'scsi') {
1197
1198         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1199         my $unit = $drive->{index} % $maxdev;
1200         my $devicetype = 'hd';
1201         my $path = '';
1202         if (drive_is_cdrom($drive)) {
1203               $devicetype = 'cd';
1204         } else {
1205               if ($drive->{file} =~ m|^/|) {
1206                   $path = $drive->{file};
1207               } else {
1208                   $path = PVE::Storage::path($storecfg, $drive->{file});
1209               }
1210
1211               if($path =~ m/^iscsi\:\/\//){
1212                   $devicetype = 'generic';
1213               } else {
1214                   if (my $info = path_is_scsi($path)) {
1215                       if ($info->{type} == 0) {
1216                           $devicetype = 'block';
1217                       } elsif ($info->{type} == 1) { # tape
1218                           $devicetype = 'generic';
1219                       }
1220                   }
1221               }
1222          }
1223
1224         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1225             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1226         } else {
1227             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1228         }
1229
1230     } elsif ($drive->{interface} eq 'ide'){
1231         $maxdev = 2;
1232         my $controller = int($drive->{index} / $maxdev);
1233         my $unit = $drive->{index} % $maxdev;
1234         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1235
1236         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1237         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1238             $device .= ",model=$model";
1239         }
1240     } elsif ($drive->{interface} eq 'sata'){
1241         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1242         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1243         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1244     } elsif ($drive->{interface} eq 'usb') {
1245         die "implement me";
1246         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1247     } else {
1248         die "unsupported interface type";
1249     }
1250
1251     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1252
1253     return $device;
1254 }
1255
1256 sub get_initiator_name {
1257     my $initiator;
1258
1259     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1260     while (defined(my $line = <$fh>)) {
1261         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1262         $initiator = $1;
1263         last;
1264     }
1265     $fh->close();
1266
1267     return $initiator;
1268 }
1269
1270 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1271 sub print_drive_full {
1272     my ($storecfg, $vmid, $drive) = @_;
1273
1274     my $path;
1275     my $volid = $drive->{file};
1276     my $format;
1277
1278     if (drive_is_cdrom($drive)) {
1279         $path = get_iso_path($storecfg, $vmid, $volid);
1280     } else {
1281         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1282         if ($storeid) {
1283             $path = PVE::Storage::path($storecfg, $volid);
1284             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1285             $format = qemu_img_format($scfg, $volname);
1286         } else {
1287             $path = $volid;
1288             $format = "raw";
1289         }
1290    }
1291
1292     my $opts = '';
1293     foreach my $o (@qemu_drive_options) {
1294         next if $o eq 'bootindex';
1295         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1296     }
1297
1298     $opts .= ",format=$format" if $format && !$drive->{format};
1299
1300     foreach my $o (qw(bps bps_rd bps_wr)) {
1301         my $v = $drive->{"m$o"};
1302         $opts .= ",$o=" . int($v*1024*1024) if $v;
1303     }
1304
1305     my $cache_direct = 0;
1306
1307     if (my $cache = $drive->{cache}) {
1308         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1309     } elsif (!drive_is_cdrom($drive)) {
1310         $opts .= ",cache=none";
1311         $cache_direct = 1;
1312     }
1313
1314     # aio native works only with O_DIRECT
1315     if (!$drive->{aio}) {
1316         if($cache_direct) {
1317             $opts .= ",aio=native";
1318         } else {
1319             $opts .= ",aio=threads";
1320         }
1321     }
1322
1323     if (!drive_is_cdrom($drive)) {
1324         my $detectzeroes;
1325         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1326             $detectzeroes = 'off';
1327         } elsif ($drive->{discard}) {
1328             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1329         } else {
1330             # This used to be our default with discard not being specified:
1331             $detectzeroes = 'on';
1332         }
1333         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1334     }
1335
1336     my $pathinfo = $path ? "file=$path," : '';
1337
1338     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1339 }
1340
1341 sub print_netdevice_full {
1342     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1343
1344     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1345
1346     my $device = $net->{model};
1347     if ($net->{model} eq 'virtio') {
1348          $device = 'virtio-net-pci';
1349      };
1350
1351     my $pciaddr = print_pci_addr("$netid", $bridges);
1352     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1353     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1354         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1355         my $vectors = $net->{queues} * 2 + 2;
1356         $tmpstr .= ",vectors=$vectors,mq=on";
1357     }
1358     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1359
1360     if ($use_old_bios_files) {
1361         my $romfile;
1362         if ($device eq 'virtio-net-pci') {
1363             $romfile = 'pxe-virtio.rom';
1364         } elsif ($device eq 'e1000') {
1365             $romfile = 'pxe-e1000.rom';
1366         } elsif ($device eq 'ne2k') {
1367             $romfile = 'pxe-ne2k_pci.rom';
1368         } elsif ($device eq 'pcnet') {
1369             $romfile = 'pxe-pcnet.rom';
1370         } elsif ($device eq 'rtl8139') {
1371             $romfile = 'pxe-rtl8139.rom';
1372         }
1373         $tmpstr .= ",romfile=$romfile" if $romfile;
1374     }
1375
1376     return $tmpstr;
1377 }
1378
1379 sub print_netdev_full {
1380     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1381
1382     my $i = '';
1383     if ($netid =~ m/^net(\d+)$/) {
1384         $i = int($1);
1385     }
1386
1387     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1388
1389     my $ifname = "tap${vmid}i$i";
1390
1391     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1392     die "interface name '$ifname' is too long (max 15 character)\n"
1393         if length($ifname) >= 16;
1394
1395     my $vhostparam = '';
1396     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1397
1398     my $vmname = $conf->{name} || "vm$vmid";
1399
1400     my $netdev = "";
1401     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1402
1403     if ($net->{bridge}) {
1404         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1405     } else {
1406         $netdev = "type=user,id=$netid,hostname=$vmname";
1407     }
1408
1409     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1410
1411     return $netdev;
1412 }
1413
1414 sub drive_is_cdrom {
1415     my ($drive) = @_;
1416
1417     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1418
1419 }
1420
1421 sub parse_numa {
1422     my ($data) = @_;
1423
1424     my $res = {};
1425
1426     foreach my $kvp (split(/,/, $data)) {
1427
1428         if ($kvp =~ m/^memory=(\S+)$/) {
1429             $res->{memory} = $1;
1430         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1431             $res->{policy} = $1;
1432         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1433             $res->{cpus}->{start} = $1;
1434             $res->{cpus}->{end} = $3;
1435         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1436             $res->{hostnodes}->{start} = $1;
1437             $res->{hostnodes}->{end} = $3;
1438         } else {
1439             return undef;
1440         }
1441     }
1442
1443     return $res;
1444 }
1445
1446 sub parse_hostpci {
1447     my ($value) = @_;
1448
1449     return undef if !$value;
1450
1451
1452     my @list = split(/,/, $value);
1453     my $found;
1454
1455     my $res = {};
1456     foreach my $kv (@list) {
1457
1458         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1459             $found = 1;
1460             if(defined($4)){
1461                 push @{$res->{pciid}}, { id => $2 , function => $4};
1462
1463             }else{
1464                 my $pcidevices = lspci($2);
1465                 $res->{pciid} = $pcidevices->{$2};
1466             }
1467         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1468             $res->{rombar} = $1;
1469         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1470             $res->{'x-vga'} = $1;
1471         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1472             $res->{pcie} = 1 if $1 == 1;
1473         } else {
1474             warn "unknown hostpci setting '$kv'\n";
1475         }
1476     }
1477
1478     return undef if !$found;
1479
1480     return $res;
1481 }
1482
1483 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1484 sub parse_net {
1485     my ($data) = @_;
1486
1487     my $res = {};
1488
1489     foreach my $kvp (split(/,/, $data)) {
1490
1491         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1492             my $model = lc($1);
1493             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1494             $res->{model} = $model;
1495             $res->{macaddr} = $mac;
1496         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1497             $res->{bridge} = $1;
1498         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1499             $res->{queues} = $1;
1500         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1501             $res->{rate} = $1;
1502         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1503             $res->{tag} = $1;
1504         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1505             $res->{trunks} = $1;
1506         } elsif ($kvp =~ m/^firewall=([01])$/) {
1507             $res->{firewall} = $1;
1508         } elsif ($kvp =~ m/^link_down=([01])$/) {
1509             $res->{link_down} = $1;
1510         } else {
1511             return undef;
1512         }
1513
1514     }
1515
1516     return undef if !$res->{model};
1517
1518     return $res;
1519 }
1520
1521 sub print_net {
1522     my $net = shift;
1523
1524     my $res = "$net->{model}";
1525     $res .= "=$net->{macaddr}" if $net->{macaddr};
1526     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1527     $res .= ",rate=$net->{rate}" if $net->{rate};
1528     $res .= ",tag=$net->{tag}" if $net->{tag};
1529     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1530     $res .= ",firewall=1" if $net->{firewall};
1531     $res .= ",link_down=1" if $net->{link_down};
1532     $res .= ",queues=$net->{queues}" if $net->{queues};
1533
1534     return $res;
1535 }
1536
1537 sub add_random_macs {
1538     my ($settings) = @_;
1539
1540     foreach my $opt (keys %$settings) {
1541         next if $opt !~ m/^net(\d+)$/;
1542         my $net = parse_net($settings->{$opt});
1543         next if !$net;
1544         $settings->{$opt} = print_net($net);
1545     }
1546 }
1547
1548 sub add_unused_volume {
1549     my ($config, $volid) = @_;
1550
1551     my $key;
1552     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1553         my $test = "unused$ind";
1554         if (my $vid = $config->{$test}) {
1555             return if $vid eq $volid; # do not add duplicates
1556         } else {
1557             $key = $test;
1558         }
1559     }
1560
1561     die "To many unused volume - please delete them first.\n" if !$key;
1562
1563     $config->{$key} = $volid;
1564
1565     return $key;
1566 }
1567
1568 sub vm_is_volid_owner {
1569     my ($storecfg, $vmid, $volid) = @_;
1570
1571     if ($volid !~  m|^/|) {
1572         my ($path, $owner);
1573         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1574         if ($owner && ($owner == $vmid)) {
1575             return 1;
1576         }
1577     }
1578
1579     return undef;
1580 }
1581
1582 sub split_flagged_list {
1583     my $text = shift || '';
1584     $text =~ s/[,;]/ /g;
1585     $text =~ s/^\s+//;
1586     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1587 }
1588
1589 sub join_flagged_list {
1590     my ($how, $lst) = @_;
1591     join $how, map { $lst->{$_} . $_ } keys %$lst;
1592 }
1593
1594 sub vmconfig_delete_pending_option {
1595     my ($conf, $key, $force) = @_;
1596
1597     delete $conf->{pending}->{$key};
1598     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1599     $pending_delete_hash->{$key} = $force ? '!' : '';
1600     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1601 }
1602
1603 sub vmconfig_undelete_pending_option {
1604     my ($conf, $key) = @_;
1605
1606     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1607     delete $pending_delete_hash->{$key};
1608
1609     if (%$pending_delete_hash) {
1610         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1611     } else {
1612         delete $conf->{pending}->{delete};
1613     }
1614 }
1615
1616 sub vmconfig_register_unused_drive {
1617     my ($storecfg, $vmid, $conf, $drive) = @_;
1618
1619     if (!drive_is_cdrom($drive)) {
1620         my $volid = $drive->{file};
1621         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1622             add_unused_volume($conf, $volid, $vmid);
1623         }
1624     }
1625 }
1626
1627 sub vmconfig_cleanup_pending {
1628     my ($conf) = @_;
1629
1630     # remove pending changes when nothing changed
1631     my $changes;
1632     foreach my $opt (keys %{$conf->{pending}}) {
1633         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1634             $changes = 1;
1635             delete $conf->{pending}->{$opt};
1636         }
1637     }
1638
1639     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1640     my $pending_delete_hash = {};
1641     while (my ($opt, $force) = each %$current_delete_hash) {
1642         if (defined($conf->{$opt})) {
1643             $pending_delete_hash->{$opt} = $force;
1644         } else {
1645             $changes = 1;
1646         }
1647     }
1648
1649     if (%$pending_delete_hash) {
1650         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1651     } else {
1652         delete $conf->{pending}->{delete};
1653     }
1654
1655     return $changes;
1656 }
1657
1658 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1659 my $smbios1_desc = {
1660     uuid => {
1661         type => 'string',
1662         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1663         format_description => 'UUID',
1664         optional => 1,
1665     },
1666     version => {
1667         type => 'string',
1668         pattern => '\S+',
1669         format_description => 'str',
1670         optional => 1,
1671     },
1672     serial => {
1673         type => 'string',
1674         pattern => '\S+',
1675         format_description => 'str',
1676         optional => 1,
1677     },
1678     manufacturer => {
1679         type => 'string',
1680         pattern => '\S+',
1681         format_description => 'name',
1682         optional => 1,
1683     },
1684     product => {
1685         type => 'string',
1686         pattern => '\S+',
1687         format_description => 'name',
1688         optional => 1,
1689     },
1690     sku => {
1691         type => 'string',
1692         pattern => '\S+',
1693         format_description => 'str',
1694         optional => 1,
1695     },
1696     family => {
1697         type => 'string',
1698         pattern => '\S+',
1699         format_description => 'str',
1700         optional => 1,
1701     },
1702 };
1703
1704 sub parse_smbios1 {
1705     my ($data) = @_;
1706
1707     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1708     warn $@ if $@;
1709     return $res;
1710 }
1711
1712 sub print_smbios1 {
1713     my ($smbios1) = @_;
1714     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1715 }
1716
1717 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1718
1719 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1720 sub verify_bootdisk {
1721     my ($value, $noerr) = @_;
1722
1723     return $value if valid_drivename($value);
1724
1725     return undef if $noerr;
1726
1727     die "invalid boot disk '$value'\n";
1728 }
1729
1730 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1731 sub verify_numa {
1732     my ($value, $noerr) = @_;
1733
1734     return $value if parse_numa($value);
1735
1736     return undef if $noerr;
1737
1738     die "unable to parse numa options\n";
1739 }
1740
1741 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1742 sub verify_net {
1743     my ($value, $noerr) = @_;
1744
1745     return $value if parse_net($value);
1746
1747     return undef if $noerr;
1748
1749     die "unable to parse network options\n";
1750 }
1751
1752 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1753 sub verify_hostpci {
1754     my ($value, $noerr) = @_;
1755
1756     return $value if parse_hostpci($value);
1757
1758     return undef if $noerr;
1759
1760     die "unable to parse pci id\n";
1761 }
1762
1763 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1764 sub verify_watchdog {
1765     my ($value, $noerr) = @_;
1766
1767     return $value if parse_watchdog($value);
1768
1769     return undef if $noerr;
1770
1771     die "unable to parse watchdog options\n";
1772 }
1773
1774 sub parse_watchdog {
1775     my ($value) = @_;
1776
1777     return undef if !$value;
1778
1779     my $res = {};
1780
1781     foreach my $p (split(/,/, $value)) {
1782         next if $p =~ m/^\s*$/;
1783
1784         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1785             $res->{model} = $2;
1786         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1787             $res->{action} = $2;
1788         } else {
1789             return undef;
1790         }
1791     }
1792
1793     return $res;
1794 }
1795
1796 sub parse_usb_device {
1797     my ($value) = @_;
1798
1799     return undef if !$value;
1800
1801     my @dl = split(/,/, $value);
1802     my $found;
1803
1804     my $res = {};
1805     foreach my $v (@dl) {
1806         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1807             $found = 1;
1808             $res->{vendorid} = $2;
1809             $res->{productid} = $4;
1810         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1811             $found = 1;
1812             $res->{hostbus} = $1;
1813             $res->{hostport} = $2;
1814         } elsif ($v =~ m/^spice$/) {
1815             $found = 1;
1816             $res->{spice} = 1;
1817         } else {
1818             return undef;
1819         }
1820     }
1821     return undef if !$found;
1822
1823     return $res;
1824 }
1825
1826 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1827 sub verify_usb_device {
1828     my ($value, $noerr) = @_;
1829
1830     return $value if parse_usb_device($value);
1831
1832     return undef if $noerr;
1833
1834     die "unable to parse usb device\n";
1835 }
1836
1837 # add JSON properties for create and set function
1838 sub json_config_properties {
1839     my $prop = shift;
1840
1841     foreach my $opt (keys %$confdesc) {
1842         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1843         $prop->{$opt} = $confdesc->{$opt};
1844     }
1845
1846     return $prop;
1847 }
1848
1849 sub check_type {
1850     my ($key, $value) = @_;
1851
1852     die "unknown setting '$key'\n" if !$confdesc->{$key};
1853
1854     my $type = $confdesc->{$key}->{type};
1855
1856     if (!defined($value)) {
1857         die "got undefined value\n";
1858     }
1859
1860     if ($value =~ m/[\n\r]/) {
1861         die "property contains a line feed\n";
1862     }
1863
1864     if ($type eq 'boolean') {
1865         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1866         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1867         die "type check ('boolean') failed - got '$value'\n";
1868     } elsif ($type eq 'integer') {
1869         return int($1) if $value =~ m/^(\d+)$/;
1870         die "type check ('integer') failed - got '$value'\n";
1871     } elsif ($type eq 'number') {
1872         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1873         die "type check ('number') failed - got '$value'\n";
1874     } elsif ($type eq 'string') {
1875         if (my $fmt = $confdesc->{$key}->{format}) {
1876             if ($fmt eq 'pve-qm-drive') {
1877                 # special case - we need to pass $key to parse_drive()
1878                 my $drive = parse_drive($key, $value);
1879                 return $value if $drive;
1880                 die "unable to parse drive options\n";
1881             }
1882             PVE::JSONSchema::check_format($fmt, $value);
1883             return $value;
1884         }
1885         $value =~ s/^\"(.*)\"$/$1/;
1886         return $value;
1887     } else {
1888         die "internal error"
1889     }
1890 }
1891
1892 sub lock_config_full {
1893     my ($vmid, $timeout, $code, @param) = @_;
1894
1895     my $filename = config_file_lock($vmid);
1896
1897     my $res = lock_file($filename, $timeout, $code, @param);
1898
1899     die $@ if $@;
1900
1901     return $res;
1902 }
1903
1904 sub lock_config_mode {
1905     my ($vmid, $timeout, $shared, $code, @param) = @_;
1906
1907     my $filename = config_file_lock($vmid);
1908
1909     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1910
1911     die $@ if $@;
1912
1913     return $res;
1914 }
1915
1916 sub lock_config {
1917     my ($vmid, $code, @param) = @_;
1918
1919     return lock_config_full($vmid, 10, $code, @param);
1920 }
1921
1922 sub cfs_config_path {
1923     my ($vmid, $node) = @_;
1924
1925     $node = $nodename if !$node;
1926     return "nodes/$node/qemu-server/$vmid.conf";
1927 }
1928
1929 sub check_iommu_support{
1930     #fixme : need to check IOMMU support
1931     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1932
1933     my $iommu=1;
1934     return $iommu;
1935
1936 }
1937
1938 sub config_file {
1939     my ($vmid, $node) = @_;
1940
1941     my $cfspath = cfs_config_path($vmid, $node);
1942     return "/etc/pve/$cfspath";
1943 }
1944
1945 sub config_file_lock {
1946     my ($vmid) = @_;
1947
1948     return "$lock_dir/lock-$vmid.conf";
1949 }
1950
1951 sub touch_config {
1952     my ($vmid) = @_;
1953
1954     my $conf = config_file($vmid);
1955     utime undef, undef, $conf;
1956 }
1957
1958 sub destroy_vm {
1959     my ($storecfg, $vmid, $keep_empty_config) = @_;
1960
1961     my $conffile = config_file($vmid);
1962
1963     my $conf = load_config($vmid);
1964
1965     check_lock($conf);
1966
1967     # only remove disks owned by this VM
1968     foreach_drive($conf, sub {
1969         my ($ds, $drive) = @_;
1970
1971         return if drive_is_cdrom($drive);
1972
1973         my $volid = $drive->{file};
1974
1975         return if !$volid || $volid =~ m|^/|;
1976
1977         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1978         return if !$path || !$owner || ($owner != $vmid);
1979
1980         PVE::Storage::vdisk_free($storecfg, $volid);
1981     });
1982
1983     if ($keep_empty_config) {
1984         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1985     } else {
1986         unlink $conffile;
1987     }
1988
1989     # also remove unused disk
1990     eval {
1991         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1992
1993         eval {
1994             PVE::Storage::foreach_volid($dl, sub {
1995                 my ($volid, $sid, $volname, $d) = @_;
1996                 PVE::Storage::vdisk_free($storecfg, $volid);
1997             });
1998         };
1999         warn $@ if $@;
2000
2001     };
2002     warn $@ if $@;
2003 }
2004
2005 sub load_config {
2006     my ($vmid, $node) = @_;
2007
2008     my $cfspath = cfs_config_path($vmid, $node);
2009
2010     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2011
2012     die "no such VM ('$vmid')\n" if !defined($conf);
2013
2014     return $conf;
2015 }
2016
2017 sub parse_vm_config {
2018     my ($filename, $raw) = @_;
2019
2020     return undef if !defined($raw);
2021
2022     my $res = {
2023         digest => Digest::SHA::sha1_hex($raw),
2024         snapshots => {},
2025         pending => {},
2026     };
2027
2028     $filename =~ m|/qemu-server/(\d+)\.conf$|
2029         || die "got strange filename '$filename'";
2030
2031     my $vmid = $1;
2032
2033     my $conf = $res;
2034     my $descr;
2035     my $section = '';
2036
2037     my @lines = split(/\n/, $raw);
2038     foreach my $line (@lines) {
2039         next if $line =~ m/^\s*$/;
2040
2041         if ($line =~ m/^\[PENDING\]\s*$/i) {
2042             $section = 'pending';
2043             if (defined($descr)) {
2044                 $descr =~ s/\s+$//;
2045                 $conf->{description} = $descr;
2046             }
2047             $descr = undef;
2048             $conf = $res->{$section} = {};
2049             next;
2050
2051         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2052             $section = $1;
2053             if (defined($descr)) {
2054                 $descr =~ s/\s+$//;
2055                 $conf->{description} = $descr;
2056             }
2057             $descr = undef;
2058             $conf = $res->{snapshots}->{$section} = {};
2059             next;
2060         }
2061
2062         if ($line =~ m/^\#(.*)\s*$/) {
2063             $descr = '' if !defined($descr);
2064             $descr .= PVE::Tools::decode_text($1) . "\n";
2065             next;
2066         }
2067
2068         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2069             $descr = '' if !defined($descr);
2070             $descr .= PVE::Tools::decode_text($2);
2071         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2072             $conf->{snapstate} = $1;
2073         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2074             my $key = $1;
2075             my $value = $2;
2076             $conf->{$key} = $value;
2077         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2078             my $value = $1;
2079             if ($section eq 'pending') {
2080                 $conf->{delete} = $value; # we parse this later
2081             } else {
2082                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2083             }
2084         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2085             my $key = $1;
2086             my $value = $2;
2087             eval { $value = check_type($key, $value); };
2088             if ($@) {
2089                 warn "vm $vmid - unable to parse value of '$key' - $@";
2090             } else {
2091                 my $fmt = $confdesc->{$key}->{format};
2092                 if ($fmt && $fmt eq 'pve-qm-drive') {
2093                     my $v = parse_drive($key, $value);
2094                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2095                         $v->{file} = $volid;
2096                         $value = print_drive($vmid, $v);
2097                     } else {
2098                         warn "vm $vmid - unable to parse value of '$key'\n";
2099                         next;
2100                     }
2101                 }
2102
2103                 if ($key eq 'cdrom') {
2104                     $conf->{ide2} = $value;
2105                 } else {
2106                     $conf->{$key} = $value;
2107                 }
2108             }
2109         }
2110     }
2111
2112     if (defined($descr)) {
2113         $descr =~ s/\s+$//;
2114         $conf->{description} = $descr;
2115     }
2116     delete $res->{snapstate}; # just to be sure
2117
2118     return $res;
2119 }
2120
2121 sub write_vm_config {
2122     my ($filename, $conf) = @_;
2123
2124     delete $conf->{snapstate}; # just to be sure
2125
2126     if ($conf->{cdrom}) {
2127         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2128         $conf->{ide2} = $conf->{cdrom};
2129         delete $conf->{cdrom};
2130     }
2131
2132     # we do not use 'smp' any longer
2133     if ($conf->{sockets}) {
2134         delete $conf->{smp};
2135     } elsif ($conf->{smp}) {
2136         $conf->{sockets} = $conf->{smp};
2137         delete $conf->{cores};
2138         delete $conf->{smp};
2139     }
2140
2141     my $used_volids = {};
2142
2143     my $cleanup_config = sub {
2144         my ($cref, $pending, $snapname) = @_;
2145
2146         foreach my $key (keys %$cref) {
2147             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2148                 $key eq 'snapstate' || $key eq 'pending';
2149             my $value = $cref->{$key};
2150             if ($key eq 'delete') {
2151                 die "propertry 'delete' is only allowed in [PENDING]\n"
2152                     if !$pending;
2153                 # fixme: check syntax?
2154                 next;
2155             }
2156             eval { $value = check_type($key, $value); };
2157             die "unable to parse value of '$key' - $@" if $@;
2158
2159             $cref->{$key} = $value;
2160
2161             if (!$snapname && valid_drivename($key)) {
2162                 my $drive = parse_drive($key, $value);
2163                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2164             }
2165         }
2166     };
2167
2168     &$cleanup_config($conf);
2169
2170     &$cleanup_config($conf->{pending}, 1);
2171
2172     foreach my $snapname (keys %{$conf->{snapshots}}) {
2173         die "internal error" if $snapname eq 'pending';
2174         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2175     }
2176
2177     # remove 'unusedX' settings if we re-add a volume
2178     foreach my $key (keys %$conf) {
2179         my $value = $conf->{$key};
2180         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2181             delete $conf->{$key};
2182         }
2183     }
2184
2185     my $generate_raw_config = sub {
2186         my ($conf, $pending) = @_;
2187
2188         my $raw = '';
2189
2190         # add description as comment to top of file
2191         if (defined(my $descr = $conf->{description})) {
2192             if ($descr) {
2193                 foreach my $cl (split(/\n/, $descr)) {
2194                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2195                 }
2196             } else {
2197                 $raw .= "#\n" if $pending;
2198             }
2199         }
2200
2201         foreach my $key (sort keys %$conf) {
2202             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2203             $raw .= "$key: $conf->{$key}\n";
2204         }
2205         return $raw;
2206     };
2207
2208     my $raw = &$generate_raw_config($conf);
2209
2210     if (scalar(keys %{$conf->{pending}})){
2211         $raw .= "\n[PENDING]\n";
2212         $raw .= &$generate_raw_config($conf->{pending}, 1);
2213     }
2214
2215     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2216         $raw .= "\n[$snapname]\n";
2217         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2218     }
2219
2220     return $raw;
2221 }
2222
2223 sub update_config_nolock {
2224     my ($vmid, $conf, $skiplock) = @_;
2225
2226     check_lock($conf) if !$skiplock;
2227
2228     my $cfspath = cfs_config_path($vmid);
2229
2230     PVE::Cluster::cfs_write_file($cfspath, $conf);
2231 }
2232
2233 sub update_config {
2234     my ($vmid, $conf, $skiplock) = @_;
2235
2236     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2237 }
2238
2239 sub load_defaults {
2240
2241     my $res = {};
2242
2243     # we use static defaults from our JSON schema configuration
2244     foreach my $key (keys %$confdesc) {
2245         if (defined(my $default = $confdesc->{$key}->{default})) {
2246             $res->{$key} = $default;
2247         }
2248     }
2249
2250     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2251     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2252
2253     return $res;
2254 }
2255
2256 sub config_list {
2257     my $vmlist = PVE::Cluster::get_vmlist();
2258     my $res = {};
2259     return $res if !$vmlist || !$vmlist->{ids};
2260     my $ids = $vmlist->{ids};
2261
2262     foreach my $vmid (keys %$ids) {
2263         my $d = $ids->{$vmid};
2264         next if !$d->{node} || $d->{node} ne $nodename;
2265         next if !$d->{type} || $d->{type} ne 'qemu';
2266         $res->{$vmid}->{exists} = 1;
2267     }
2268     return $res;
2269 }
2270
2271 # test if VM uses local resources (to prevent migration)
2272 sub check_local_resources {
2273     my ($conf, $noerr) = @_;
2274
2275     my $loc_res = 0;
2276
2277     $loc_res = 1 if $conf->{hostusb}; # old syntax
2278     $loc_res = 1 if $conf->{hostpci}; # old syntax
2279
2280     foreach my $k (keys %$conf) {
2281         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2282         # sockets are safe: they will recreated be on the target side post-migrate
2283         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2284         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2285     }
2286
2287     die "VM uses local resources\n" if $loc_res && !$noerr;
2288
2289     return $loc_res;
2290 }
2291
2292 # check if used storages are available on all nodes (use by migrate)
2293 sub check_storage_availability {
2294     my ($storecfg, $conf, $node) = @_;
2295
2296     foreach_drive($conf, sub {
2297         my ($ds, $drive) = @_;
2298
2299         my $volid = $drive->{file};
2300         return if !$volid;
2301
2302         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2303         return if !$sid;
2304
2305         # check if storage is available on both nodes
2306         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2307         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2308    });
2309 }
2310
2311 # list nodes where all VM images are available (used by has_feature API)
2312 sub shared_nodes {
2313     my ($conf, $storecfg) = @_;
2314
2315     my $nodelist = PVE::Cluster::get_nodelist();
2316     my $nodehash = { map { $_ => 1 } @$nodelist };
2317     my $nodename = PVE::INotify::nodename();
2318
2319     foreach_drive($conf, sub {
2320         my ($ds, $drive) = @_;
2321
2322         my $volid = $drive->{file};
2323         return if !$volid;
2324
2325         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2326         if ($storeid) {
2327             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2328             if ($scfg->{disable}) {
2329                 $nodehash = {};
2330             } elsif (my $avail = $scfg->{nodes}) {
2331                 foreach my $node (keys %$nodehash) {
2332                     delete $nodehash->{$node} if !$avail->{$node};
2333                 }
2334             } elsif (!$scfg->{shared}) {
2335                 foreach my $node (keys %$nodehash) {
2336                     delete $nodehash->{$node} if $node ne $nodename
2337                 }
2338             }
2339         }
2340     });
2341
2342     return $nodehash
2343 }
2344
2345 sub check_lock {
2346     my ($conf) = @_;
2347
2348     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2349 }
2350
2351 sub check_cmdline {
2352     my ($pidfile, $pid) = @_;
2353
2354     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2355     if (defined($fh)) {
2356         my $line = <$fh>;
2357         $fh->close;
2358         return undef if !$line;
2359         my @param = split(/\0/, $line);
2360
2361         my $cmd = $param[0];
2362         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2363
2364         for (my $i = 0; $i < scalar (@param); $i++) {
2365             my $p = $param[$i];
2366             next if !$p;
2367             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2368                 my $p = $param[$i+1];
2369                 return 1 if $p && ($p eq $pidfile);
2370                 return undef;
2371             }
2372         }
2373     }
2374     return undef;
2375 }
2376
2377 sub check_running {
2378     my ($vmid, $nocheck, $node) = @_;
2379
2380     my $filename = config_file($vmid, $node);
2381
2382     die "unable to find configuration file for VM $vmid - no such machine\n"
2383         if !$nocheck && ! -f $filename;
2384
2385     my $pidfile = pidfile_name($vmid);
2386
2387     if (my $fd = IO::File->new("<$pidfile")) {
2388         my $st = stat($fd);
2389         my $line = <$fd>;
2390         close($fd);
2391
2392         my $mtime = $st->mtime;
2393         if ($mtime > time()) {
2394             warn "file '$filename' modified in future\n";
2395         }
2396
2397         if ($line =~ m/^(\d+)$/) {
2398             my $pid = $1;
2399             if (check_cmdline($pidfile, $pid)) {
2400                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2401                     return $pid;
2402                 }
2403             }
2404         }
2405     }
2406
2407     return undef;
2408 }
2409
2410 sub vzlist {
2411
2412     my $vzlist = config_list();
2413
2414     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2415
2416     while (defined(my $de = $fd->read)) {
2417         next if $de !~ m/^(\d+)\.pid$/;
2418         my $vmid = $1;
2419         next if !defined($vzlist->{$vmid});
2420         if (my $pid = check_running($vmid)) {
2421             $vzlist->{$vmid}->{pid} = $pid;
2422         }
2423     }
2424
2425     return $vzlist;
2426 }
2427
2428 sub disksize {
2429     my ($storecfg, $conf) = @_;
2430
2431     my $bootdisk = $conf->{bootdisk};
2432     return undef if !$bootdisk;
2433     return undef if !valid_drivename($bootdisk);
2434
2435     return undef if !$conf->{$bootdisk};
2436
2437     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2438     return undef if !defined($drive);
2439
2440     return undef if drive_is_cdrom($drive);
2441
2442     my $volid = $drive->{file};
2443     return undef if !$volid;
2444
2445     return $drive->{size};
2446 }
2447
2448 my $last_proc_pid_stat;
2449
2450 # get VM status information
2451 # This must be fast and should not block ($full == false)
2452 # We only query KVM using QMP if $full == true (this can be slow)
2453 sub vmstatus {
2454     my ($opt_vmid, $full) = @_;
2455
2456     my $res = {};
2457
2458     my $storecfg = PVE::Storage::config();
2459
2460     my $list = vzlist();
2461     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2462
2463     my $cpucount = $cpuinfo->{cpus} || 1;
2464
2465     foreach my $vmid (keys %$list) {
2466         next if $opt_vmid && ($vmid ne $opt_vmid);
2467
2468         my $cfspath = cfs_config_path($vmid);
2469         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2470
2471         my $d = {};
2472         $d->{pid} = $list->{$vmid}->{pid};
2473
2474         # fixme: better status?
2475         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2476
2477         my $size = disksize($storecfg, $conf);
2478         if (defined($size)) {
2479             $d->{disk} = 0; # no info available
2480             $d->{maxdisk} = $size;
2481         } else {
2482             $d->{disk} = 0;
2483             $d->{maxdisk} = 0;
2484         }
2485
2486         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2487         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2488         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2489
2490         $d->{name} = $conf->{name} || "VM $vmid";
2491         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2492
2493         if ($conf->{balloon}) {
2494             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2495             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2496         }
2497
2498         $d->{uptime} = 0;
2499         $d->{cpu} = 0;
2500         $d->{mem} = 0;
2501
2502         $d->{netout} = 0;
2503         $d->{netin} = 0;
2504
2505         $d->{diskread} = 0;
2506         $d->{diskwrite} = 0;
2507
2508         $d->{template} = is_template($conf);
2509
2510         $res->{$vmid} = $d;
2511     }
2512
2513     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2514     foreach my $dev (keys %$netdev) {
2515         next if $dev !~ m/^tap([1-9]\d*)i/;
2516         my $vmid = $1;
2517         my $d = $res->{$vmid};
2518         next if !$d;
2519
2520         $d->{netout} += $netdev->{$dev}->{receive};
2521         $d->{netin} += $netdev->{$dev}->{transmit};
2522
2523         if ($full) {
2524             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2525             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2526         }
2527
2528     }
2529
2530     my $ctime = gettimeofday;
2531
2532     foreach my $vmid (keys %$list) {
2533
2534         my $d = $res->{$vmid};
2535         my $pid = $d->{pid};
2536         next if !$pid;
2537
2538         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2539         next if !$pstat; # not running
2540
2541         my $used = $pstat->{utime} + $pstat->{stime};
2542
2543         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2544
2545         if ($pstat->{vsize}) {
2546             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2547         }
2548
2549         my $old = $last_proc_pid_stat->{$pid};
2550         if (!$old) {
2551             $last_proc_pid_stat->{$pid} = {
2552                 time => $ctime,
2553                 used => $used,
2554                 cpu => 0,
2555             };
2556             next;
2557         }
2558
2559         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2560
2561         if ($dtime > 1000) {
2562             my $dutime = $used -  $old->{used};
2563
2564             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2565             $last_proc_pid_stat->{$pid} = {
2566                 time => $ctime,
2567                 used => $used,
2568                 cpu => $d->{cpu},
2569             };
2570         } else {
2571             $d->{cpu} = $old->{cpu};
2572         }
2573     }
2574
2575     return $res if !$full;
2576
2577     my $qmpclient = PVE::QMPClient->new();
2578
2579     my $ballooncb = sub {
2580         my ($vmid, $resp) = @_;
2581
2582         my $info = $resp->{'return'};
2583         return if !$info->{max_mem};
2584
2585         my $d = $res->{$vmid};
2586
2587         # use memory assigned to VM
2588         $d->{maxmem} = $info->{max_mem};
2589         $d->{balloon} = $info->{actual};
2590
2591         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2592             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2593             $d->{freemem} = $info->{free_mem};
2594         }
2595
2596         $d->{ballooninfo} = $info;
2597     };
2598
2599     my $blockstatscb = sub {
2600         my ($vmid, $resp) = @_;
2601         my $data = $resp->{'return'} || [];
2602         my $totalrdbytes = 0;
2603         my $totalwrbytes = 0;
2604
2605         for my $blockstat (@$data) {
2606             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2607             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2608
2609             $blockstat->{device} =~ s/drive-//;
2610             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2611         }
2612         $res->{$vmid}->{diskread} = $totalrdbytes;
2613         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2614     };
2615
2616     my $statuscb = sub {
2617         my ($vmid, $resp) = @_;
2618
2619         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2620         # this fails if ballon driver is not loaded, so this must be
2621         # the last commnand (following command are aborted if this fails).
2622         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2623
2624         my $status = 'unknown';
2625         if (!defined($status = $resp->{'return'}->{status})) {
2626             warn "unable to get VM status\n";
2627             return;
2628         }
2629
2630         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2631     };
2632
2633     foreach my $vmid (keys %$list) {
2634         next if $opt_vmid && ($vmid ne $opt_vmid);
2635         next if !$res->{$vmid}->{pid}; # not running
2636         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2637     }
2638
2639     $qmpclient->queue_execute(undef, 1);
2640
2641     foreach my $vmid (keys %$list) {
2642         next if $opt_vmid && ($vmid ne $opt_vmid);
2643         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2644     }
2645
2646     return $res;
2647 }
2648
2649 sub foreach_dimm {
2650     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2651
2652     my $dimm_id = 0;
2653     my $current_size = 1024;
2654     my $dimm_size = 512;
2655     return if $current_size == $memory;
2656
2657     for (my $j = 0; $j < 8; $j++) {
2658         for (my $i = 0; $i < 32; $i++) {
2659             my $name = "dimm${dimm_id}";
2660             $dimm_id++;
2661             my $numanode = $i % $sockets;
2662             $current_size += $dimm_size;
2663             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2664             return  $current_size if $current_size >= $memory;
2665         }
2666         $dimm_size *= 2;
2667     }
2668 }
2669
2670 sub foreach_reverse_dimm {
2671     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2672
2673     my $dimm_id = 253;
2674     my $current_size = 4177920;
2675     my $dimm_size = 65536;
2676     return if $current_size == $memory;
2677
2678     for (my $j = 0; $j < 8; $j++) {
2679         for (my $i = 0; $i < 32; $i++) {
2680             my $name = "dimm${dimm_id}";
2681             $dimm_id--;
2682             my $numanode = $i % $sockets;
2683             $current_size -= $dimm_size;
2684             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2685             return  $current_size if $current_size <= $memory;
2686         }
2687         $dimm_size /= 2;
2688     }
2689 }
2690
2691 sub foreach_drive {
2692     my ($conf, $func) = @_;
2693
2694     foreach my $ds (keys %$conf) {
2695         next if !valid_drivename($ds);
2696
2697         my $drive = parse_drive($ds, $conf->{$ds});
2698         next if !$drive;
2699
2700         &$func($ds, $drive);
2701     }
2702 }
2703
2704 sub foreach_volid {
2705     my ($conf, $func) = @_;
2706
2707     my $volhash = {};
2708
2709     my $test_volid = sub {
2710         my ($volid, $is_cdrom) = @_;
2711
2712         return if !$volid;
2713
2714         $volhash->{$volid} = $is_cdrom || 0;
2715     };
2716
2717     foreach_drive($conf, sub {
2718         my ($ds, $drive) = @_;
2719         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2720     });
2721
2722     foreach my $snapname (keys %{$conf->{snapshots}}) {
2723         my $snap = $conf->{snapshots}->{$snapname};
2724         &$test_volid($snap->{vmstate}, 0);
2725         foreach_drive($snap, sub {
2726             my ($ds, $drive) = @_;
2727             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2728         });
2729     }
2730
2731     foreach my $volid (keys %$volhash) {
2732         &$func($volid, $volhash->{$volid});
2733     }
2734 }
2735
2736 sub vga_conf_has_spice {
2737     my ($vga) = @_;
2738
2739     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2740
2741     return $1 || 1;
2742 }
2743
2744 sub config_to_command {
2745     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2746
2747     my $cmd = [];
2748     my $globalFlags = [];
2749     my $machineFlags = [];
2750     my $rtcFlags = [];
2751     my $cpuFlags = [];
2752     my $devices = [];
2753     my $pciaddr = '';
2754     my $bridges = {};
2755     my $kvmver = kvm_user_version();
2756     my $vernum = 0; # unknown
2757     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2758         $vernum = $1*1000000+$2*1000;
2759     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2760         $vernum = $1*1000000+$2*1000+$3;
2761     }
2762
2763     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2764
2765     my $have_ovz = -f '/proc/vz/vestat';
2766
2767     my $q35 = machine_type_is_q35($conf);
2768     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2769     my $machine_type = $forcemachine || $conf->{machine};
2770     my $use_old_bios_files = undef;
2771     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2772
2773     my $cpuunits = defined($conf->{cpuunits}) ?
2774             $conf->{cpuunits} : $defaults->{cpuunits};
2775
2776     push @$cmd, '/usr/bin/systemd-run';
2777     push @$cmd, '--scope';
2778     push @$cmd, '--slice', "qemu";
2779     push @$cmd, '--unit', $vmid;
2780     # set KillMode=none, so that systemd don't kill those scopes
2781     # at shutdown (pve-manager service should stop the VMs instead)
2782     push @$cmd, '-p', "KillMode=none";
2783     push @$cmd, '-p', "CPUShares=$cpuunits";
2784     if ($conf->{cpulimit}) {
2785         my $cpulimit = int($conf->{cpulimit} * 100);
2786         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2787     }
2788
2789     push @$cmd, '/usr/bin/kvm';
2790
2791     push @$cmd, '-id', $vmid;
2792
2793     my $use_virtio = 0;
2794
2795     my $qmpsocket = qmp_socket($vmid);
2796     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2797     push @$cmd, '-mon', "chardev=qmp,mode=control";
2798
2799     my $socket = vnc_socket($vmid);
2800     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2801
2802     push @$cmd, '-pidfile' , pidfile_name($vmid);
2803
2804     push @$cmd, '-daemonize';
2805
2806     if ($conf->{smbios1}) {
2807         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2808     }
2809
2810     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2811         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2812         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2813         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2814         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2815         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2816         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2817     }
2818
2819     if ($q35) {
2820         # the q35 chipset support native usb2, so we enable usb controller
2821         # by default for this machine type
2822         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2823     } else {
2824         $pciaddr = print_pci_addr("piix3", $bridges);
2825         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2826
2827         my $use_usb2 = 0;
2828         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2829             next if !$conf->{"usb$i"};
2830             $use_usb2 = 1;
2831         }
2832         # include usb device config
2833         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2834     }
2835
2836     my $vga = $conf->{vga};
2837
2838     my $qxlnum = vga_conf_has_spice($vga);
2839     $vga = 'qxl' if $qxlnum;
2840
2841     if (!$vga) {
2842         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2843                                 $conf->{ostype} eq 'win7' ||
2844                                 $conf->{ostype} eq 'w2k8')) {
2845             $vga = 'std';
2846         } else {
2847             $vga = 'cirrus';
2848         }
2849     }
2850
2851     # enable absolute mouse coordinates (needed by vnc)
2852     my $tablet;
2853     if (defined($conf->{tablet})) {
2854         $tablet = $conf->{tablet};
2855     } else {
2856         $tablet = $defaults->{tablet};
2857         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2858         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2859     }
2860
2861     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2862
2863     my $nohyperv;
2864     # host pci devices
2865     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2866         my $d = parse_hostpci($conf->{"hostpci$i"});
2867         next if !$d;
2868
2869         my $pcie = $d->{pcie};
2870         if($pcie){
2871             die "q35 machine model is not enabled" if !$q35;
2872             $pciaddr = print_pcie_addr("hostpci$i");
2873         }else{
2874             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2875         }
2876
2877         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2878         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2879         if ($xvga && $xvga ne '') {
2880             push @$cpuFlags, 'kvm=off';
2881             $vga = 'none';
2882             $nohyperv = 1;
2883             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2884                 $xvga = "";
2885             }
2886         }
2887         my $pcidevices = $d->{pciid};
2888         my $multifunction = 1 if @$pcidevices > 1;
2889
2890         my $j=0;
2891         foreach my $pcidevice (@$pcidevices) {
2892
2893             my $id = "hostpci$i";
2894             $id .= ".$j" if $multifunction;
2895             my $addr = $pciaddr;
2896             $addr .= ".$j" if $multifunction;
2897             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2898
2899             if($j == 0){
2900                 $devicestr .= "$rombar$xvga";
2901                 $devicestr .= ",multifunction=on" if $multifunction;
2902             }
2903
2904             push @$devices, '-device', $devicestr;
2905             $j++;
2906         }
2907     }
2908
2909     # usb devices
2910     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2911         my $d = parse_usb_device($conf->{"usb$i"});
2912         next if !$d;
2913         if ($d->{vendorid} && $d->{productid}) {
2914             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2915         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2916             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2917         } elsif ($d->{spice}) {
2918             # usb redir support for spice
2919             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2920             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2921         }
2922     }
2923
2924     # serial devices
2925     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2926         if (my $path = $conf->{"serial$i"}) {
2927             if ($path eq 'socket') {
2928                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2929                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2930                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2931             } else {
2932                 die "no such serial device\n" if ! -c $path;
2933                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2934                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2935             }
2936         }
2937     }
2938
2939     # parallel devices
2940     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2941         if (my $path = $conf->{"parallel$i"}) {
2942             die "no such parallel device\n" if ! -c $path;
2943             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2944             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2945             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2946         }
2947     }
2948
2949     my $vmname = $conf->{name} || "vm$vmid";
2950
2951     push @$cmd, '-name', $vmname;
2952
2953     my $sockets = 1;
2954     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2955     $sockets = $conf->{sockets} if  $conf->{sockets};
2956
2957     my $cores = $conf->{cores} || 1;
2958
2959     my $maxcpus = $sockets * $cores;
2960
2961     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2962
2963     my $allowed_vcpus = $cpuinfo->{cpus};
2964
2965     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2966         if ($allowed_vcpus < $maxcpus);
2967
2968     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2969
2970     push @$cmd, '-nodefaults';
2971
2972     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2973
2974     my $bootindex_hash = {};
2975     my $i = 1;
2976     foreach my $o (split(//, $bootorder)) {
2977         $bootindex_hash->{$o} = $i*100;
2978         $i++;
2979     }
2980
2981     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2982
2983     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2984
2985     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2986
2987     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2988
2989     # time drift fix
2990     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2991
2992     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2993     my $useLocaltime = $conf->{localtime};
2994
2995     if (my $ost = $conf->{ostype}) {
2996         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2997
2998         if ($ost =~ m/^w/) { # windows
2999             $useLocaltime = 1 if !defined($conf->{localtime});
3000
3001             # use time drift fix when acpi is enabled
3002             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3003                 $tdf = 1 if !defined($conf->{tdf});
3004             }
3005         }
3006
3007         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
3008             $ost eq 'wvista') {
3009             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3010             push @$cmd, '-no-hpet';
3011             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3012                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
3013                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
3014                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
3015
3016             } else {
3017                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
3018             }
3019         }
3020
3021         if ($ost eq 'win7' || $ost eq 'win8') {
3022             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
3023         }
3024     }
3025
3026     push @$rtcFlags, 'driftfix=slew' if $tdf;
3027
3028     if ($nokvm) {
3029         push @$machineFlags, 'accel=tcg';
3030     } else {
3031         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3032     }
3033
3034     if ($machine_type) {
3035         push @$machineFlags, "type=${machine_type}";
3036     }
3037
3038     if ($conf->{startdate}) {
3039         push @$rtcFlags, "base=$conf->{startdate}";
3040     } elsif ($useLocaltime) {
3041         push @$rtcFlags, 'base=localtime';
3042     }
3043
3044     my $cpu = $nokvm ? "qemu64" : "kvm64";
3045     $cpu = $conf->{cpu} if $conf->{cpu};
3046
3047     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3048
3049     push @$cpuFlags , '-x2apic'
3050         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3051
3052     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3053
3054     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3055
3056     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3057
3058         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3059         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3060     }
3061
3062     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3063
3064     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3065
3066     push @$cmd, '-cpu', $cpu;
3067
3068     my $memory = $conf->{memory} || $defaults->{memory};
3069     my $static_memory = 0;
3070     my $dimm_memory = 0;
3071
3072     if ($hotplug_features->{memory}) {
3073         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3074         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3075         $static_memory = $STATICMEM;
3076         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3077         $dimm_memory = $memory - $static_memory;
3078         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3079
3080     } else {
3081
3082         $static_memory = $memory;
3083         push @$cmd, '-m', $static_memory;
3084     }
3085
3086     if ($conf->{numa}) {
3087
3088         my $numa_totalmemory = undef;
3089         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3090             next if !$conf->{"numa$i"};
3091             my $numa = parse_numa($conf->{"numa$i"});
3092             next if !$numa;
3093             # memory
3094             die "missing numa node$i memory value\n" if !$numa->{memory};
3095             my $numa_memory = $numa->{memory};
3096             $numa_totalmemory += $numa_memory;
3097             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3098
3099             # cpus
3100             my $cpus_start = $numa->{cpus}->{start};
3101             die "missing numa node$i cpus\n" if !defined($cpus_start);
3102             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3103             my $cpus = $cpus_start;
3104             if (defined($cpus_end)) {
3105                 $cpus .= "-$cpus_end";
3106                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3107             }
3108
3109             # hostnodes
3110             my $hostnodes_start = $numa->{hostnodes}->{start};
3111             if (defined($hostnodes_start)) {
3112                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3113                 my $hostnodes = $hostnodes_start;
3114                 if (defined($hostnodes_end)) {
3115                     $hostnodes .= "-$hostnodes_end";
3116                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3117                 }
3118
3119                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3120                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3121                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3122                 }
3123
3124                 # policy
3125                 my $policy = $numa->{policy};
3126                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3127                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3128             }
3129
3130             push @$cmd, '-object', $numa_object;
3131             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3132         }
3133
3134         die "total memory for NUMA nodes must be equal to vm static memory\n"
3135             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3136
3137         #if no custom tology, we split memory and cores across numa nodes
3138         if(!$numa_totalmemory) {
3139
3140             my $numa_memory = ($static_memory / $sockets) . "M";
3141
3142             for (my $i = 0; $i < $sockets; $i++)  {
3143
3144                 my $cpustart = ($cores * $i);
3145                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3146                 my $cpus = $cpustart;
3147                 $cpus .= "-$cpuend" if $cpuend;
3148
3149                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3150                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3151             }
3152         }
3153     }
3154
3155     if ($hotplug_features->{memory}) {
3156         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3157             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3158             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3159             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3160
3161             #if dimm_memory is not aligned to dimm map
3162             if($current_size > $memory) {
3163                  $conf->{memory} = $current_size;
3164                  update_config_nolock($vmid, $conf, 1);
3165             }
3166         });
3167     }
3168
3169     push @$cmd, '-S' if $conf->{freeze};
3170
3171     # set keyboard layout
3172     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3173     push @$cmd, '-k', $kb if $kb;
3174
3175     # enable sound
3176     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3177     #push @$cmd, '-soundhw', 'es1370';
3178     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3179
3180     if($conf->{agent}) {
3181         my $qgasocket = qmp_socket($vmid, 1);
3182         my $pciaddr = print_pci_addr("qga0", $bridges);
3183         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3184         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3185         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3186     }
3187
3188     my $spice_port;
3189
3190     if ($qxlnum) {
3191         if ($qxlnum > 1) {
3192             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3193                 for(my $i = 1; $i < $qxlnum; $i++){
3194                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3195                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3196                 }
3197             } else {
3198                 # assume other OS works like Linux
3199                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3200                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3201             }
3202         }
3203
3204         my $pciaddr = print_pci_addr("spice", $bridges);
3205
3206         my $nodename = PVE::INotify::nodename();
3207         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3208         $spice_port = PVE::Tools::next_spice_port($pfamily);
3209
3210         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3211
3212         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3213         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3214         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3215     }
3216
3217     # enable balloon by default, unless explicitly disabled
3218     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3219         $pciaddr = print_pci_addr("balloon0", $bridges);
3220         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3221     }
3222
3223     if ($conf->{watchdog}) {
3224         my $wdopts = parse_watchdog($conf->{watchdog});
3225         $pciaddr = print_pci_addr("watchdog", $bridges);
3226         my $watchdog = $wdopts->{model} || 'i6300esb';
3227         push @$devices, '-device', "$watchdog$pciaddr";
3228         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3229     }
3230
3231     my $vollist = [];
3232     my $scsicontroller = {};
3233     my $ahcicontroller = {};
3234     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3235
3236     # Add iscsi initiator name if available
3237     if (my $initiator = get_initiator_name()) {
3238         push @$devices, '-iscsi', "initiator-name=$initiator";
3239     }
3240
3241     foreach_drive($conf, sub {
3242         my ($ds, $drive) = @_;
3243
3244         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3245             push @$vollist, $drive->{file};
3246         }
3247
3248         $use_virtio = 1 if $ds =~ m/^virtio/;
3249
3250         if (drive_is_cdrom ($drive)) {
3251             if ($bootindex_hash->{d}) {
3252                 $drive->{bootindex} = $bootindex_hash->{d};
3253                 $bootindex_hash->{d} += 1;
3254             }
3255         } else {
3256             if ($bootindex_hash->{c}) {
3257                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3258                 $bootindex_hash->{c} += 1;
3259             }
3260         }
3261
3262         if($drive->{interface} eq 'virtio'){
3263            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3264         }
3265
3266         if ($drive->{interface} eq 'scsi') {
3267
3268             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3269
3270             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3271             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3272
3273             my $iothread = '';
3274             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3275                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3276                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3277             }
3278
3279             my $queues = '';
3280             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3281                 $queues = ",num_queues=$drive->{queues}";
3282             }
3283
3284             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3285             $scsicontroller->{$controller}=1;
3286         }
3287
3288         if ($drive->{interface} eq 'sata') {
3289            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3290            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3291            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3292            $ahcicontroller->{$controller}=1;
3293         }
3294
3295         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3296         push @$devices, '-drive',$drive_cmd;
3297         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3298     });
3299
3300     for (my $i = 0; $i < $MAX_NETS; $i++) {
3301          next if !$conf->{"net$i"};
3302          my $d = parse_net($conf->{"net$i"});
3303          next if !$d;
3304
3305          $use_virtio = 1 if $d->{model} eq 'virtio';
3306
3307          if ($bootindex_hash->{n}) {
3308             $d->{bootindex} = $bootindex_hash->{n};
3309             $bootindex_hash->{n} += 1;
3310          }
3311
3312          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3313          push @$devices, '-netdev', $netdevfull;
3314
3315          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3316          push @$devices, '-device', $netdevicefull;
3317     }
3318
3319     if (!$q35) {
3320         # add pci bridges
3321         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3322            $bridges->{1} = 1;
3323            $bridges->{2} = 1;
3324         }
3325
3326         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3327
3328         while (my ($k, $v) = each %$bridges) {
3329             $pciaddr = print_pci_addr("pci.$k");
3330             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3331         }
3332     }
3333
3334     # add custom args
3335     if ($conf->{args}) {
3336         my $aa = PVE::Tools::split_args($conf->{args});
3337         push @$cmd, @$aa;
3338     }
3339
3340     push @$cmd, @$devices;
3341     push @$cmd, '-rtc', join(',', @$rtcFlags)
3342         if scalar(@$rtcFlags);
3343     push @$cmd, '-machine', join(',', @$machineFlags)
3344         if scalar(@$machineFlags);
3345     push @$cmd, '-global', join(',', @$globalFlags)
3346         if scalar(@$globalFlags);
3347
3348     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3349 }
3350
3351 sub vnc_socket {
3352     my ($vmid) = @_;
3353     return "${var_run_tmpdir}/$vmid.vnc";
3354 }
3355
3356 sub spice_port {
3357     my ($vmid) = @_;
3358
3359     my $res = vm_mon_cmd($vmid, 'query-spice');
3360
3361     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3362 }
3363
3364 sub qmp_socket {
3365     my ($vmid, $qga) = @_;
3366     my $sockettype = $qga ? 'qga' : 'qmp';
3367     return "${var_run_tmpdir}/$vmid.$sockettype";
3368 }
3369
3370 sub pidfile_name {
3371     my ($vmid) = @_;
3372     return "${var_run_tmpdir}/$vmid.pid";
3373 }
3374
3375 sub vm_devices_list {
3376     my ($vmid) = @_;
3377
3378     my $res = vm_mon_cmd($vmid, 'query-pci');
3379     my $devices = {};
3380     foreach my $pcibus (@$res) {
3381         foreach my $device (@{$pcibus->{devices}}) {
3382             next if !$device->{'qdev_id'};
3383             if ($device->{'pci_bridge'}) {
3384                 $devices->{$device->{'qdev_id'}} = 1;
3385                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3386                     next if !$bridge_device->{'qdev_id'};
3387                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3388                     $devices->{$device->{'qdev_id'}}++;
3389                 }
3390             } else {
3391                 $devices->{$device->{'qdev_id'}} = 1;
3392             }
3393         }
3394     }
3395
3396     my $resblock = vm_mon_cmd($vmid, 'query-block');
3397     foreach my $block (@$resblock) {
3398         if($block->{device} =~ m/^drive-(\S+)/){
3399                 $devices->{$1} = 1;
3400         }
3401     }
3402
3403     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3404     foreach my $mice (@$resmice) {
3405         if ($mice->{name} eq 'QEMU HID Tablet') {
3406             $devices->{tablet} = 1;
3407             last;
3408         }
3409     }
3410
3411     return $devices;
3412 }
3413
3414 sub vm_deviceplug {
3415     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3416
3417     my $q35 = machine_type_is_q35($conf);
3418
3419     my $devices_list = vm_devices_list($vmid);
3420     return 1 if defined($devices_list->{$deviceid});
3421
3422     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3423
3424     if ($deviceid eq 'tablet') {
3425
3426         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3427
3428     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3429
3430         qemu_iothread_add($vmid, $deviceid, $device);
3431
3432         qemu_driveadd($storecfg, $vmid, $device);
3433         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3434
3435         qemu_deviceadd($vmid, $devicefull);
3436         eval { qemu_deviceaddverify($vmid, $deviceid); };
3437         if (my $err = $@) {
3438             eval { qemu_drivedel($vmid, $deviceid); };
3439             warn $@ if $@;
3440             die $err;
3441         }
3442
3443     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3444
3445
3446         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3447         my $pciaddr = print_pci_addr($deviceid);
3448         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3449
3450         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3451
3452         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3453             qemu_iothread_add($vmid, $deviceid, $device);
3454             $devicefull .= ",iothread=iothread-$deviceid";
3455         }
3456
3457         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3458             $devicefull .= ",num_queues=$device->{queues}";
3459         }
3460
3461         qemu_deviceadd($vmid, $devicefull);
3462         qemu_deviceaddverify($vmid, $deviceid);
3463
3464     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3465
3466         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3467         qemu_driveadd($storecfg, $vmid, $device);
3468
3469         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3470         eval { qemu_deviceadd($vmid, $devicefull); };
3471         if (my $err = $@) {
3472             eval { qemu_drivedel($vmid, $deviceid); };
3473             warn $@ if $@;
3474             die $err;
3475         }
3476
3477     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3478
3479         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3480
3481         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3482         my $use_old_bios_files = undef;
3483         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3484
3485         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3486         qemu_deviceadd($vmid, $netdevicefull);
3487         eval { qemu_deviceaddverify($vmid, $deviceid); };
3488         if (my $err = $@) {
3489             eval { qemu_netdevdel($vmid, $deviceid); };
3490             warn $@ if $@;
3491             die $err;
3492         }
3493
3494     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3495
3496         my $bridgeid = $2;
3497         my $pciaddr = print_pci_addr($deviceid);
3498         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3499
3500         qemu_deviceadd($vmid, $devicefull);
3501         qemu_deviceaddverify($vmid, $deviceid);
3502
3503     } else {
3504         die "can't hotplug device '$deviceid'\n";
3505     }
3506
3507     return 1;
3508 }
3509
3510 # fixme: this should raise exceptions on error!
3511 sub vm_deviceunplug {
3512     my ($vmid, $conf, $deviceid) = @_;
3513
3514     my $devices_list = vm_devices_list($vmid);
3515     return 1 if !defined($devices_list->{$deviceid});
3516
3517     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3518
3519     if ($deviceid eq 'tablet') {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522
3523     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3524
3525         qemu_devicedel($vmid, $deviceid);
3526         qemu_devicedelverify($vmid, $deviceid);
3527         qemu_drivedel($vmid, $deviceid);
3528         qemu_iothread_del($conf, $vmid, $deviceid);
3529
3530     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3531
3532         qemu_devicedel($vmid, $deviceid);
3533         qemu_devicedelverify($vmid, $deviceid);
3534         qemu_iothread_del($conf, $vmid, $deviceid);
3535
3536     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3537
3538         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3539         my $device = parse_drive($deviceid, $conf->{$deviceid});
3540         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3541
3542         qemu_devicedel($vmid, $deviceid);
3543         qemu_drivedel($vmid, $deviceid);
3544         qemu_deletescsihw($conf, $vmid, $deviceid);
3545
3546     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3547
3548         qemu_devicedel($vmid, $deviceid);
3549         qemu_devicedelverify($vmid, $deviceid);
3550         qemu_netdevdel($vmid, $deviceid);
3551
3552     } else {
3553         die "can't unplug device '$deviceid'\n";
3554     }
3555
3556     return 1;
3557 }
3558
3559 sub qemu_deviceadd {
3560     my ($vmid, $devicefull) = @_;
3561
3562     $devicefull = "driver=".$devicefull;
3563     my %options =  split(/[=,]/, $devicefull);
3564
3565     vm_mon_cmd($vmid, "device_add" , %options);
3566 }
3567
3568 sub qemu_devicedel {
3569     my ($vmid, $deviceid) = @_;
3570
3571     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3572 }
3573
3574 sub qemu_iothread_add {
3575     my($vmid, $deviceid, $device) = @_;
3576
3577     if ($device->{iothread}) {
3578         my $iothreads = vm_iothreads_list($vmid);
3579         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3580     }
3581 }
3582
3583 sub qemu_iothread_del {
3584     my($conf, $vmid, $deviceid) = @_;
3585
3586     my $device = parse_drive($deviceid, $conf->{$deviceid});
3587     if ($device->{iothread}) {
3588         my $iothreads = vm_iothreads_list($vmid);
3589         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3590     }
3591 }
3592
3593 sub qemu_objectadd {
3594     my($vmid, $objectid, $qomtype) = @_;
3595
3596     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3597
3598     return 1;
3599 }
3600
3601 sub qemu_objectdel {
3602     my($vmid, $objectid) = @_;
3603
3604     vm_mon_cmd($vmid, "object-del", id => $objectid);
3605
3606     return 1;
3607 }
3608
3609 sub qemu_driveadd {
3610     my ($storecfg, $vmid, $device) = @_;
3611
3612     my $drive = print_drive_full($storecfg, $vmid, $device);
3613     $drive =~ s/\\/\\\\/g;
3614     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3615
3616     # If the command succeeds qemu prints: "OK"
3617     return 1 if $ret =~ m/OK/s;
3618
3619     die "adding drive failed: $ret\n";
3620 }
3621
3622 sub qemu_drivedel {
3623     my($vmid, $deviceid) = @_;
3624
3625     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3626     $ret =~ s/^\s+//;
3627
3628     return 1 if $ret eq "";
3629
3630     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3631     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3632
3633     die "deleting drive $deviceid failed : $ret\n";
3634 }
3635
3636 sub qemu_deviceaddverify {
3637     my ($vmid, $deviceid) = @_;
3638
3639     for (my $i = 0; $i <= 5; $i++) {
3640          my $devices_list = vm_devices_list($vmid);
3641          return 1 if defined($devices_list->{$deviceid});
3642          sleep 1;
3643     }
3644
3645     die "error on hotplug device '$deviceid'\n";
3646 }
3647
3648
3649 sub qemu_devicedelverify {
3650     my ($vmid, $deviceid) = @_;
3651
3652     # need to verify that the device is correctly removed as device_del
3653     # is async and empty return is not reliable
3654
3655     for (my $i = 0; $i <= 5; $i++) {
3656          my $devices_list = vm_devices_list($vmid);
3657          return 1 if !defined($devices_list->{$deviceid});
3658          sleep 1;
3659     }
3660
3661     die "error on hot-unplugging device '$deviceid'\n";
3662 }
3663
3664 sub qemu_findorcreatescsihw {
3665     my ($storecfg, $conf, $vmid, $device) = @_;
3666
3667     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3668
3669     my $scsihwid="$controller_prefix$controller";
3670     my $devices_list = vm_devices_list($vmid);
3671
3672     if(!defined($devices_list->{$scsihwid})) {
3673         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3674     }
3675
3676     return 1;
3677 }
3678
3679 sub qemu_deletescsihw {
3680     my ($conf, $vmid, $opt) = @_;
3681
3682     my $device = parse_drive($opt, $conf->{$opt});
3683
3684     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3685         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3686         return 1;
3687     }
3688
3689     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3690
3691     my $devices_list = vm_devices_list($vmid);
3692     foreach my $opt (keys %{$devices_list}) {
3693         if (PVE::QemuServer::valid_drivename($opt)) {
3694             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3695             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3696                 return 1;
3697             }
3698         }
3699     }
3700
3701     my $scsihwid="scsihw$controller";
3702
3703     vm_deviceunplug($vmid, $conf, $scsihwid);
3704
3705     return 1;
3706 }
3707
3708 sub qemu_add_pci_bridge {
3709     my ($storecfg, $conf, $vmid, $device) = @_;
3710
3711     my $bridges = {};
3712
3713     my $bridgeid;
3714
3715     print_pci_addr($device, $bridges);
3716
3717     while (my ($k, $v) = each %$bridges) {
3718         $bridgeid = $k;
3719     }
3720     return 1 if !defined($bridgeid) || $bridgeid < 1;
3721
3722     my $bridge = "pci.$bridgeid";
3723     my $devices_list = vm_devices_list($vmid);
3724
3725     if (!defined($devices_list->{$bridge})) {
3726         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3727     }
3728
3729     return 1;
3730 }
3731
3732 sub qemu_set_link_status {
3733     my ($vmid, $device, $up) = @_;
3734
3735     vm_mon_cmd($vmid, "set_link", name => $device,
3736                up => $up ? JSON::true : JSON::false);
3737 }
3738
3739 sub qemu_netdevadd {
3740     my ($vmid, $conf, $device, $deviceid) = @_;
3741
3742     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3743     my %options =  split(/[=,]/, $netdev);
3744
3745     vm_mon_cmd($vmid, "netdev_add",  %options);
3746     return 1;
3747 }
3748
3749 sub qemu_netdevdel {
3750     my ($vmid, $deviceid) = @_;
3751
3752     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3753 }
3754
3755 sub qemu_cpu_hotplug {
3756     my ($vmid, $conf, $vcpus) = @_;
3757
3758     my $sockets = 1;
3759     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3760     $sockets = $conf->{sockets} if  $conf->{sockets};
3761     my $cores = $conf->{cores} || 1;
3762     my $maxcpus = $sockets * $cores;
3763
3764     $vcpus = $maxcpus if !$vcpus;
3765
3766     die "you can't add more vcpus than maxcpus\n"
3767         if $vcpus > $maxcpus;
3768
3769     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3770     die "online cpu unplug is not yet possible\n"
3771         if $vcpus < $currentvcpus;
3772
3773     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3774     die "vcpus in running vm is different than configuration\n"
3775         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3776
3777     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3778         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3779     }
3780 }
3781
3782 sub qemu_memory_hotplug {
3783     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3784
3785     return $value if !check_running($vmid);
3786
3787     my $memory = $conf->{memory} || $defaults->{memory};
3788     $value = $defaults->{memory} if !$value;
3789     return $value if $value == $memory;
3790
3791     my $static_memory = $STATICMEM;
3792     my $dimm_memory = $memory - $static_memory;
3793
3794     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3795     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3796
3797
3798     my $sockets = 1;
3799     $sockets = $conf->{sockets} if $conf->{sockets};
3800
3801     if($value > $memory) {
3802
3803         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3804             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3805
3806                 return if $current_size <= $conf->{memory};
3807
3808                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3809                 if (my $err = $@) {
3810                     eval { qemu_objectdel($vmid, "mem-$name"); };
3811                     die $err;
3812                 }
3813
3814                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3815                 if (my $err = $@) {
3816                     eval { qemu_objectdel($vmid, "mem-$name"); };
3817                     die $err;
3818                 }
3819                 #update conf after each succesful module hotplug
3820                 $conf->{memory} = $current_size;
3821                 update_config_nolock($vmid, $conf, 1);
3822         });
3823
3824     } else {
3825
3826         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3827             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3828
3829                 return if $current_size >= $conf->{memory};
3830                 print "try to unplug memory dimm $name\n";
3831
3832                 my $retry = 0;
3833                 while (1) {
3834                     eval { qemu_devicedel($vmid, $name) };
3835                     sleep 3;
3836                     my $dimm_list = qemu_dimm_list($vmid);
3837                     last if !$dimm_list->{$name};
3838                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3839                     $retry++;
3840                 }
3841
3842                 #update conf after each succesful module unplug
3843                 $conf->{memory} = $current_size;
3844
3845                 eval { qemu_objectdel($vmid, "mem-$name"); };
3846                 update_config_nolock($vmid, $conf, 1);
3847         });
3848     }
3849 }
3850
3851 sub qemu_dimm_list {
3852     my ($vmid) = @_;
3853
3854     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3855     my $dimms = {};
3856
3857     foreach my $dimm (@$dimmarray) {
3858
3859         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3860         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3861         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3862         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3863         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3864     }
3865     return $dimms;
3866 }
3867
3868 sub qemu_block_set_io_throttle {
3869     my ($vmid, $deviceid,
3870         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3871         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3872
3873     return if !check_running($vmid) ;
3874
3875     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3876         bps => int($bps),
3877         bps_rd => int($bps_rd),
3878         bps_wr => int($bps_wr),
3879         iops => int($iops),
3880         iops_rd => int($iops_rd),
3881         iops_wr => int($iops_wr),
3882         bps_max => int($bps_max),
3883         bps_rd_max => int($bps_rd_max),
3884         bps_wr_max => int($bps_wr_max),
3885         iops_max => int($iops_max),
3886         iops_rd_max => int($iops_rd_max),
3887         iops_wr_max => int($iops_wr_max)
3888     );
3889
3890 }
3891
3892 # old code, only used to shutdown old VM after update
3893 sub __read_avail {
3894     my ($fh, $timeout) = @_;
3895
3896     my $sel = new IO::Select;
3897     $sel->add($fh);
3898
3899     my $res = '';
3900     my $buf;
3901
3902     my @ready;
3903     while (scalar (@ready = $sel->can_read($timeout))) {
3904         my $count;
3905         if ($count = $fh->sysread($buf, 8192)) {
3906             if ($buf =~ /^(.*)\(qemu\) $/s) {
3907                 $res .= $1;
3908                 last;
3909             } else {
3910                 $res .= $buf;
3911             }
3912         } else {
3913             if (!defined($count)) {
3914                 die "$!\n";
3915             }
3916             last;
3917         }
3918     }
3919
3920     die "monitor read timeout\n" if !scalar(@ready);
3921
3922     return $res;
3923 }
3924
3925 # old code, only used to shutdown old VM after update
3926 sub vm_monitor_command {
3927     my ($vmid, $cmdstr, $nocheck) = @_;
3928
3929     my $res;
3930
3931     eval {
3932         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3933
3934         my $sname = "${var_run_tmpdir}/$vmid.mon";
3935
3936         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3937             die "unable to connect to VM $vmid socket - $!\n";
3938
3939         my $timeout = 3;
3940
3941         # hack: migrate sometime blocks the monitor (when migrate_downtime
3942         # is set)
3943         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3944             $timeout = 60*60; # 1 hour
3945         }
3946
3947         # read banner;
3948         my $data = __read_avail($sock, $timeout);
3949
3950         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3951             die "got unexpected qemu monitor banner\n";
3952         }
3953
3954         my $sel = new IO::Select;
3955         $sel->add($sock);
3956
3957         if (!scalar(my @ready = $sel->can_write($timeout))) {
3958             die "monitor write error - timeout";
3959         }
3960
3961         my $fullcmd = "$cmdstr\r";
3962
3963         # syslog('info', "VM $vmid monitor command: $cmdstr");
3964
3965         my $b;
3966         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3967             die "monitor write error - $!";
3968         }
3969
3970         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3971
3972         $timeout = 20;
3973
3974         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3975             $timeout = 60*60; # 1 hour
3976         } elsif ($cmdstr =~ m/^(eject|change)/) {
3977             $timeout = 60; # note: cdrom mount command is slow
3978         }
3979         if ($res = __read_avail($sock, $timeout)) {
3980
3981             my @lines = split("\r?\n", $res);
3982
3983             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3984
3985             $res = join("\n", @lines);
3986             $res .= "\n";
3987         }
3988     };
3989
3990     my $err = $@;
3991
3992     if ($err) {
3993         syslog("err", "VM $vmid monitor command failed - $err");
3994         die $err;
3995     }
3996
3997     return $res;
3998 }
3999
4000 sub qemu_block_resize {
4001     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4002
4003     my $running = check_running($vmid);
4004
4005     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4006
4007     return if !$running;
4008
4009     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4010
4011 }
4012
4013 sub qemu_volume_snapshot {
4014     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4015
4016     my $running = check_running($vmid);
4017
4018     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4019         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4020     } else {
4021         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4022     }
4023 }
4024
4025 sub qemu_volume_snapshot_delete {
4026     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4027
4028     my $running = check_running($vmid);
4029
4030     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4031
4032     return if !$running;
4033
4034     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4035 }
4036
4037 sub set_migration_caps {
4038     my ($vmid) = @_;
4039
4040     my $cap_ref = [];
4041
4042     my $enabled_cap = {
4043         "auto-converge" => 1,
4044         "xbzrle" => 1,
4045         "x-rdma-pin-all" => 0,
4046         "zero-blocks" => 0,
4047         "compress" => 0
4048     };
4049
4050     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4051
4052     for my $supported_capability (@$supported_capabilities) {
4053         push @$cap_ref, {
4054             capability => $supported_capability->{capability},
4055             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4056         };
4057     }
4058
4059     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4060 }
4061
4062 my $fast_plug_option = {
4063     'lock' => 1,
4064     'name' => 1,
4065     'onboot' => 1,
4066     'shares' => 1,
4067     'startup' => 1,
4068     'description' => 1,
4069 };
4070
4071 # hotplug changes in [PENDING]
4072 # $selection hash can be used to only apply specified options, for
4073 # example: { cores => 1 } (only apply changed 'cores')
4074 # $errors ref is used to return error messages
4075 sub vmconfig_hotplug_pending {
4076     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4077
4078     my $defaults = load_defaults();
4079
4080     # commit values which do not have any impact on running VM first
4081     # Note: those option cannot raise errors, we we do not care about
4082     # $selection and always apply them.
4083
4084     my $add_error = sub {
4085         my ($opt, $msg) = @_;
4086         $errors->{$opt} = "hotplug problem - $msg";
4087     };
4088
4089     my $changes = 0;
4090     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4091         if ($fast_plug_option->{$opt}) {
4092             $conf->{$opt} = $conf->{pending}->{$opt};
4093             delete $conf->{pending}->{$opt};
4094             $changes = 1;
4095         }
4096     }
4097
4098     if ($changes) {
4099         update_config_nolock($vmid, $conf, 1);
4100         $conf = load_config($vmid); # update/reload
4101     }
4102
4103     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4104
4105     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4106     while (my ($opt, $force) = each %$pending_delete_hash) {
4107         next if $selection && !$selection->{$opt};
4108         eval {
4109             if ($opt eq 'hotplug') {
4110                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4111             } elsif ($opt eq 'tablet') {
4112                 die "skip\n" if !$hotplug_features->{usb};
4113                 if ($defaults->{tablet}) {
4114                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4115                 } else {
4116                     vm_deviceunplug($vmid, $conf, $opt);
4117                 }
4118             } elsif ($opt eq 'vcpus') {
4119                 die "skip\n" if !$hotplug_features->{cpu};
4120                 qemu_cpu_hotplug($vmid, $conf, undef);
4121             } elsif ($opt eq 'balloon') {
4122                 # enable balloon device is not hotpluggable
4123                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4124             } elsif ($fast_plug_option->{$opt}) {
4125                 # do nothing
4126             } elsif ($opt =~ m/^net(\d+)$/) {
4127                 die "skip\n" if !$hotplug_features->{network};
4128                 vm_deviceunplug($vmid, $conf, $opt);
4129             } elsif (valid_drivename($opt)) {
4130                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4131                 vm_deviceunplug($vmid, $conf, $opt);
4132                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4133             } elsif ($opt =~ m/^memory$/) {
4134                 die "skip\n" if !$hotplug_features->{memory};
4135                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4136             } elsif ($opt eq 'cpuunits') {
4137                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4138             } elsif ($opt eq 'cpulimit') {
4139                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4140             } else {
4141                 die "skip\n";
4142             }
4143         };
4144         if (my $err = $@) {
4145             &$add_error($opt, $err) if $err ne "skip\n";
4146         } else {
4147             # save new config if hotplug was successful
4148             delete $conf->{$opt};
4149             vmconfig_undelete_pending_option($conf, $opt);
4150             update_config_nolock($vmid, $conf, 1);
4151             $conf = load_config($vmid); # update/reload
4152         }
4153     }
4154
4155     foreach my $opt (keys %{$conf->{pending}}) {
4156         next if $selection && !$selection->{$opt};
4157         my $value = $conf->{pending}->{$opt};
4158         eval {
4159             if ($opt eq 'hotplug') {
4160                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4161             } elsif ($opt eq 'tablet') {
4162                 die "skip\n" if !$hotplug_features->{usb};
4163                 if ($value == 1) {
4164                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4165                 } elsif ($value == 0) {
4166                     vm_deviceunplug($vmid, $conf, $opt);
4167                 }
4168             } elsif ($opt eq 'vcpus') {
4169                 die "skip\n" if !$hotplug_features->{cpu};
4170                 qemu_cpu_hotplug($vmid, $conf, $value);
4171             } elsif ($opt eq 'balloon') {
4172                 # enable/disable balloning device is not hotpluggable
4173                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4174                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4175                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4176
4177                 # allow manual ballooning if shares is set to zero
4178                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4179                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4180                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4181                 }
4182             } elsif ($opt =~ m/^net(\d+)$/) {
4183                 # some changes can be done without hotplug
4184                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4185                                     $vmid, $opt, $value);
4186             } elsif (valid_drivename($opt)) {
4187                 # some changes can be done without hotplug
4188                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4189                                      $vmid, $opt, $value, 1);
4190             } elsif ($opt =~ m/^memory$/) { #dimms
4191                 die "skip\n" if !$hotplug_features->{memory};
4192                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4193             } elsif ($opt eq 'cpuunits') {
4194                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4195             } elsif ($opt eq 'cpulimit') {
4196                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4197                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4198             } else {
4199                 die "skip\n";  # skip non-hot-pluggable options
4200             }
4201         };
4202         if (my $err = $@) {
4203             &$add_error($opt, $err) if $err ne "skip\n";
4204         } else {
4205             # save new config if hotplug was successful
4206             $conf->{$opt} = $value;
4207             delete $conf->{pending}->{$opt};
4208             update_config_nolock($vmid, $conf, 1);
4209             $conf = load_config($vmid); # update/reload
4210         }
4211     }
4212 }
4213
4214 sub try_deallocate_drive {
4215     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4216
4217     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4218         my $volid = $drive->{file};
4219         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4220             my $sid = PVE::Storage::parse_volume_id($volid);
4221             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4222
4223             # check if the disk is really unused
4224             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4225                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4226             PVE::Storage::vdisk_free($storecfg, $volid);
4227             return 1;
4228         } else {
4229             # If vm is not owner of this disk remove from config
4230             return 1;
4231         }
4232     }
4233
4234     return undef;
4235 }
4236
4237 sub vmconfig_delete_or_detach_drive {
4238     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4239
4240     my $drive = parse_drive($opt, $conf->{$opt});
4241
4242     my $rpcenv = PVE::RPCEnvironment::get();
4243     my $authuser = $rpcenv->get_user();
4244
4245     if ($force) {
4246         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4247         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4248     } else {
4249         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4250     }
4251 }
4252
4253 sub vmconfig_apply_pending {
4254     my ($vmid, $conf, $storecfg) = @_;
4255
4256     # cold plug
4257
4258     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4259     while (my ($opt, $force) = each %$pending_delete_hash) {
4260         die "internal error" if $opt =~ m/^unused/;
4261         $conf = load_config($vmid); # update/reload
4262         if (!defined($conf->{$opt})) {
4263             vmconfig_undelete_pending_option($conf, $opt);
4264             update_config_nolock($vmid, $conf, 1);
4265         } elsif (valid_drivename($opt)) {
4266             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4267             vmconfig_undelete_pending_option($conf, $opt);
4268             delete $conf->{$opt};
4269             update_config_nolock($vmid, $conf, 1);
4270         } else {
4271             vmconfig_undelete_pending_option($conf, $opt);
4272             delete $conf->{$opt};
4273             update_config_nolock($vmid, $conf, 1);
4274         }
4275     }
4276
4277     $conf = load_config($vmid); # update/reload
4278
4279     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4280         $conf = load_config($vmid); # update/reload
4281
4282         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4283             # skip if nothing changed
4284         } elsif (valid_drivename($opt)) {
4285             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4286                 if defined($conf->{$opt});
4287             $conf->{$opt} = $conf->{pending}->{$opt};
4288         } else {
4289             $conf->{$opt} = $conf->{pending}->{$opt};
4290         }
4291
4292         delete $conf->{pending}->{$opt};
4293         update_config_nolock($vmid, $conf, 1);
4294     }
4295 }
4296
4297 my $safe_num_ne = sub {
4298     my ($a, $b) = @_;
4299
4300     return 0 if !defined($a) && !defined($b);
4301     return 1 if !defined($a);
4302     return 1 if !defined($b);
4303
4304     return $a != $b;
4305 };
4306
4307 my $safe_string_ne = sub {
4308     my ($a, $b) = @_;
4309
4310     return 0 if !defined($a) && !defined($b);
4311     return 1 if !defined($a);
4312     return 1 if !defined($b);
4313
4314     return $a ne $b;
4315 };
4316
4317 sub vmconfig_update_net {
4318     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4319
4320     my $newnet = parse_net($value);
4321
4322     if ($conf->{$opt}) {
4323         my $oldnet = parse_net($conf->{$opt});
4324
4325         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4326             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4327             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4328             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4329
4330             # for non online change, we try to hot-unplug
4331             die "skip\n" if !$hotplug;
4332             vm_deviceunplug($vmid, $conf, $opt);
4333         } else {
4334
4335             die "internal error" if $opt !~ m/net(\d+)/;
4336             my $iface = "tap${vmid}i$1";
4337
4338             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4339                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4340             }
4341
4342             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4343                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4344                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4345                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4346                 PVE::Network::tap_unplug($iface);
4347                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4348             }
4349
4350             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4351                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4352             }
4353
4354             return 1;
4355         }
4356     }
4357
4358     if ($hotplug) {
4359         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4360     } else {
4361         die "skip\n";
4362     }
4363 }
4364
4365 sub vmconfig_update_disk {
4366     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4367
4368     # fixme: do we need force?
4369
4370     my $drive = parse_drive($opt, $value);
4371
4372     if ($conf->{$opt}) {
4373
4374         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4375
4376             my $media = $drive->{media} || 'disk';
4377             my $oldmedia = $old_drive->{media} || 'disk';
4378             die "unable to change media type\n" if $media ne $oldmedia;
4379
4380             if (!drive_is_cdrom($old_drive)) {
4381
4382                 if ($drive->{file} ne $old_drive->{file}) {
4383
4384                     die "skip\n" if !$hotplug;
4385
4386                     # unplug and register as unused
4387                     vm_deviceunplug($vmid, $conf, $opt);
4388                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4389
4390                 } else {
4391                     # update existing disk
4392
4393                     # skip non hotpluggable value
4394                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4395                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4396                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4397                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4398                         die "skip\n";
4399                     }
4400
4401                     # apply throttle
4402                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4403                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4404                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4405                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4406                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4407                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4408                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4409                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4410                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4411                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4412                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4413                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4414
4415                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4416                                                    ($drive->{mbps} || 0)*1024*1024,
4417                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4418                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4419                                                    $drive->{iops} || 0,
4420                                                    $drive->{iops_rd} || 0,
4421                                                    $drive->{iops_wr} || 0,
4422                                                    ($drive->{mbps_max} || 0)*1024*1024,
4423                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4424                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4425                                                    $drive->{iops_max} || 0,
4426                                                    $drive->{iops_rd_max} || 0,
4427                                                    $drive->{iops_wr_max} || 0);
4428
4429                     }
4430
4431                     return 1;
4432                 }
4433
4434             } else { # cdrom
4435
4436                 if ($drive->{file} eq 'none') {
4437                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4438                 } else {
4439                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4440                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4441                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4442                 }
4443
4444                 return 1;
4445             }
4446         }
4447     }
4448
4449     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4450     # hotplug new disks
4451     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4452     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4453 }
4454
4455 sub vm_start {
4456     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4457         $forcemachine, $spice_ticket) = @_;
4458
4459     lock_config($vmid, sub {
4460         my $conf = load_config($vmid, $migratedfrom);
4461
4462         die "you can't start a vm if it's a template\n" if is_template($conf);
4463
4464         check_lock($conf) if !$skiplock;
4465
4466         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4467
4468         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4469             vmconfig_apply_pending($vmid, $conf, $storecfg);
4470             $conf = load_config($vmid); # update/reload
4471         }
4472
4473         my $defaults = load_defaults();
4474
4475         # set environment variable useful inside network script
4476         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4477
4478         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4479
4480         my $migrate_port = 0;
4481         my $migrate_uri;
4482         if ($statefile) {
4483             if ($statefile eq 'tcp') {
4484                 my $localip = "localhost";
4485                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4486                 my $nodename = PVE::INotify::nodename();
4487                 if ($datacenterconf->{migration_unsecure}) {
4488                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4489                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4490                 }
4491                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4492                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4493                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4494                 push @$cmd, '-incoming', $migrate_uri;
4495                 push @$cmd, '-S';
4496             } else {
4497                 push @$cmd, '-loadstate', $statefile;
4498             }
4499         } elsif ($paused) {
4500             push @$cmd, '-S';
4501         }
4502
4503         # host pci devices
4504         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4505           my $d = parse_hostpci($conf->{"hostpci$i"});
4506           next if !$d;
4507           my $pcidevices = $d->{pciid};
4508           foreach my $pcidevice (@$pcidevices) {
4509                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4510
4511                 my $info = pci_device_info("0000:$pciid");
4512                 die "IOMMU not present\n" if !check_iommu_support();
4513                 die "no pci device info for device '$pciid'\n" if !$info;
4514                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4515                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4516           }
4517         }
4518
4519         PVE::Storage::activate_volumes($storecfg, $vollist);
4520
4521         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4522                     umask => 0077); };
4523         my $err = $@;
4524         die "start failed: $err" if $err;
4525
4526         print "migration listens on $migrate_uri\n" if $migrate_uri;
4527
4528         if ($statefile && $statefile ne 'tcp')  {
4529             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4530             warn $@ if $@;
4531         }
4532
4533         if ($migratedfrom) {
4534
4535             eval {
4536                 set_migration_caps($vmid);
4537             };
4538             warn $@ if $@;
4539
4540             if ($spice_port) {
4541                 print "spice listens on port $spice_port\n";
4542                 if ($spice_ticket) {
4543                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4544                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4545                 }
4546             }
4547
4548         } else {
4549
4550             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4551                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4552                     if $conf->{balloon};
4553             }
4554
4555             foreach my $opt (keys %$conf) {
4556                 next if $opt !~  m/^net\d+$/;
4557                 my $nicconf = parse_net($conf->{$opt});
4558                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4559             }
4560         }
4561
4562         vm_mon_cmd_nocheck($vmid, 'qom-set',
4563                     path => "machine/peripheral/balloon0",
4564                     property => "guest-stats-polling-interval",
4565                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4566
4567     });
4568 }
4569
4570 sub vm_mon_cmd {
4571     my ($vmid, $execute, %params) = @_;
4572
4573     my $cmd = { execute => $execute, arguments => \%params };
4574     vm_qmp_command($vmid, $cmd);
4575 }
4576
4577 sub vm_mon_cmd_nocheck {
4578     my ($vmid, $execute, %params) = @_;
4579
4580     my $cmd = { execute => $execute, arguments => \%params };
4581     vm_qmp_command($vmid, $cmd, 1);
4582 }
4583
4584 sub vm_qmp_command {
4585     my ($vmid, $cmd, $nocheck) = @_;
4586
4587     my $res;
4588
4589     my $timeout;
4590     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4591         $timeout = $cmd->{arguments}->{timeout};
4592         delete $cmd->{arguments}->{timeout};
4593     }
4594
4595     eval {
4596         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4597         my $sname = qmp_socket($vmid);
4598         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4599             my $qmpclient = PVE::QMPClient->new();
4600
4601             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4602         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4603             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4604                 if scalar(%{$cmd->{arguments}});
4605             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4606         } else {
4607             die "unable to open monitor socket\n";
4608         }
4609     };
4610     if (my $err = $@) {
4611         syslog("err", "VM $vmid qmp command failed - $err");
4612         die $err;
4613     }
4614
4615     return $res;
4616 }
4617
4618 sub vm_human_monitor_command {
4619     my ($vmid, $cmdline) = @_;
4620
4621     my $res;
4622
4623     my $cmd = {
4624         execute => 'human-monitor-command',
4625         arguments => { 'command-line' => $cmdline},
4626     };
4627
4628     return vm_qmp_command($vmid, $cmd);
4629 }
4630
4631 sub vm_commandline {
4632     my ($storecfg, $vmid) = @_;
4633
4634     my $conf = load_config($vmid);
4635
4636     my $defaults = load_defaults();
4637
4638     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4639
4640     return join(' ', @$cmd);
4641 }
4642
4643 sub vm_reset {
4644     my ($vmid, $skiplock) = @_;
4645
4646     lock_config($vmid, sub {
4647
4648         my $conf = load_config($vmid);
4649
4650         check_lock($conf) if !$skiplock;
4651
4652         vm_mon_cmd($vmid, "system_reset");
4653     });
4654 }
4655
4656 sub get_vm_volumes {
4657     my ($conf) = @_;
4658
4659     my $vollist = [];
4660     foreach_volid($conf, sub {
4661         my ($volid, $is_cdrom) = @_;
4662
4663         return if $volid =~ m|^/|;
4664
4665         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4666         return if !$sid;
4667
4668         push @$vollist, $volid;
4669     });
4670
4671     return $vollist;
4672 }
4673
4674 sub vm_stop_cleanup {
4675     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4676
4677     eval {
4678
4679         if (!$keepActive) {
4680             my $vollist = get_vm_volumes($conf);
4681             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4682         }
4683
4684         foreach my $ext (qw(mon qmp pid vnc qga)) {
4685             unlink "/var/run/qemu-server/${vmid}.$ext";
4686         }
4687
4688         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4689     };
4690     warn $@ if $@; # avoid errors - just warn
4691 }
4692
4693 # Note: use $nockeck to skip tests if VM configuration file exists.
4694 # We need that when migration VMs to other nodes (files already moved)
4695 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4696 sub vm_stop {
4697     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4698
4699     $force = 1 if !defined($force) && !$shutdown;
4700
4701     if ($migratedfrom){
4702         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4703         kill 15, $pid if $pid;
4704         my $conf = load_config($vmid, $migratedfrom);
4705         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4706         return;
4707     }
4708
4709     lock_config($vmid, sub {
4710
4711         my $pid = check_running($vmid, $nocheck);
4712         return if !$pid;
4713
4714         my $conf;
4715         if (!$nocheck) {
4716             $conf = load_config($vmid);
4717             check_lock($conf) if !$skiplock;
4718             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4719                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4720                 $timeout = $opts->{down} if $opts->{down};
4721             }
4722         }
4723
4724         $timeout = 60 if !defined($timeout);
4725
4726         eval {
4727             if ($shutdown) {
4728                 if (defined($conf) && $conf->{agent}) {
4729                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4730                 } else {
4731                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4732                 }
4733             } else {
4734                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4735             }
4736         };
4737         my $err = $@;
4738
4739         if (!$err) {
4740             my $count = 0;
4741             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4742                 $count++;
4743                 sleep 1;
4744             }
4745
4746             if ($count >= $timeout) {
4747                 if ($force) {
4748                     warn "VM still running - terminating now with SIGTERM\n";
4749                     kill 15, $pid;
4750                 } else {
4751                     die "VM quit/powerdown failed - got timeout\n";
4752                 }
4753             } else {
4754                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4755                 return;
4756             }
4757         } else {
4758             if ($force) {
4759                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4760                 kill 15, $pid;
4761             } else {
4762                 die "VM quit/powerdown failed\n";
4763             }
4764         }
4765
4766         # wait again
4767         $timeout = 10;
4768
4769         my $count = 0;
4770         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4771             $count++;
4772             sleep 1;
4773         }
4774
4775         if ($count >= $timeout) {
4776             warn "VM still running - terminating now with SIGKILL\n";
4777             kill 9, $pid;
4778             sleep 1;
4779         }
4780
4781         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4782    });
4783 }
4784
4785 sub vm_suspend {
4786     my ($vmid, $skiplock) = @_;
4787
4788     lock_config($vmid, sub {
4789
4790         my $conf = load_config($vmid);
4791
4792         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4793
4794         vm_mon_cmd($vmid, "stop");
4795     });
4796 }
4797
4798 sub vm_resume {
4799     my ($vmid, $skiplock, $nocheck) = @_;
4800
4801     lock_config($vmid, sub {
4802
4803         if (!$nocheck) {
4804
4805             my $conf = load_config($vmid);
4806
4807             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4808
4809             vm_mon_cmd($vmid, "cont");
4810
4811         } else {
4812             vm_mon_cmd_nocheck($vmid, "cont");
4813         }
4814     });
4815 }
4816
4817 sub vm_sendkey {
4818     my ($vmid, $skiplock, $key) = @_;
4819
4820     lock_config($vmid, sub {
4821
4822         my $conf = load_config($vmid);
4823
4824         # there is no qmp command, so we use the human monitor command
4825         vm_human_monitor_command($vmid, "sendkey $key");
4826     });
4827 }
4828
4829 sub vm_destroy {
4830     my ($storecfg, $vmid, $skiplock) = @_;
4831
4832     lock_config($vmid, sub {
4833
4834         my $conf = load_config($vmid);
4835
4836         check_lock($conf) if !$skiplock;
4837
4838         if (!check_running($vmid)) {
4839             destroy_vm($storecfg, $vmid);
4840         } else {
4841             die "VM $vmid is running - destroy failed\n";
4842         }
4843     });
4844 }
4845
4846 # pci helpers
4847
4848 sub file_write {
4849     my ($filename, $buf) = @_;
4850
4851     my $fh = IO::File->new($filename, "w");
4852     return undef if !$fh;
4853
4854     my $res = print $fh $buf;
4855
4856     $fh->close();
4857
4858     return $res;
4859 }
4860
4861 sub pci_device_info {
4862     my ($name) = @_;
4863
4864     my $res;
4865
4866     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4867     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4868
4869     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4870     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4871
4872     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4873     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4874
4875     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4876     return undef if !defined($product) || $product !~ s/^0x//;
4877
4878     $res = {
4879         name => $name,
4880         vendor => $vendor,
4881         product => $product,
4882         domain => $domain,
4883         bus => $bus,
4884         slot => $slot,
4885         func => $func,
4886         irq => $irq,
4887         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4888     };
4889
4890     return $res;
4891 }
4892
4893 sub pci_dev_reset {
4894     my ($dev) = @_;
4895
4896     my $name = $dev->{name};
4897
4898     my $fn = "$pcisysfs/devices/$name/reset";
4899
4900     return file_write($fn, "1");
4901 }
4902
4903 sub pci_dev_bind_to_vfio {
4904     my ($dev) = @_;
4905
4906     my $name = $dev->{name};
4907
4908     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4909
4910     if (!-d $vfio_basedir) {
4911         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4912     }
4913     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4914
4915     my $testdir = "$vfio_basedir/$name";
4916     return 1 if -d $testdir;
4917
4918     my $data = "$dev->{vendor} $dev->{product}";
4919     return undef if !file_write("$vfio_basedir/new_id", $data);
4920
4921     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4922     if (!file_write($fn, $name)) {
4923         return undef if -f $fn;
4924     }
4925
4926     $fn = "$vfio_basedir/bind";
4927     if (! -d $testdir) {
4928         return undef if !file_write($fn, $name);
4929     }
4930
4931     return -d $testdir;
4932 }
4933
4934 sub pci_dev_group_bind_to_vfio {
4935     my ($pciid) = @_;
4936
4937     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4938
4939     if (!-d $vfio_basedir) {
4940         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4941     }
4942     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4943
4944     # get IOMMU group devices
4945     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4946       my @devs = grep /^0000:/, readdir($D);
4947     closedir($D);
4948
4949     foreach my $pciid (@devs) {
4950         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4951
4952         # pci bridges, switches or root ports are not supported
4953         # they have a pci_bus subdirectory so skip them
4954         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4955
4956         my $info = pci_device_info($1);
4957         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4958     }
4959
4960     return 1;
4961 }
4962
4963 sub print_pci_addr {
4964     my ($id, $bridges) = @_;
4965
4966     my $res = '';
4967     my $devices = {
4968         piix3 => { bus => 0, addr => 1 },
4969         #addr2 : first videocard
4970         balloon0 => { bus => 0, addr => 3 },
4971         watchdog => { bus => 0, addr => 4 },
4972         scsihw0 => { bus => 0, addr => 5 },
4973         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4974         scsihw1 => { bus => 0, addr => 6 },
4975         ahci0 => { bus => 0, addr => 7 },
4976         qga0 => { bus => 0, addr => 8 },
4977         spice => { bus => 0, addr => 9 },
4978         virtio0 => { bus => 0, addr => 10 },
4979         virtio1 => { bus => 0, addr => 11 },
4980         virtio2 => { bus => 0, addr => 12 },
4981         virtio3 => { bus => 0, addr => 13 },
4982         virtio4 => { bus => 0, addr => 14 },
4983         virtio5 => { bus => 0, addr => 15 },
4984         hostpci0 => { bus => 0, addr => 16 },
4985         hostpci1 => { bus => 0, addr => 17 },
4986         net0 => { bus => 0, addr => 18 },
4987         net1 => { bus => 0, addr => 19 },
4988         net2 => { bus => 0, addr => 20 },
4989         net3 => { bus => 0, addr => 21 },
4990         net4 => { bus => 0, addr => 22 },
4991         net5 => { bus => 0, addr => 23 },
4992         vga1 => { bus => 0, addr => 24 },
4993         vga2 => { bus => 0, addr => 25 },
4994         vga3 => { bus => 0, addr => 26 },
4995         hostpci2 => { bus => 0, addr => 27 },
4996         hostpci3 => { bus => 0, addr => 28 },
4997         #addr29 : usb-host (pve-usb.cfg)
4998         'pci.1' => { bus => 0, addr => 30 },
4999         'pci.2' => { bus => 0, addr => 31 },
5000         'net6' => { bus => 1, addr => 1 },
5001         'net7' => { bus => 1, addr => 2 },
5002         'net8' => { bus => 1, addr => 3 },
5003         'net9' => { bus => 1, addr => 4 },
5004         'net10' => { bus => 1, addr => 5 },
5005         'net11' => { bus => 1, addr => 6 },
5006         'net12' => { bus => 1, addr => 7 },
5007         'net13' => { bus => 1, addr => 8 },
5008         'net14' => { bus => 1, addr => 9 },
5009         'net15' => { bus => 1, addr => 10 },
5010         'net16' => { bus => 1, addr => 11 },
5011         'net17' => { bus => 1, addr => 12 },
5012         'net18' => { bus => 1, addr => 13 },
5013         'net19' => { bus => 1, addr => 14 },
5014         'net20' => { bus => 1, addr => 15 },
5015         'net21' => { bus => 1, addr => 16 },
5016         'net22' => { bus => 1, addr => 17 },
5017         'net23' => { bus => 1, addr => 18 },
5018         'net24' => { bus => 1, addr => 19 },
5019         'net25' => { bus => 1, addr => 20 },
5020         'net26' => { bus => 1, addr => 21 },
5021         'net27' => { bus => 1, addr => 22 },
5022         'net28' => { bus => 1, addr => 23 },
5023         'net29' => { bus => 1, addr => 24 },
5024         'net30' => { bus => 1, addr => 25 },
5025         'net31' => { bus => 1, addr => 26 },
5026         'virtio6' => { bus => 2, addr => 1 },
5027         'virtio7' => { bus => 2, addr => 2 },
5028         'virtio8' => { bus => 2, addr => 3 },
5029         'virtio9' => { bus => 2, addr => 4 },
5030         'virtio10' => { bus => 2, addr => 5 },
5031         'virtio11' => { bus => 2, addr => 6 },
5032         'virtio12' => { bus => 2, addr => 7 },
5033         'virtio13' => { bus => 2, addr => 8 },
5034         'virtio14' => { bus => 2, addr => 9 },
5035         'virtio15' => { bus => 2, addr => 10 },
5036         'virtioscsi0' => { bus => 3, addr => 1 },
5037         'virtioscsi1' => { bus => 3, addr => 2 },
5038         'virtioscsi2' => { bus => 3, addr => 3 },
5039         'virtioscsi3' => { bus => 3, addr => 4 },
5040         'virtioscsi4' => { bus => 3, addr => 5 },
5041         'virtioscsi5' => { bus => 3, addr => 6 },
5042         'virtioscsi6' => { bus => 3, addr => 7 },
5043         'virtioscsi7' => { bus => 3, addr => 8 },
5044         'virtioscsi8' => { bus => 3, addr => 9 },
5045         'virtioscsi9' => { bus => 3, addr => 10 },
5046         'virtioscsi10' => { bus => 3, addr => 11 },
5047         'virtioscsi11' => { bus => 3, addr => 12 },
5048         'virtioscsi12' => { bus => 3, addr => 13 },
5049         'virtioscsi13' => { bus => 3, addr => 14 },
5050         'virtioscsi14' => { bus => 3, addr => 15 },
5051         'virtioscsi15' => { bus => 3, addr => 16 },
5052         'virtioscsi16' => { bus => 3, addr => 17 },
5053         'virtioscsi17' => { bus => 3, addr => 18 },
5054         'virtioscsi18' => { bus => 3, addr => 19 },
5055         'virtioscsi19' => { bus => 3, addr => 20 },
5056         'virtioscsi20' => { bus => 3, addr => 21 },
5057         'virtioscsi21' => { bus => 3, addr => 22 },
5058         'virtioscsi22' => { bus => 3, addr => 23 },
5059         'virtioscsi23' => { bus => 3, addr => 24 },
5060         'virtioscsi24' => { bus => 3, addr => 25 },
5061         'virtioscsi25' => { bus => 3, addr => 26 },
5062         'virtioscsi26' => { bus => 3, addr => 27 },
5063         'virtioscsi27' => { bus => 3, addr => 28 },
5064         'virtioscsi28' => { bus => 3, addr => 29 },
5065         'virtioscsi29' => { bus => 3, addr => 30 },
5066         'virtioscsi30' => { bus => 3, addr => 31 },
5067
5068     };
5069
5070     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5071            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5072            my $bus = $devices->{$id}->{bus};
5073            $res = ",bus=pci.$bus,addr=$addr";
5074            $bridges->{$bus} = 1 if $bridges;
5075     }
5076     return $res;
5077
5078 }
5079
5080 sub print_pcie_addr {
5081     my ($id) = @_;
5082
5083     my $res = '';
5084     my $devices = {
5085         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5086         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5087         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5088         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5089     };
5090
5091     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5092            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5093            my $bus = $devices->{$id}->{bus};
5094            $res = ",bus=$bus,addr=$addr";
5095     }
5096     return $res;
5097
5098 }
5099
5100 # vzdump restore implementaion
5101
5102 sub tar_archive_read_firstfile {
5103     my $archive = shift;
5104
5105     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5106
5107     # try to detect archive type first
5108     my $pid = open (TMP, "tar tf '$archive'|") ||
5109         die "unable to open file '$archive'\n";
5110     my $firstfile = <TMP>;
5111     kill 15, $pid;
5112     close TMP;
5113
5114     die "ERROR: archive contaions no data\n" if !$firstfile;
5115     chomp $firstfile;
5116
5117     return $firstfile;
5118 }
5119
5120 sub tar_restore_cleanup {
5121     my ($storecfg, $statfile) = @_;
5122
5123     print STDERR "starting cleanup\n";
5124
5125     if (my $fd = IO::File->new($statfile, "r")) {
5126         while (defined(my $line = <$fd>)) {
5127             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5128                 my $volid = $2;
5129                 eval {
5130                     if ($volid =~ m|^/|) {
5131                         unlink $volid || die 'unlink failed\n';
5132                     } else {
5133                         PVE::Storage::vdisk_free($storecfg, $volid);
5134                     }
5135                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5136                 };
5137                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5138             } else {
5139                 print STDERR "unable to parse line in statfile - $line";
5140             }
5141         }
5142         $fd->close();
5143     }
5144 }
5145
5146 sub restore_archive {
5147     my ($archive, $vmid, $user, $opts) = @_;
5148
5149     my $format = $opts->{format};
5150     my $comp;
5151
5152     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5153         $format = 'tar' if !$format;
5154         $comp = 'gzip';
5155     } elsif ($archive =~ m/\.tar$/) {
5156         $format = 'tar' if !$format;
5157     } elsif ($archive =~ m/.tar.lzo$/) {
5158         $format = 'tar' if !$format;
5159         $comp = 'lzop';
5160     } elsif ($archive =~ m/\.vma$/) {
5161         $format = 'vma' if !$format;
5162     } elsif ($archive =~ m/\.vma\.gz$/) {
5163         $format = 'vma' if !$format;
5164         $comp = 'gzip';
5165     } elsif ($archive =~ m/\.vma\.lzo$/) {
5166         $format = 'vma' if !$format;
5167         $comp = 'lzop';
5168     } else {
5169         $format = 'vma' if !$format; # default
5170     }
5171
5172     # try to detect archive format
5173     if ($format eq 'tar') {
5174         return restore_tar_archive($archive, $vmid, $user, $opts);
5175     } else {
5176         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5177     }
5178 }
5179
5180 sub restore_update_config_line {
5181     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5182
5183     return if $line =~ m/^\#qmdump\#/;
5184     return if $line =~ m/^\#vzdump\#/;
5185     return if $line =~ m/^lock:/;
5186     return if $line =~ m/^unused\d+:/;
5187     return if $line =~ m/^parent:/;
5188     return if $line =~ m/^template:/; # restored VM is never a template
5189
5190     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5191         # try to convert old 1.X settings
5192         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5193         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5194             my ($model, $macaddr) = split(/\=/, $devconfig);
5195             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5196             my $net = {
5197                 model => $model,
5198                 bridge => "vmbr$ind",
5199                 macaddr => $macaddr,
5200             };
5201             my $netstr = print_net($net);
5202
5203             print $outfd "net$cookie->{netcount}: $netstr\n";
5204             $cookie->{netcount}++;
5205         }
5206     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5207         my ($id, $netstr) = ($1, $2);
5208         my $net = parse_net($netstr);
5209         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5210         $netstr = print_net($net);
5211         print $outfd "$id: $netstr\n";
5212     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5213         my $virtdev = $1;
5214         my $value = $3;
5215         if ($line =~ m/backup=no/) {
5216             print $outfd "#$line";
5217         } elsif ($virtdev && $map->{$virtdev}) {
5218             my $di = parse_drive($virtdev, $value);
5219             delete $di->{format}; # format can change on restore
5220             $di->{file} = $map->{$virtdev};
5221             $value = print_drive($vmid, $di);
5222             print $outfd "$virtdev: $value\n";
5223         } else {
5224             print $outfd $line;
5225         }
5226     } else {
5227         print $outfd $line;
5228     }
5229 }
5230
5231 sub scan_volids {
5232     my ($cfg, $vmid) = @_;
5233
5234     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5235
5236     my $volid_hash = {};
5237     foreach my $storeid (keys %$info) {
5238         foreach my $item (@{$info->{$storeid}}) {
5239             next if !($item->{volid} && $item->{size});
5240             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5241             $volid_hash->{$item->{volid}} = $item;
5242         }
5243     }
5244
5245     return $volid_hash;
5246 }
5247
5248 sub is_volume_in_use {
5249     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5250
5251     my $path = PVE::Storage::path($storecfg, $volid);
5252
5253     my $scan_config = sub {
5254         my ($cref, $snapname) = @_;
5255
5256         foreach my $key (keys %$cref) {
5257             my $value = $cref->{$key};
5258             if (valid_drivename($key)) {
5259                 next if $skip_drive && $key eq $skip_drive;
5260                 my $drive = parse_drive($key, $value);
5261                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5262                 return 1 if $volid eq $drive->{file};
5263                 if ($drive->{file} =~ m!^/!) {
5264                     return 1 if $drive->{file} eq $path;
5265                 } else {
5266                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5267                     next if !$storeid;
5268                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5269                     next if !$scfg;
5270                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5271                 }
5272             }
5273         }
5274
5275         return 0;
5276     };
5277
5278     return 1 if &$scan_config($conf);
5279
5280     undef $skip_drive;
5281
5282     foreach my $snapname (keys %{$conf->{snapshots}}) {
5283         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5284     }
5285
5286     return 0;
5287 }
5288
5289 sub update_disksize {
5290     my ($vmid, $conf, $volid_hash) = @_;
5291
5292     my $changes;
5293
5294     my $used = {};
5295
5296     # Note: it is allowed to define multiple storages with same path (alias), so
5297     # we need to check both 'volid' and real 'path' (two different volid can point
5298     # to the same path).
5299
5300     my $usedpath = {};
5301
5302     # update size info
5303     foreach my $opt (keys %$conf) {
5304         if (valid_drivename($opt)) {
5305             my $drive = parse_drive($opt, $conf->{$opt});
5306             my $volid = $drive->{file};
5307             next if !$volid;
5308
5309             $used->{$volid} = 1;
5310             if ($volid_hash->{$volid} &&
5311                 (my $path = $volid_hash->{$volid}->{path})) {
5312                 $usedpath->{$path} = 1;
5313             }
5314
5315             next if drive_is_cdrom($drive);
5316             next if !$volid_hash->{$volid};
5317
5318             $drive->{size} = $volid_hash->{$volid}->{size};
5319             my $new = print_drive($vmid, $drive);
5320             if ($new ne $conf->{$opt}) {
5321                 $changes = 1;
5322                 $conf->{$opt} = $new;
5323             }
5324         }
5325     }
5326
5327     # remove 'unusedX' entry if volume is used
5328     foreach my $opt (keys %$conf) {
5329         next if $opt !~ m/^unused\d+$/;
5330         my $volid = $conf->{$opt};
5331         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5332         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5333             $changes = 1;
5334             delete $conf->{$opt};
5335         }
5336     }
5337
5338     foreach my $volid (sort keys %$volid_hash) {
5339         next if $volid =~ m/vm-$vmid-state-/;
5340         next if $used->{$volid};
5341         my $path = $volid_hash->{$volid}->{path};
5342         next if !$path; # just to be sure
5343         next if $usedpath->{$path};
5344         $changes = 1;
5345         add_unused_volume($conf, $volid);
5346         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5347     }
5348
5349     return $changes;
5350 }
5351
5352 sub rescan {
5353     my ($vmid, $nolock) = @_;
5354
5355     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5356
5357     my $volid_hash = scan_volids($cfg, $vmid);
5358
5359     my $updatefn =  sub {
5360         my ($vmid) = @_;
5361
5362         my $conf = load_config($vmid);
5363
5364         check_lock($conf);
5365
5366         my $vm_volids = {};
5367         foreach my $volid (keys %$volid_hash) {
5368             my $info = $volid_hash->{$volid};
5369             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5370         }
5371
5372         my $changes = update_disksize($vmid, $conf, $vm_volids);
5373
5374         update_config_nolock($vmid, $conf, 1) if $changes;
5375     };
5376
5377     if (defined($vmid)) {
5378         if ($nolock) {
5379             &$updatefn($vmid);
5380         } else {
5381             lock_config($vmid, $updatefn, $vmid);
5382         }
5383     } else {
5384         my $vmlist = config_list();
5385         foreach my $vmid (keys %$vmlist) {
5386             if ($nolock) {
5387                 &$updatefn($vmid);
5388             } else {
5389                 lock_config($vmid, $updatefn, $vmid);
5390             }
5391         }
5392     }
5393 }
5394
5395 sub restore_vma_archive {
5396     my ($archive, $vmid, $user, $opts, $comp) = @_;
5397
5398     my $input = $archive eq '-' ? "<&STDIN" : undef;
5399     my $readfrom = $archive;
5400
5401     my $uncomp = '';
5402     if ($comp) {
5403         $readfrom = '-';
5404         my $qarchive = PVE::Tools::shellquote($archive);
5405         if ($comp eq 'gzip') {
5406             $uncomp = "zcat $qarchive|";
5407         } elsif ($comp eq 'lzop') {
5408             $uncomp = "lzop -d -c $qarchive|";
5409         } else {
5410             die "unknown compression method '$comp'\n";
5411         }
5412
5413     }
5414
5415     my $tmpdir = "/var/tmp/vzdumptmp$$";
5416     rmtree $tmpdir;
5417
5418     # disable interrupts (always do cleanups)
5419     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5420         warn "got interrupt - ignored\n";
5421     };
5422
5423     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5424     POSIX::mkfifo($mapfifo, 0600);
5425     my $fifofh;
5426
5427     my $openfifo = sub {
5428         open($fifofh, '>', $mapfifo) || die $!;
5429     };
5430
5431     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5432
5433     my $oldtimeout;
5434     my $timeout = 5;
5435
5436     my $devinfo = {};
5437
5438     my $rpcenv = PVE::RPCEnvironment::get();
5439
5440     my $conffile = config_file($vmid);
5441     my $tmpfn = "$conffile.$$.tmp";
5442
5443     # Note: $oldconf is undef if VM does not exists
5444     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5445
5446     my $print_devmap = sub {
5447         my $virtdev_hash = {};
5448
5449         my $cfgfn = "$tmpdir/qemu-server.conf";
5450
5451         # we can read the config - that is already extracted
5452         my $fh = IO::File->new($cfgfn, "r") ||
5453             "unable to read qemu-server.conf - $!\n";
5454
5455         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5456         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5457             if -f $fwcfgfn;
5458
5459         while (defined(my $line = <$fh>)) {
5460             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5461                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5462                 die "archive does not contain data for drive '$virtdev'\n"
5463                     if !$devinfo->{$devname};
5464                 if (defined($opts->{storage})) {
5465                     $storeid = $opts->{storage} || 'local';
5466                 } elsif (!$storeid) {
5467                     $storeid = 'local';
5468                 }
5469                 $format = 'raw' if !$format;
5470                 $devinfo->{$devname}->{devname} = $devname;
5471                 $devinfo->{$devname}->{virtdev} = $virtdev;
5472                 $devinfo->{$devname}->{format} = $format;
5473                 $devinfo->{$devname}->{storeid} = $storeid;
5474
5475                 # check permission on storage
5476                 my $pool = $opts->{pool}; # todo: do we need that?
5477                 if ($user ne 'root@pam') {
5478                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5479                 }
5480
5481                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5482             }
5483         }
5484
5485         foreach my $devname (keys %$devinfo) {
5486             die "found no device mapping information for device '$devname'\n"
5487                 if !$devinfo->{$devname}->{virtdev};
5488         }
5489
5490         my $cfg = cfs_read_file('storage.cfg');
5491
5492         # create empty/temp config
5493         if ($oldconf) {
5494             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5495             foreach_drive($oldconf, sub {
5496                 my ($ds, $drive) = @_;
5497
5498                 return if drive_is_cdrom($drive);
5499
5500                 my $volid = $drive->{file};
5501
5502                 return if !$volid || $volid =~ m|^/|;
5503
5504                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5505                 return if !$path || !$owner || ($owner != $vmid);
5506
5507                 # Note: only delete disk we want to restore
5508                 # other volumes will become unused
5509                 if ($virtdev_hash->{$ds}) {
5510                     PVE::Storage::vdisk_free($cfg, $volid);
5511                 }
5512             });
5513         }
5514
5515         my $map = {};
5516         foreach my $virtdev (sort keys %$virtdev_hash) {
5517             my $d = $virtdev_hash->{$virtdev};
5518             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5519             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5520
5521             # test if requested format is supported
5522             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5523             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5524             $d->{format} = $defFormat if !$supported;
5525
5526             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5527                                                   $d->{format}, undef, $alloc_size);
5528             print STDERR "new volume ID is '$volid'\n";
5529             $d->{volid} = $volid;
5530             my $path = PVE::Storage::path($cfg, $volid);
5531
5532             PVE::Storage::activate_volumes($cfg,[$volid]);
5533
5534             my $write_zeros = 1;
5535             # fixme: what other storages types initialize volumes with zero?
5536             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5537                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5538                 $write_zeros = 0;
5539             }
5540
5541             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5542
5543             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5544             $map->{$virtdev} = $volid;
5545         }
5546
5547         $fh->seek(0, 0) || die "seek failed - $!\n";
5548
5549         my $outfd = new IO::File ($tmpfn, "w") ||
5550             die "unable to write config for VM $vmid\n";
5551
5552         my $cookie = { netcount => 0 };
5553         while (defined(my $line = <$fh>)) {
5554             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5555         }
5556
5557         $fh->close();
5558         $outfd->close();
5559     };
5560
5561     eval {
5562         # enable interrupts
5563         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5564             die "interrupted by signal\n";
5565         };
5566         local $SIG{ALRM} = sub { die "got timeout\n"; };
5567
5568         $oldtimeout = alarm($timeout);
5569
5570         my $parser = sub {
5571             my $line = shift;
5572
5573             print "$line\n";
5574
5575             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5576                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5577                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5578             } elsif ($line =~ m/^CTIME: /) {
5579                 # we correctly received the vma config, so we can disable
5580                 # the timeout now for disk allocation (set to 10 minutes, so
5581                 # that we always timeout if something goes wrong)
5582                 alarm(600);
5583                 &$print_devmap();
5584                 print $fifofh "done\n";
5585                 my $tmp = $oldtimeout || 0;
5586                 $oldtimeout = undef;
5587                 alarm($tmp);
5588                 close($fifofh);
5589             }
5590         };
5591
5592         print "restore vma archive: $cmd\n";
5593         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5594     };
5595     my $err = $@;
5596
5597     alarm($oldtimeout) if $oldtimeout;
5598
5599     my $vollist = [];
5600     foreach my $devname (keys %$devinfo) {
5601         my $volid = $devinfo->{$devname}->{volid};
5602         push @$vollist, $volid if $volid;
5603     }
5604
5605     my $cfg = cfs_read_file('storage.cfg');
5606     PVE::Storage::deactivate_volumes($cfg, $vollist);
5607
5608     unlink $mapfifo;
5609
5610     if ($err) {
5611         rmtree $tmpdir;
5612         unlink $tmpfn;
5613
5614         foreach my $devname (keys %$devinfo) {
5615             my $volid = $devinfo->{$devname}->{volid};
5616             next if !$volid;
5617             eval {
5618                 if ($volid =~ m|^/|) {
5619                     unlink $volid || die 'unlink failed\n';
5620                 } else {
5621                     PVE::Storage::vdisk_free($cfg, $volid);
5622                 }
5623                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5624             };
5625             print STDERR "unable to cleanup '$volid' - $@" if $@;
5626         }
5627         die $err;
5628     }
5629
5630     rmtree $tmpdir;
5631
5632     rename($tmpfn, $conffile) ||
5633         die "unable to commit configuration file '$conffile'\n";
5634
5635     PVE::Cluster::cfs_update(); # make sure we read new file
5636
5637     eval { rescan($vmid, 1); };
5638     warn $@ if $@;
5639 }
5640
5641 sub restore_tar_archive {
5642     my ($archive, $vmid, $user, $opts) = @_;
5643
5644     if ($archive ne '-') {
5645         my $firstfile = tar_archive_read_firstfile($archive);
5646         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5647             if $firstfile ne 'qemu-server.conf';
5648     }
5649
5650     my $storecfg = cfs_read_file('storage.cfg');
5651
5652     # destroy existing data - keep empty config
5653     my $vmcfgfn = config_file($vmid);
5654     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5655
5656     my $tocmd = "/usr/lib/qemu-server/qmextract";
5657
5658     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5659     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5660     $tocmd .= ' --prealloc' if $opts->{prealloc};
5661     $tocmd .= ' --info' if $opts->{info};
5662
5663     # tar option "xf" does not autodetect compression when read from STDIN,
5664     # so we pipe to zcat
5665     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5666         PVE::Tools::shellquote("--to-command=$tocmd");
5667
5668     my $tmpdir = "/var/tmp/vzdumptmp$$";
5669     mkpath $tmpdir;
5670
5671     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5672     local $ENV{VZDUMP_VMID} = $vmid;
5673     local $ENV{VZDUMP_USER} = $user;
5674
5675     my $conffile = config_file($vmid);
5676     my $tmpfn = "$conffile.$$.tmp";
5677
5678     # disable interrupts (always do cleanups)
5679     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5680         print STDERR "got interrupt - ignored\n";
5681     };
5682
5683     eval {
5684         # enable interrupts
5685         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5686             die "interrupted by signal\n";
5687         };
5688
5689         if ($archive eq '-') {
5690             print "extracting archive from STDIN\n";
5691             run_command($cmd, input => "<&STDIN");
5692         } else {
5693             print "extracting archive '$archive'\n";
5694             run_command($cmd);
5695         }
5696
5697         return if $opts->{info};
5698
5699         # read new mapping
5700         my $map = {};
5701         my $statfile = "$tmpdir/qmrestore.stat";
5702         if (my $fd = IO::File->new($statfile, "r")) {
5703             while (defined (my $line = <$fd>)) {
5704                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5705                     $map->{$1} = $2 if $1;
5706                 } else {
5707                     print STDERR "unable to parse line in statfile - $line\n";
5708                 }
5709             }
5710             $fd->close();
5711         }
5712
5713         my $confsrc = "$tmpdir/qemu-server.conf";
5714
5715         my $srcfd = new IO::File($confsrc, "r") ||
5716             die "unable to open file '$confsrc'\n";
5717
5718         my $outfd = new IO::File ($tmpfn, "w") ||
5719             die "unable to write config for VM $vmid\n";
5720
5721         my $cookie = { netcount => 0 };
5722         while (defined (my $line = <$srcfd>)) {
5723             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5724         }
5725
5726         $srcfd->close();
5727         $outfd->close();
5728     };
5729     my $err = $@;
5730
5731     if ($err) {
5732
5733         unlink $tmpfn;
5734
5735         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5736
5737         die $err;
5738     }
5739
5740     rmtree $tmpdir;
5741
5742     rename $tmpfn, $conffile ||
5743         die "unable to commit configuration file '$conffile'\n";
5744
5745     PVE::Cluster::cfs_update(); # make sure we read new file
5746
5747     eval { rescan($vmid, 1); };
5748     warn $@ if $@;
5749 };
5750
5751
5752 # Internal snapshots
5753
5754 # NOTE: Snapshot create/delete involves several non-atomic
5755 # action, and can take a long time.
5756 # So we try to avoid locking the file and use 'lock' variable
5757 # inside the config file instead.
5758
5759 my $snapshot_copy_config = sub {
5760     my ($source, $dest) = @_;
5761
5762     foreach my $k (keys %$source) {
5763         next if $k eq 'snapshots';
5764         next if $k eq 'snapstate';
5765         next if $k eq 'snaptime';
5766         next if $k eq 'vmstate';
5767         next if $k eq 'lock';
5768         next if $k eq 'digest';
5769         next if $k eq 'description';
5770         next if $k =~ m/^unused\d+$/;
5771
5772         $dest->{$k} = $source->{$k};
5773     }
5774 };
5775
5776 my $snapshot_apply_config = sub {
5777     my ($conf, $snap) = @_;
5778
5779     # copy snapshot list
5780     my $newconf = {
5781         snapshots => $conf->{snapshots},
5782     };
5783
5784     # keep description and list of unused disks
5785     foreach my $k (keys %$conf) {
5786         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5787         $newconf->{$k} = $conf->{$k};
5788     }
5789
5790     &$snapshot_copy_config($snap, $newconf);
5791
5792     return $newconf;
5793 };
5794
5795 sub foreach_writable_storage {
5796     my ($conf, $func) = @_;
5797
5798     my $sidhash = {};
5799
5800     foreach my $ds (keys %$conf) {
5801         next if !valid_drivename($ds);
5802
5803         my $drive = parse_drive($ds, $conf->{$ds});
5804         next if !$drive;
5805         next if drive_is_cdrom($drive);
5806
5807         my $volid = $drive->{file};
5808
5809         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5810         $sidhash->{$sid} = $sid if $sid;
5811     }
5812
5813     foreach my $sid (sort keys %$sidhash) {
5814         &$func($sid);
5815     }
5816 }
5817
5818 my $alloc_vmstate_volid = sub {
5819     my ($storecfg, $vmid, $conf, $snapname) = @_;
5820
5821     # Note: we try to be smart when selecting a $target storage
5822
5823     my $target;
5824
5825     # search shared storage first
5826     foreach_writable_storage($conf, sub {
5827         my ($sid) = @_;
5828         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5829         return if !$scfg->{shared};
5830
5831         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5832     });
5833
5834     if (!$target) {
5835         # now search local storage
5836         foreach_writable_storage($conf, sub {
5837             my ($sid) = @_;
5838             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5839             return if $scfg->{shared};
5840
5841             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5842         });
5843     }
5844
5845     $target = 'local' if !$target;
5846
5847     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5848     # we abort live save after $conf->{memory}, so we need at max twice that space
5849     my $size = $conf->{memory}*2 + $driver_state_size;
5850
5851     my $name = "vm-$vmid-state-$snapname";
5852     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5853     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5854     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5855
5856     return $volid;
5857 };
5858
5859 my $snapshot_prepare = sub {
5860     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5861
5862     my $snap;
5863
5864     my $updatefn =  sub {
5865
5866         my $conf = load_config($vmid);
5867
5868         die "you can't take a snapshot if it's a template\n"
5869             if is_template($conf);
5870
5871         check_lock($conf);
5872
5873         $conf->{lock} = 'snapshot';
5874
5875         die "snapshot name '$snapname' already used\n"
5876             if defined($conf->{snapshots}->{$snapname});
5877
5878         my $storecfg = PVE::Storage::config();
5879         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5880
5881         $snap = $conf->{snapshots}->{$snapname} = {};
5882
5883         if ($save_vmstate && check_running($vmid)) {
5884             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5885         }
5886
5887         &$snapshot_copy_config($conf, $snap);
5888
5889         $snap->{snapstate} = "prepare";
5890         $snap->{snaptime} = time();
5891         $snap->{description} = $comment if $comment;
5892
5893         # always overwrite machine if we save vmstate. This makes sure we
5894         # can restore it later using correct machine type
5895         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5896
5897         update_config_nolock($vmid, $conf, 1);
5898     };
5899
5900     lock_config($vmid, $updatefn);
5901
5902     return $snap;
5903 };
5904
5905 my $snapshot_commit = sub {
5906     my ($vmid, $snapname) = @_;
5907
5908     my $updatefn = sub {
5909
5910         my $conf = load_config($vmid);
5911
5912         die "missing snapshot lock\n"
5913             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5914
5915         my $has_machine_config = defined($conf->{machine});
5916
5917         my $snap = $conf->{snapshots}->{$snapname};
5918
5919         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5920
5921         die "wrong snapshot state\n"
5922             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5923
5924         delete $snap->{snapstate};
5925         delete $conf->{lock};
5926
5927         my $newconf = &$snapshot_apply_config($conf, $snap);
5928
5929         delete $newconf->{machine} if !$has_machine_config;
5930
5931         $newconf->{parent} = $snapname;
5932
5933         update_config_nolock($vmid, $newconf, 1);
5934     };
5935
5936     lock_config($vmid, $updatefn);
5937 };
5938
5939 sub snapshot_rollback {
5940     my ($vmid, $snapname) = @_;
5941
5942     my $prepare = 1;
5943
5944     my $storecfg = PVE::Storage::config();
5945
5946     my $conf = load_config($vmid);
5947
5948     my $get_snapshot_config = sub {
5949
5950         die "you can't rollback if vm is a template\n" if is_template($conf);
5951
5952         my $res = $conf->{snapshots}->{$snapname};
5953
5954         die "snapshot '$snapname' does not exist\n" if !defined($res);
5955
5956         return $res;
5957     };
5958
5959     my $snap = &$get_snapshot_config();
5960
5961     foreach_drive($snap, sub {
5962         my ($ds, $drive) = @_;
5963
5964         return if drive_is_cdrom($drive);
5965
5966         my $volid = $drive->{file};
5967
5968         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5969     });
5970
5971     my $updatefn = sub {
5972
5973         $conf = load_config($vmid);
5974
5975         $snap = &$get_snapshot_config();
5976
5977         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5978             if $snap->{snapstate};
5979
5980         if ($prepare) {
5981             check_lock($conf);
5982             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5983         }
5984
5985         die "unable to rollback vm $vmid: vm is running\n"
5986             if check_running($vmid);
5987
5988         if ($prepare) {
5989             $conf->{lock} = 'rollback';
5990         } else {
5991             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5992             delete $conf->{lock};
5993         }
5994
5995         my $forcemachine;
5996
5997         if (!$prepare) {
5998             my $has_machine_config = defined($conf->{machine});
5999
6000             # copy snapshot config to current config
6001             $conf = &$snapshot_apply_config($conf, $snap);
6002             $conf->{parent} = $snapname;
6003
6004             # Note: old code did not store 'machine', so we try to be smart
6005             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6006             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6007             # we remove the 'machine' configuration if not explicitly specified
6008             # in the original config.
6009             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6010         }
6011
6012         update_config_nolock($vmid, $conf, 1);
6013
6014         if (!$prepare && $snap->{vmstate}) {
6015             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6016             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6017         }
6018     };
6019
6020     lock_config($vmid, $updatefn);
6021
6022     foreach_drive($snap, sub {
6023         my ($ds, $drive) = @_;
6024
6025         return if drive_is_cdrom($drive);
6026
6027         my $volid = $drive->{file};
6028         my $device = "drive-$ds";
6029
6030         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6031     });
6032
6033     $prepare = 0;
6034     lock_config($vmid, $updatefn);
6035 }
6036
6037 my $savevm_wait = sub {
6038     my ($vmid) = @_;
6039
6040     for(;;) {
6041         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6042         if (!$stat->{status}) {
6043             die "savevm not active\n";
6044         } elsif ($stat->{status} eq 'active') {
6045             sleep(1);
6046             next;
6047         } elsif ($stat->{status} eq 'completed') {
6048             last;
6049         } else {
6050             die "query-savevm returned status '$stat->{status}'\n";
6051         }
6052     }
6053 };
6054
6055 sub do_snapshots_with_qemu {
6056     my ($storecfg, $volid) = @_;
6057
6058     my $storage_name = PVE::Storage::parse_volume_id($volid);
6059
6060     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6061         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6062         return 1;
6063     }
6064
6065     if ($volid =~ m/\.(qcow2|qed)$/){
6066         return 1;
6067     }
6068
6069     return undef;
6070 }
6071
6072 sub snapshot_create {
6073     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6074
6075     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6076
6077     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6078
6079     my $config = load_config($vmid);
6080
6081     my $running = check_running($vmid);
6082
6083     my $freezefs = $running && $config->{agent};
6084     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6085
6086     my $drivehash = {};
6087
6088     if ($freezefs) {
6089         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6090         warn "guest-fsfreeze-freeze problems - $@" if $@;
6091     }
6092
6093     eval {
6094         # create internal snapshots of all drives
6095
6096         my $storecfg = PVE::Storage::config();
6097
6098         if ($running) {
6099             if ($snap->{vmstate}) {
6100                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6101                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6102                 &$savevm_wait($vmid);
6103             } else {
6104                 vm_mon_cmd($vmid, "savevm-start");
6105             }
6106         };
6107
6108         foreach_drive($snap, sub {
6109             my ($ds, $drive) = @_;
6110
6111             return if drive_is_cdrom($drive);
6112
6113             my $volid = $drive->{file};
6114             my $device = "drive-$ds";
6115
6116             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6117             $drivehash->{$ds} = 1;
6118        });
6119     };
6120     my $err = $@;
6121
6122     if ($running) {
6123         eval { vm_mon_cmd($vmid, "savevm-end")  };
6124         warn $@ if $@;
6125
6126         if ($freezefs) {
6127             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6128             warn "guest-fsfreeze-thaw problems - $@" if $@;
6129         }
6130
6131         # savevm-end is async, we need to wait
6132         for (;;) {
6133             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6134             if (!$stat->{bytes}) {
6135                 last;
6136             } else {
6137                 print "savevm not yet finished\n";
6138                 sleep(1);
6139                 next;
6140             }
6141         }
6142     }
6143
6144     if ($err) {
6145         warn "snapshot create failed: starting cleanup\n";
6146         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6147         warn $@ if $@;
6148         die $err;
6149     }
6150
6151     &$snapshot_commit($vmid, $snapname);
6152 }
6153
6154 # Note: $drivehash is only set when called from snapshot_create.
6155 sub snapshot_delete {
6156     my ($vmid, $snapname, $force, $drivehash) = @_;
6157
6158     my $prepare = 1;
6159
6160     my $snap;
6161     my $unused = [];
6162
6163     my $unlink_parent = sub {
6164         my ($confref, $new_parent) = @_;
6165
6166         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6167             if ($new_parent) {
6168                 $confref->{parent} = $new_parent;
6169             } else {
6170                 delete $confref->{parent};
6171             }
6172         }
6173     };
6174
6175     my $updatefn =  sub {
6176         my ($remove_drive) = @_;
6177
6178         my $conf = load_config($vmid);
6179
6180         if (!$drivehash) {
6181             check_lock($conf);
6182             die "you can't delete a snapshot if vm is a template\n"
6183                 if is_template($conf);
6184         }
6185
6186         $snap = $conf->{snapshots}->{$snapname};
6187
6188         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6189
6190         # remove parent refs
6191         if (!$prepare) {
6192             &$unlink_parent($conf, $snap->{parent});
6193             foreach my $sn (keys %{$conf->{snapshots}}) {
6194                 next if $sn eq $snapname;
6195                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6196             }
6197         }
6198
6199         if ($remove_drive) {
6200             if ($remove_drive eq 'vmstate') {
6201                 delete $snap->{$remove_drive};
6202             } else {
6203                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6204                 my $volid = $drive->{file};
6205                 delete $snap->{$remove_drive};
6206                 add_unused_volume($conf, $volid);
6207             }
6208         }
6209
6210         if ($prepare) {
6211             $snap->{snapstate} = 'delete';
6212         } else {
6213             delete $conf->{snapshots}->{$snapname};
6214             delete $conf->{lock} if $drivehash;
6215             foreach my $volid (@$unused) {
6216                 add_unused_volume($conf, $volid);
6217             }
6218         }
6219
6220         update_config_nolock($vmid, $conf, 1);
6221     };
6222
6223     lock_config($vmid, $updatefn);
6224
6225     # now remove vmstate file
6226
6227     my $storecfg = PVE::Storage::config();
6228
6229     if ($snap->{vmstate}) {
6230         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6231         if (my $err = $@) {
6232             die $err if !$force;
6233             warn $err;
6234         }
6235         # save changes (remove vmstate from snapshot)
6236         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6237     };
6238
6239     # now remove all internal snapshots
6240     foreach_drive($snap, sub {
6241         my ($ds, $drive) = @_;
6242
6243         return if drive_is_cdrom($drive);
6244
6245         my $volid = $drive->{file};
6246         my $device = "drive-$ds";
6247
6248         if (!$drivehash || $drivehash->{$ds}) {
6249             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6250             if (my $err = $@) {
6251                 die $err if !$force;
6252                 warn $err;
6253             }
6254         }
6255
6256         # save changes (remove drive fron snapshot)
6257         lock_config($vmid, $updatefn, $ds) if !$force;
6258         push @$unused, $volid;
6259     });
6260
6261     # now cleanup config
6262     $prepare = 0;
6263     lock_config($vmid, $updatefn);
6264 }
6265
6266 sub has_feature {
6267     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6268
6269     my $err;
6270     foreach_drive($conf, sub {
6271         my ($ds, $drive) = @_;
6272
6273         return if drive_is_cdrom($drive);
6274         my $volid = $drive->{file};
6275         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6276     });
6277
6278     return $err ? 0 : 1;
6279 }
6280
6281 sub template_create {
6282     my ($vmid, $conf, $disk) = @_;
6283
6284     my $storecfg = PVE::Storage::config();
6285
6286     foreach_drive($conf, sub {
6287         my ($ds, $drive) = @_;
6288
6289         return if drive_is_cdrom($drive);
6290         return if $disk && $ds ne $disk;
6291
6292         my $volid = $drive->{file};
6293         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6294
6295         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6296         $drive->{file} = $voliddst;
6297         $conf->{$ds} = print_drive($vmid, $drive);
6298         update_config_nolock($vmid, $conf, 1);
6299     });
6300 }
6301
6302 sub is_template {
6303     my ($conf) = @_;
6304
6305     return 1 if defined $conf->{template} && $conf->{template} == 1;
6306 }
6307
6308 sub qemu_img_convert {
6309     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6310
6311     my $storecfg = PVE::Storage::config();
6312     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6313     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6314
6315     if ($src_storeid && $dst_storeid) {
6316
6317         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6318
6319         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6320         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6321
6322         my $src_format = qemu_img_format($src_scfg, $src_volname);
6323         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6324
6325         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6326         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6327
6328         my $cmd = [];
6329         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6330         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6331         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6332
6333         my $parser = sub {
6334             my $line = shift;
6335             if($line =~ m/\((\S+)\/100\%\)/){
6336                 my $percent = $1;
6337                 my $transferred = int($size * $percent / 100);
6338                 my $remaining = $size - $transferred;
6339
6340                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6341             }
6342
6343         };
6344
6345         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6346         my $err = $@;
6347         die "copy failed: $err" if $err;
6348     }
6349 }
6350
6351 sub qemu_img_format {
6352     my ($scfg, $volname) = @_;
6353
6354     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6355         return $1;
6356     } else {
6357         return "raw";
6358     }
6359 }
6360
6361 sub qemu_drive_mirror {
6362     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6363
6364     my $storecfg = PVE::Storage::config();
6365     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6366
6367     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6368
6369     my $format = qemu_img_format($dst_scfg, $dst_volname);
6370
6371     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6372
6373     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6374     $opts->{format} = $format if $format;
6375
6376     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6377
6378     eval {
6379     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6380         while (1) {
6381             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6382             my $stat = @$stats[0];
6383             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6384             die "error job is not mirroring" if $stat->{type} ne "mirror";
6385
6386             my $busy = $stat->{busy};
6387             my $ready = $stat->{ready};
6388
6389             if (my $total = $stat->{len}) {
6390                 my $transferred = $stat->{offset} || 0;
6391                 my $remaining = $total - $transferred;
6392                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6393
6394                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6395             }
6396
6397
6398             if ($stat->{ready} eq 'true') {
6399
6400                 last if $vmiddst != $vmid;
6401
6402                 # try to switch the disk if source and destination are on the same guest
6403                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6404                 last if !$@;
6405                 die $@ if $@ !~ m/cannot be completed/;
6406             }
6407             sleep 1;
6408         }
6409
6410
6411     };
6412     my $err = $@;
6413
6414     my $cancel_job = sub {
6415         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6416         while (1) {
6417             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6418             my $stat = @$stats[0];
6419             last if !$stat;
6420             sleep 1;
6421         }
6422     };
6423
6424     if ($err) {
6425         eval { &$cancel_job(); };
6426         die "mirroring error: $err";
6427     }
6428
6429     if ($vmiddst != $vmid) {
6430         # if we clone a disk for a new target vm, we don't switch the disk
6431         &$cancel_job(); # so we call block-job-cancel
6432     }
6433 }
6434
6435 sub clone_disk {
6436     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6437         $newvmid, $storage, $format, $full, $newvollist) = @_;
6438
6439     my $newvolid;
6440
6441     if (!$full) {
6442         print "create linked clone of drive $drivename ($drive->{file})\n";
6443         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6444         push @$newvollist, $newvolid;
6445     } else {
6446         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6447         $storeid = $storage if $storage;
6448
6449         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6450         if (!$format) {
6451             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6452             $format = qemu_img_format($scfg, $volname);
6453         }
6454
6455         # test if requested format is supported - else use default
6456         my $supported = grep { $_ eq $format } @$validFormats;
6457         $format = $defFormat if !$supported;
6458
6459         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6460
6461         print "create full clone of drive $drivename ($drive->{file})\n";
6462         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6463         push @$newvollist, $newvolid;
6464
6465         PVE::Storage::activate_volumes($storecfg, $newvollist);
6466
6467         if (!$running || $snapname) {
6468             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6469         } else {
6470             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6471         }
6472     }
6473
6474     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6475
6476     my $disk = $drive;
6477     $disk->{format} = undef;
6478     $disk->{file} = $newvolid;
6479     $disk->{size} = $size;
6480
6481     return $disk;
6482 }
6483
6484 # this only works if VM is running
6485 sub get_current_qemu_machine {
6486     my ($vmid) = @_;
6487
6488     my $cmd = { execute => 'query-machines', arguments => {} };
6489     my $res = vm_qmp_command($vmid, $cmd);
6490
6491     my ($current, $default);
6492     foreach my $e (@$res) {
6493         $default = $e->{name} if $e->{'is-default'};
6494         $current = $e->{name} if $e->{'is-current'};
6495     }
6496
6497     # fallback to the default machine if current is not supported by qemu
6498     return $current || $default || 'pc';
6499 }
6500
6501 sub qemu_machine_feature_enabled {
6502     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6503
6504     my $current_major;
6505     my $current_minor;
6506
6507     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6508
6509         $current_major = $3;
6510         $current_minor = $4;
6511
6512     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6513
6514         $current_major = $1;
6515         $current_minor = $2;
6516     }
6517
6518     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6519
6520
6521 }
6522
6523 sub qemu_machine_pxe {
6524     my ($vmid, $conf, $machine) = @_;
6525
6526     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6527
6528     foreach my $opt (keys %$conf) {
6529         next if $opt !~ m/^net(\d+)$/;
6530         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6531         next if !$net;
6532         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6533         return $machine.".pxe" if $romfile =~ m/pxe/;
6534         last;
6535     }
6536
6537 }
6538
6539 sub qemu_use_old_bios_files {
6540     my ($machine_type) = @_;
6541
6542     return if !$machine_type;
6543
6544     my $use_old_bios_files = undef;
6545
6546     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6547         $machine_type = $1;
6548         $use_old_bios_files = 1;
6549     } else {
6550         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6551         # load new efi bios files on migration. So this hack is required to allow
6552         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6553         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6554         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6555     }
6556
6557     return ($use_old_bios_files, $machine_type);
6558 }
6559
6560 sub lspci {
6561
6562     my $devices = {};
6563
6564     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6565             my (undef, $id, $function) = @_;
6566             my $res = { id => $id, function => $function};
6567             push @{$devices->{$id}}, $res;
6568     });
6569
6570     return $devices;
6571 }
6572
6573 sub vm_iothreads_list {
6574     my ($vmid) = @_;
6575
6576     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6577
6578     my $iothreads = {};
6579     foreach my $iothread (@$res) {
6580         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6581     }
6582
6583     return $iothreads;
6584 }
6585
6586 sub scsihw_infos {
6587     my ($conf, $drive) = @_;
6588
6589     my $maxdev = 0;
6590
6591     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6592         $maxdev = 7;
6593     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6594         $maxdev = 1;
6595     } else {
6596         $maxdev = 256;
6597     }
6598
6599     my $controller = int($drive->{index} / $maxdev);
6600     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6601
6602     return ($maxdev, $controller, $controller_prefix);
6603 }
6604
6605 # bash completion helper
6606
6607 sub complete_backup_archives {
6608     my ($cmdname, $pname, $cvalue) = @_;
6609
6610     my $cfg = PVE::Storage::config();
6611
6612     my $storeid;
6613
6614     if ($cvalue =~ m/^([^:]+):/) {
6615         $storeid = $1;
6616     }
6617
6618     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6619
6620     my $res = [];
6621     foreach my $id (keys %$data) {
6622         foreach my $item (@{$data->{$id}}) {
6623             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6624             push @$res, $item->{volid} if defined($item->{volid});
6625         }
6626     }
6627
6628     return $res;
6629 }
6630
6631 my $complete_vmid_full = sub {
6632     my ($running) = @_;
6633
6634     my $idlist = vmstatus();
6635
6636     my $res = [];
6637
6638     foreach my $id (keys %$idlist) {
6639         my $d = $idlist->{$id};
6640         if (defined($running)) {
6641             next if $d->{template};
6642             next if $running && $d->{status} ne 'running';
6643             next if !$running && $d->{status} eq 'running';
6644         }
6645         push @$res, $id;
6646
6647     }
6648     return $res;
6649 };
6650
6651 sub complete_vmid {
6652     return &$complete_vmid_full();
6653 }
6654
6655 sub complete_vmid_stopped {
6656     return &$complete_vmid_full(0);
6657 }
6658
6659 sub complete_vmid_running {
6660     return &$complete_vmid_full(1);
6661 }
6662
6663 sub complete_storage {
6664
6665     my $cfg = PVE::Storage::config();
6666     my $ids = $cfg->{ids};
6667
6668     my $res = [];
6669     foreach my $sid (keys %$ids) {
6670         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6671         next if !$ids->{$sid}->{content}->{images};
6672         push @$res, $sid;
6673     }
6674
6675     return $res;
6676 }
6677
6678 1;