PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbdesc = {
 731     optional => 1,
 732     type => 'string', format => 'pve-qm-usb-device',
 733     typetext => 'host=HOSTUSBDEVICE|spice',
 734     description => <<EODESCR,
 735 Configure an USB device (n is 0 to 4). This can be used to
 736 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 737
 738 'bus-port(.port)*' (decimal numbers) or
 739 'vendor_id:product_id' (hexadeciaml numbers)
 740
 741 You can use the 'lsusb -t' command to list existing usb devices.
 742
 743 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 744
 745 The value 'spice' can be used to add a usb redirection devices for spice.
 746
 747 EODESCR
 748 };
 749 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 750
 751 my $hostpcidesc = {
 752         optional => 1,
 753         type => 'string', format => 'pve-qm-hostpci',
 754         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 755         description => <<EODESCR,
 756 Map host pci devices. HOSTPCIDEVICE syntax is:
 757
 758 'bus:dev.func' (hexadecimal numbers)
 759
 760 You can us the 'lspci' command to list existing pci devices.
 761
 762 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 763
 764 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 765
 766 Experimental: user reported problems with this option.
 767 EODESCR
 768 };
 769 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 770
 771 my $serialdesc = {
 772         optional => 1,
 773         type => 'string',
 774         pattern => '(/dev/.+|socket)',
 775         description =>  <<EODESCR,
 776 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 777
 778 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 779
 780 Experimental: user reported problems with this option.
 781 EODESCR
 782 };
 783
 784 my $paralleldesc= {
 785         optional => 1,
 786         type => 'string',
 787         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 788         description =>  <<EODESCR,
 789 Map host parallel devices (n is 0 to 2).
 790
 791 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 792
 793 Experimental: user reported problems with this option.
 794 EODESCR
 795 };
 796
 797 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 798     $confdesc->{"parallel$i"} = $paralleldesc;
 799 }
 800
 801 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 802     $confdesc->{"serial$i"} = $serialdesc;
 803 }
 804
 805 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 806     $confdesc->{"hostpci$i"} = $hostpcidesc;
 807 }
 808
 809 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 810     $drivename_hash->{"ide$i"} = 1;
 811     $confdesc->{"ide$i"} = $idedesc;
 812 }
 813
 814 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 815     $drivename_hash->{"sata$i"} = 1;
 816     $confdesc->{"sata$i"} = $satadesc;
 817 }
 818
 819 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 820     $drivename_hash->{"scsi$i"} = 1;
 821     $confdesc->{"scsi$i"} = $scsidesc ;
 822 }
 823
 824 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 825     $drivename_hash->{"virtio$i"} = 1;
 826     $confdesc->{"virtio$i"} = $virtiodesc;
 827 }
 828
 829 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 830     $confdesc->{"usb$i"} = $usbdesc;
 831 }
 832
 833 my $unuseddesc = {
 834     optional => 1,
 835     type => 'string', format => 'pve-volume-id',
 836     description => "Reference to unused volumes.",
 837 };
 838
 839 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 840     $confdesc->{"unused$i"} = $unuseddesc;
 841 }
 842
 843 my $kvm_api_version = 0;
 844
 845 sub kvm_version {
 846
 847     return $kvm_api_version if $kvm_api_version;
 848
 849     my $fh = IO::File->new("</dev/kvm") ||
 850         return 0;
 851
 852     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 853         $kvm_api_version = $v;
 854     }
 855
 856     $fh->close();
 857
 858     return  $kvm_api_version;
 859 }
 860
 861 my $kvm_user_version;
 862
 863 sub kvm_user_version {
 864
 865     return $kvm_user_version if $kvm_user_version;
 866
 867     $kvm_user_version = 'unknown';
 868
 869     my $tmp = `kvm -help 2>/dev/null`;
 870
 871     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 872         $kvm_user_version = $2;
 873     }
 874
 875     return $kvm_user_version;
 876
 877 }
 878
 879 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 880
 881 sub disknames {
 882     # order is important - used to autoselect boot disk
 883     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 884             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 885             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 886             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 887 }
 888
 889 sub valid_drivename {
 890     my $dev = shift;
 891
 892     return defined($drivename_hash->{$dev});
 893 }
 894
 895 sub option_exists {
 896     my $key = shift;
 897     return defined($confdesc->{$key});
 898 }
 899
 900 sub nic_models {
 901     return $nic_model_list;
 902 }
 903
 904 sub os_list_description {
 905
 906     return {
 907         other => 'Other',
 908         wxp => 'Windows XP',
 909         w2k => 'Windows 2000',
 910         w2k3 =>, 'Windows 2003',
 911         w2k8 => 'Windows 2008',
 912         wvista => 'Windows Vista',
 913         win7 => 'Windows 7',
 914         win8 => 'Windows 8/2012',
 915         l24 => 'Linux 2.4',
 916         l26 => 'Linux 2.6',
 917     };
 918 }
 919
 920 my $cdrom_path;
 921
 922 sub get_cdrom_path {
 923
 924     return  $cdrom_path if $cdrom_path;
 925
 926     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 927     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 928     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 929 }
 930
 931 sub get_iso_path {
 932     my ($storecfg, $vmid, $cdrom) = @_;
 933
 934     if ($cdrom eq 'cdrom') {
 935         return get_cdrom_path();
 936     } elsif ($cdrom eq 'none') {
 937         return '';
 938     } elsif ($cdrom =~ m|^/|) {
 939         return $cdrom;
 940     } else {
 941         return PVE::Storage::path($storecfg, $cdrom);
 942     }
 943 }
 944
 945 # try to convert old style file names to volume IDs
 946 sub filename_to_volume_id {
 947     my ($vmid, $file, $media) = @_;
 948
 949     if (!($file eq 'none' || $file eq 'cdrom' ||
 950           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 951
 952         return undef if $file =~ m|/|;
 953
 954         if ($media && $media eq 'cdrom') {
 955             $file = "local:iso/$file";
 956         } else {
 957             $file = "local:$vmid/$file";
 958         }
 959     }
 960
 961     return $file;
 962 }
 963
 964 sub verify_media_type {
 965     my ($opt, $vtype, $media) = @_;
 966
 967     return if !$media;
 968
 969     my $etype;
 970     if ($media eq 'disk') {
 971         $etype = 'images';
 972     } elsif ($media eq 'cdrom') {
 973         $etype = 'iso';
 974     } else {
 975         die "internal error";
 976     }
 977
 978     return if ($vtype eq $etype);
 979
 980     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 981 }
 982
 983 sub cleanup_drive_path {
 984     my ($opt, $storecfg, $drive) = @_;
 985
 986     # try to convert filesystem paths to volume IDs
 987
 988     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 989         ($drive->{file} !~ m|^/dev/.+|) &&
 990         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 991         ($drive->{file} !~ m/^\d+$/)) {
 992         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 993         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 994         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 995         verify_media_type($opt, $vtype, $drive->{media});
 996         $drive->{file} = $volid;
 997     }
 998
 999     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1000 }
1001
1002 sub create_conf_nolock {
1003     my ($vmid, $settings) = @_;
1004
1005     my $filename = config_file($vmid);
1006
1007     die "configuration file '$filename' already exists\n" if -f $filename;
1008
1009     my $defaults = load_defaults();
1010
1011     $settings->{name} = "vm$vmid" if !$settings->{name};
1012     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
1013
1014     my $data = '';
1015     foreach my $opt (keys %$settings) {
1016         next if !$confdesc->{$opt};
1017
1018         my $value = $settings->{$opt};
1019         next if !$value;
1020
1021         $data .= "$opt: $value\n";
1022     }
1023
1024     PVE::Tools::file_set_contents($filename, $data);
1025 }
1026
1027 sub parse_hotplug_features {
1028     my ($data) = @_;
1029
1030     my $res = {};
1031
1032     return $res if $data eq '0';
1033
1034     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1035
1036     foreach my $feature (PVE::Tools::split_list($data)) {
1037         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1038             $res->{$1} = 1;
1039         } else {
1040             warn "ignoring unknown hotplug feature '$feature'\n";
1041         }
1042     }
1043     return $res;
1044 }
1045
1046 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1047 sub pve_verify_hotplug_features {
1048     my ($value, $noerr) = @_;
1049
1050     return $value if parse_hotplug_features($value);
1051
1052     return undef if $noerr;
1053
1054     die "unable to parse hotplug option\n";
1055 }
1056
1057 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1058 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1059 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1060 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1061 #        [,iothread=on][,serial=serial][,model=model]
1062
1063 sub parse_drive {
1064     my ($key, $data) = @_;
1065
1066     my ($interface, $index);
1067
1068     if ($key =~ m/^([^\d]+)(\d+)$/) {
1069         $interface = $1;
1070         $index = $2;
1071     } else {
1072         return undef;
1073     }
1074
1075     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1076                                      : $confdesc->{$key}->{format};
1077     if (!$desc) {
1078         warn "invalid drive key: $key\n";
1079         return undef;
1080     }
1081     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1082     return undef if !$res;
1083     $res->{interface} = $interface;
1084     $res->{index} = $index;
1085
1086     my $error = 0;
1087     foreach my $opt (qw(bps bps_rd bps_wr)) {
1088         if (my $bps = defined(delete $res->{$opt})) {
1089             if (defined($res->{"m$opt"})) {
1090                 warn "both $opt and m$opt specified\n";
1091                 ++$error;
1092                 next;
1093             }
1094             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1095         }
1096     }
1097     return undef if $error;
1098
1099     return undef if $res->{mbps_rd} && $res->{mbps};
1100     return undef if $res->{mbps_wr} && $res->{mbps};
1101     return undef if $res->{iops_rd} && $res->{iops};
1102     return undef if $res->{iops_wr} && $res->{iops};
1103
1104     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1105         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1106         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1107         return undef if $res->{interface} eq 'virtio';
1108     }
1109
1110     if (my $size = $res->{size}) {
1111         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1112     }
1113
1114     return $res;
1115 }
1116
1117 sub print_drive {
1118     my ($vmid, $drive) = @_;
1119     my $data = { %$drive };
1120     delete $data->{$_} for qw(index interface);
1121     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1122 }
1123
1124 sub scsi_inquiry {
1125     my($fh, $noerr) = @_;
1126
1127     my $SG_IO = 0x2285;
1128     my $SG_GET_VERSION_NUM = 0x2282;
1129
1130     my $versionbuf = "\x00" x 8;
1131     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1132     if (!$ret) {
1133         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1134         return undef;
1135     }
1136     my $version = unpack("I", $versionbuf);
1137     if ($version < 30000) {
1138         die "scsi generic interface too old\n"  if !$noerr;
1139         return undef;
1140     }
1141
1142     my $buf = "\x00" x 36;
1143     my $sensebuf = "\x00" x 8;
1144     my $cmd = pack("C x3 C x1", 0x12, 36);
1145
1146     # see /usr/include/scsi/sg.h
1147     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1148
1149     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1150                       length($sensebuf), 0, length($buf), $buf,
1151                       $cmd, $sensebuf, 6000);
1152
1153     $ret = ioctl($fh, $SG_IO, $packet);
1154     if (!$ret) {
1155         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1156         return undef;
1157     }
1158
1159     my @res = unpack($sg_io_hdr_t, $packet);
1160     if ($res[17] || $res[18]) {
1161         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1162         return undef;
1163     }
1164
1165     my $res = {};
1166     (my $byte0, my $byte1, $res->{vendor},
1167      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1168
1169     $res->{removable} = $byte1 & 128 ? 1 : 0;
1170     $res->{type} = $byte0 & 31;
1171
1172     return $res;
1173 }
1174
1175 sub path_is_scsi {
1176     my ($path) = @_;
1177
1178     my $fh = IO::File->new("+<$path") || return undef;
1179     my $res = scsi_inquiry($fh, 1);
1180     close($fh);
1181
1182     return $res;
1183 }
1184
1185 sub machine_type_is_q35 {
1186     my ($conf) = @_;
1187
1188     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1189 }
1190
1191 sub print_tabletdevice_full {
1192     my ($conf) = @_;
1193
1194     my $q35 = machine_type_is_q35($conf);
1195
1196     # we use uhci for old VMs because tablet driver was buggy in older qemu
1197     my $usbbus = $q35 ? "ehci" : "uhci";
1198
1199     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1200 }
1201
1202 sub print_drivedevice_full {
1203     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1204
1205     my $device = '';
1206     my $maxdev = 0;
1207
1208     if ($drive->{interface} eq 'virtio') {
1209         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1210         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1211         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1212     } elsif ($drive->{interface} eq 'scsi') {
1213
1214         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1215         my $unit = $drive->{index} % $maxdev;
1216         my $devicetype = 'hd';
1217         my $path = '';
1218         if (drive_is_cdrom($drive)) {
1219               $devicetype = 'cd';
1220         } else {
1221               if ($drive->{file} =~ m|^/|) {
1222                   $path = $drive->{file};
1223               } else {
1224                   $path = PVE::Storage::path($storecfg, $drive->{file});
1225               }
1226
1227               if($path =~ m/^iscsi\:\/\//){
1228                   $devicetype = 'generic';
1229               } else {
1230                   if (my $info = path_is_scsi($path)) {
1231                       if ($info->{type} == 0) {
1232                           $devicetype = 'block';
1233                       } elsif ($info->{type} == 1) { # tape
1234                           $devicetype = 'generic';
1235                       }
1236                   }
1237               }
1238          }
1239
1240         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1241             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1242         } else {
1243             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1244         }
1245
1246     } elsif ($drive->{interface} eq 'ide'){
1247         $maxdev = 2;
1248         my $controller = int($drive->{index} / $maxdev);
1249         my $unit = $drive->{index} % $maxdev;
1250         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1251
1252         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1253         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1254             $device .= ",model=$model";
1255         }
1256     } elsif ($drive->{interface} eq 'sata'){
1257         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1258         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1259         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1260     } elsif ($drive->{interface} eq 'usb') {
1261         die "implement me";
1262         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1263     } else {
1264         die "unsupported interface type";
1265     }
1266
1267     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1268
1269     return $device;
1270 }
1271
1272 sub get_initiator_name {
1273     my $initiator;
1274
1275     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1276     while (defined(my $line = <$fh>)) {
1277         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1278         $initiator = $1;
1279         last;
1280     }
1281     $fh->close();
1282
1283     return $initiator;
1284 }
1285
1286 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1287 sub print_drive_full {
1288     my ($storecfg, $vmid, $drive) = @_;
1289
1290     my $path;
1291     my $volid = $drive->{file};
1292     my $format;
1293
1294     if (drive_is_cdrom($drive)) {
1295         $path = get_iso_path($storecfg, $vmid, $volid);
1296     } else {
1297         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1298         if ($storeid) {
1299             $path = PVE::Storage::path($storecfg, $volid);
1300             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1301             $format = qemu_img_format($scfg, $volname);
1302         } else {
1303             $path = $volid;
1304             $format = "raw";
1305         }
1306    }
1307
1308     my $opts = '';
1309     foreach my $o (@qemu_drive_options) {
1310         next if $o eq 'bootindex';
1311         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1312     }
1313
1314     $opts .= ",format=$format" if $format && !$drive->{format};
1315
1316     foreach my $o (qw(bps bps_rd bps_wr)) {
1317         my $v = $drive->{"m$o"};
1318         $opts .= ",$o=" . int($v*1024*1024) if $v;
1319     }
1320
1321     my $cache_direct = 0;
1322
1323     if (my $cache = $drive->{cache}) {
1324         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1325     } elsif (!drive_is_cdrom($drive)) {
1326         $opts .= ",cache=none";
1327         $cache_direct = 1;
1328     }
1329
1330     # aio native works only with O_DIRECT
1331     if (!$drive->{aio}) {
1332         if($cache_direct) {
1333             $opts .= ",aio=native";
1334         } else {
1335             $opts .= ",aio=threads";
1336         }
1337     }
1338
1339     if (!drive_is_cdrom($drive)) {
1340         my $detectzeroes;
1341         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1342             $detectzeroes = 'off';
1343         } elsif ($drive->{discard}) {
1344             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1345         } else {
1346             # This used to be our default with discard not being specified:
1347             $detectzeroes = 'on';
1348         }
1349         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1350     }
1351
1352     my $pathinfo = $path ? "file=$path," : '';
1353
1354     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1355 }
1356
1357 sub print_netdevice_full {
1358     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1359
1360     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1361
1362     my $device = $net->{model};
1363     if ($net->{model} eq 'virtio') {
1364          $device = 'virtio-net-pci';
1365      };
1366
1367     my $pciaddr = print_pci_addr("$netid", $bridges);
1368     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1369     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1370         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1371         my $vectors = $net->{queues} * 2 + 2;
1372         $tmpstr .= ",vectors=$vectors,mq=on";
1373     }
1374     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1375
1376     if ($use_old_bios_files) {
1377         my $romfile;
1378         if ($device eq 'virtio-net-pci') {
1379             $romfile = 'pxe-virtio.rom';
1380         } elsif ($device eq 'e1000') {
1381             $romfile = 'pxe-e1000.rom';
1382         } elsif ($device eq 'ne2k') {
1383             $romfile = 'pxe-ne2k_pci.rom';
1384         } elsif ($device eq 'pcnet') {
1385             $romfile = 'pxe-pcnet.rom';
1386         } elsif ($device eq 'rtl8139') {
1387             $romfile = 'pxe-rtl8139.rom';
1388         }
1389         $tmpstr .= ",romfile=$romfile" if $romfile;
1390     }
1391
1392     return $tmpstr;
1393 }
1394
1395 sub print_netdev_full {
1396     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1397
1398     my $i = '';
1399     if ($netid =~ m/^net(\d+)$/) {
1400         $i = int($1);
1401     }
1402
1403     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1404
1405     my $ifname = "tap${vmid}i$i";
1406
1407     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1408     die "interface name '$ifname' is too long (max 15 character)\n"
1409         if length($ifname) >= 16;
1410
1411     my $vhostparam = '';
1412     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1413
1414     my $vmname = $conf->{name} || "vm$vmid";
1415
1416     my $netdev = "";
1417     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1418
1419     if ($net->{bridge}) {
1420         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1421     } else {
1422         $netdev = "type=user,id=$netid,hostname=$vmname";
1423     }
1424
1425     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1426
1427     return $netdev;
1428 }
1429
1430 sub drive_is_cdrom {
1431     my ($drive) = @_;
1432
1433     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1434
1435 }
1436
1437 sub parse_numa {
1438     my ($data) = @_;
1439
1440     my $res = {};
1441
1442     foreach my $kvp (split(/,/, $data)) {
1443
1444         if ($kvp =~ m/^memory=(\S+)$/) {
1445             $res->{memory} = $1;
1446         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1447             $res->{policy} = $1;
1448         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1449             $res->{cpus}->{start} = $1;
1450             $res->{cpus}->{end} = $3;
1451         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1452             $res->{hostnodes}->{start} = $1;
1453             $res->{hostnodes}->{end} = $3;
1454         } else {
1455             return undef;
1456         }
1457     }
1458
1459     return $res;
1460 }
1461
1462 sub parse_hostpci {
1463     my ($value) = @_;
1464
1465     return undef if !$value;
1466
1467
1468     my @list = split(/,/, $value);
1469     my $found;
1470
1471     my $res = {};
1472     foreach my $kv (@list) {
1473
1474         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1475             $found = 1;
1476             if(defined($4)){
1477                 push @{$res->{pciid}}, { id => $2 , function => $4};
1478
1479             }else{
1480                 my $pcidevices = lspci($2);
1481                 $res->{pciid} = $pcidevices->{$2};
1482             }
1483         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1484             $res->{rombar} = $1;
1485         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1486             $res->{'x-vga'} = $1;
1487         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1488             $res->{pcie} = 1 if $1 == 1;
1489         } else {
1490             warn "unknown hostpci setting '$kv'\n";
1491         }
1492     }
1493
1494     return undef if !$found;
1495
1496     return $res;
1497 }
1498
1499 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1500 sub parse_net {
1501     my ($data) = @_;
1502
1503     my $res = {};
1504
1505     foreach my $kvp (split(/,/, $data)) {
1506
1507         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1508             my $model = lc($1);
1509             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1510             $res->{model} = $model;
1511             $res->{macaddr} = $mac;
1512         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1513             $res->{bridge} = $1;
1514         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1515             $res->{queues} = $1;
1516         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1517             $res->{rate} = $1;
1518         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1519             $res->{tag} = $1;
1520         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1521             $res->{trunks} = $1;
1522         } elsif ($kvp =~ m/^firewall=([01])$/) {
1523             $res->{firewall} = $1;
1524         } elsif ($kvp =~ m/^link_down=([01])$/) {
1525             $res->{link_down} = $1;
1526         } else {
1527             return undef;
1528         }
1529
1530     }
1531
1532     return undef if !$res->{model};
1533
1534     return $res;
1535 }
1536
1537 sub print_net {
1538     my $net = shift;
1539
1540     my $res = "$net->{model}";
1541     $res .= "=$net->{macaddr}" if $net->{macaddr};
1542     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1543     $res .= ",rate=$net->{rate}" if $net->{rate};
1544     $res .= ",tag=$net->{tag}" if $net->{tag};
1545     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1546     $res .= ",firewall=1" if $net->{firewall};
1547     $res .= ",link_down=1" if $net->{link_down};
1548     $res .= ",queues=$net->{queues}" if $net->{queues};
1549
1550     return $res;
1551 }
1552
1553 sub add_random_macs {
1554     my ($settings) = @_;
1555
1556     foreach my $opt (keys %$settings) {
1557         next if $opt !~ m/^net(\d+)$/;
1558         my $net = parse_net($settings->{$opt});
1559         next if !$net;
1560         $settings->{$opt} = print_net($net);
1561     }
1562 }
1563
1564 sub add_unused_volume {
1565     my ($config, $volid) = @_;
1566
1567     my $key;
1568     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1569         my $test = "unused$ind";
1570         if (my $vid = $config->{$test}) {
1571             return if $vid eq $volid; # do not add duplicates
1572         } else {
1573             $key = $test;
1574         }
1575     }
1576
1577     die "To many unused volume - please delete them first.\n" if !$key;
1578
1579     $config->{$key} = $volid;
1580
1581     return $key;
1582 }
1583
1584 sub vm_is_volid_owner {
1585     my ($storecfg, $vmid, $volid) = @_;
1586
1587     if ($volid !~  m|^/|) {
1588         my ($path, $owner);
1589         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1590         if ($owner && ($owner == $vmid)) {
1591             return 1;
1592         }
1593     }
1594
1595     return undef;
1596 }
1597
1598 sub split_flagged_list {
1599     my $text = shift || '';
1600     $text =~ s/[,;]/ /g;
1601     $text =~ s/^\s+//;
1602     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1603 }
1604
1605 sub join_flagged_list {
1606     my ($how, $lst) = @_;
1607     join $how, map { $lst->{$_} . $_ } keys %$lst;
1608 }
1609
1610 sub vmconfig_delete_pending_option {
1611     my ($conf, $key, $force) = @_;
1612
1613     delete $conf->{pending}->{$key};
1614     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1615     $pending_delete_hash->{$key} = $force ? '!' : '';
1616     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1617 }
1618
1619 sub vmconfig_undelete_pending_option {
1620     my ($conf, $key) = @_;
1621
1622     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1623     delete $pending_delete_hash->{$key};
1624
1625     if (%$pending_delete_hash) {
1626         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1627     } else {
1628         delete $conf->{pending}->{delete};
1629     }
1630 }
1631
1632 sub vmconfig_register_unused_drive {
1633     my ($storecfg, $vmid, $conf, $drive) = @_;
1634
1635     if (!drive_is_cdrom($drive)) {
1636         my $volid = $drive->{file};
1637         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1638             add_unused_volume($conf, $volid, $vmid);
1639         }
1640     }
1641 }
1642
1643 sub vmconfig_cleanup_pending {
1644     my ($conf) = @_;
1645
1646     # remove pending changes when nothing changed
1647     my $changes;
1648     foreach my $opt (keys %{$conf->{pending}}) {
1649         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1650             $changes = 1;
1651             delete $conf->{pending}->{$opt};
1652         }
1653     }
1654
1655     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1656     my $pending_delete_hash = {};
1657     while (my ($opt, $force) = each %$current_delete_hash) {
1658         if (defined($conf->{$opt})) {
1659             $pending_delete_hash->{$opt} = $force;
1660         } else {
1661             $changes = 1;
1662         }
1663     }
1664
1665     if (%$pending_delete_hash) {
1666         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1667     } else {
1668         delete $conf->{pending}->{delete};
1669     }
1670
1671     return $changes;
1672 }
1673
1674 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1675 my $smbios1_desc = {
1676     uuid => {
1677         type => 'string',
1678         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1679         format_description => 'UUID',
1680         optional => 1,
1681     },
1682     version => {
1683         type => 'string',
1684         pattern => '\S+',
1685         format_description => 'str',
1686         optional => 1,
1687     },
1688     serial => {
1689         type => 'string',
1690         pattern => '\S+',
1691         format_description => 'str',
1692         optional => 1,
1693     },
1694     manufacturer => {
1695         type => 'string',
1696         pattern => '\S+',
1697         format_description => 'name',
1698         optional => 1,
1699     },
1700     product => {
1701         type => 'string',
1702         pattern => '\S+',
1703         format_description => 'name',
1704         optional => 1,
1705     },
1706     sku => {
1707         type => 'string',
1708         pattern => '\S+',
1709         format_description => 'str',
1710         optional => 1,
1711     },
1712     family => {
1713         type => 'string',
1714         pattern => '\S+',
1715         format_description => 'str',
1716         optional => 1,
1717     },
1718 };
1719
1720 sub parse_smbios1 {
1721     my ($data) = @_;
1722
1723     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1724     warn $@ if $@;
1725     return $res;
1726 }
1727
1728 sub print_smbios1 {
1729     my ($smbios1) = @_;
1730     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1731 }
1732
1733 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1734
1735 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1736 sub verify_bootdisk {
1737     my ($value, $noerr) = @_;
1738
1739     return $value if valid_drivename($value);
1740
1741     return undef if $noerr;
1742
1743     die "invalid boot disk '$value'\n";
1744 }
1745
1746 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1747 sub verify_numa {
1748     my ($value, $noerr) = @_;
1749
1750     return $value if parse_numa($value);
1751
1752     return undef if $noerr;
1753
1754     die "unable to parse numa options\n";
1755 }
1756
1757 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1758 sub verify_net {
1759     my ($value, $noerr) = @_;
1760
1761     return $value if parse_net($value);
1762
1763     return undef if $noerr;
1764
1765     die "unable to parse network options\n";
1766 }
1767
1768 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1769 sub verify_hostpci {
1770     my ($value, $noerr) = @_;
1771
1772     return $value if parse_hostpci($value);
1773
1774     return undef if $noerr;
1775
1776     die "unable to parse pci id\n";
1777 }
1778
1779 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1780 sub verify_watchdog {
1781     my ($value, $noerr) = @_;
1782
1783     return $value if parse_watchdog($value);
1784
1785     return undef if $noerr;
1786
1787     die "unable to parse watchdog options\n";
1788 }
1789
1790 sub parse_watchdog {
1791     my ($value) = @_;
1792
1793     return undef if !$value;
1794
1795     my $res = {};
1796
1797     foreach my $p (split(/,/, $value)) {
1798         next if $p =~ m/^\s*$/;
1799
1800         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1801             $res->{model} = $2;
1802         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1803             $res->{action} = $2;
1804         } else {
1805             return undef;
1806         }
1807     }
1808
1809     return $res;
1810 }
1811
1812 sub parse_usb_device {
1813     my ($value) = @_;
1814
1815     return undef if !$value;
1816
1817     my @dl = split(/,/, $value);
1818     my $found;
1819
1820     my $res = {};
1821     foreach my $v (@dl) {
1822         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1823             $found = 1;
1824             $res->{vendorid} = $2;
1825             $res->{productid} = $4;
1826         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1827             $found = 1;
1828             $res->{hostbus} = $1;
1829             $res->{hostport} = $2;
1830         } elsif ($v =~ m/^spice$/) {
1831             $found = 1;
1832             $res->{spice} = 1;
1833         } else {
1834             return undef;
1835         }
1836     }
1837     return undef if !$found;
1838
1839     return $res;
1840 }
1841
1842 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1843 sub verify_usb_device {
1844     my ($value, $noerr) = @_;
1845
1846     return $value if parse_usb_device($value);
1847
1848     return undef if $noerr;
1849
1850     die "unable to parse usb device\n";
1851 }
1852
1853 # add JSON properties for create and set function
1854 sub json_config_properties {
1855     my $prop = shift;
1856
1857     foreach my $opt (keys %$confdesc) {
1858         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1859         $prop->{$opt} = $confdesc->{$opt};
1860     }
1861
1862     return $prop;
1863 }
1864
1865 sub check_type {
1866     my ($key, $value) = @_;
1867
1868     die "unknown setting '$key'\n" if !$confdesc->{$key};
1869
1870     my $type = $confdesc->{$key}->{type};
1871
1872     if (!defined($value)) {
1873         die "got undefined value\n";
1874     }
1875
1876     if ($value =~ m/[\n\r]/) {
1877         die "property contains a line feed\n";
1878     }
1879
1880     if ($type eq 'boolean') {
1881         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1882         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1883         die "type check ('boolean') failed - got '$value'\n";
1884     } elsif ($type eq 'integer') {
1885         return int($1) if $value =~ m/^(\d+)$/;
1886         die "type check ('integer') failed - got '$value'\n";
1887     } elsif ($type eq 'number') {
1888         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1889         die "type check ('number') failed - got '$value'\n";
1890     } elsif ($type eq 'string') {
1891         if (my $fmt = $confdesc->{$key}->{format}) {
1892             if ($fmt eq 'pve-qm-drive') {
1893                 # special case - we need to pass $key to parse_drive()
1894                 my $drive = parse_drive($key, $value);
1895                 return $value if $drive;
1896                 die "unable to parse drive options\n";
1897             }
1898             PVE::JSONSchema::check_format($fmt, $value);
1899             return $value;
1900         }
1901         $value =~ s/^\"(.*)\"$/$1/;
1902         return $value;
1903     } else {
1904         die "internal error"
1905     }
1906 }
1907
1908 sub lock_config_full {
1909     my ($vmid, $timeout, $code, @param) = @_;
1910
1911     my $filename = config_file_lock($vmid);
1912
1913     my $res = lock_file($filename, $timeout, $code, @param);
1914
1915     die $@ if $@;
1916
1917     return $res;
1918 }
1919
1920 sub lock_config_mode {
1921     my ($vmid, $timeout, $shared, $code, @param) = @_;
1922
1923     my $filename = config_file_lock($vmid);
1924
1925     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1926
1927     die $@ if $@;
1928
1929     return $res;
1930 }
1931
1932 sub lock_config {
1933     my ($vmid, $code, @param) = @_;
1934
1935     return lock_config_full($vmid, 10, $code, @param);
1936 }
1937
1938 sub cfs_config_path {
1939     my ($vmid, $node) = @_;
1940
1941     $node = $nodename if !$node;
1942     return "nodes/$node/qemu-server/$vmid.conf";
1943 }
1944
1945 sub check_iommu_support{
1946     #fixme : need to check IOMMU support
1947     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1948
1949     my $iommu=1;
1950     return $iommu;
1951
1952 }
1953
1954 sub config_file {
1955     my ($vmid, $node) = @_;
1956
1957     my $cfspath = cfs_config_path($vmid, $node);
1958     return "/etc/pve/$cfspath";
1959 }
1960
1961 sub config_file_lock {
1962     my ($vmid) = @_;
1963
1964     return "$lock_dir/lock-$vmid.conf";
1965 }
1966
1967 sub touch_config {
1968     my ($vmid) = @_;
1969
1970     my $conf = config_file($vmid);
1971     utime undef, undef, $conf;
1972 }
1973
1974 sub destroy_vm {
1975     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1976
1977     my $conffile = config_file($vmid);
1978
1979     my $conf = load_config($vmid);
1980
1981     check_lock($conf) if !$skiplock;
1982
1983     # only remove disks owned by this VM
1984     foreach_drive($conf, sub {
1985         my ($ds, $drive) = @_;
1986
1987         return if drive_is_cdrom($drive);
1988
1989         my $volid = $drive->{file};
1990
1991         return if !$volid || $volid =~ m|^/|;
1992
1993         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1994         return if !$path || !$owner || ($owner != $vmid);
1995
1996         PVE::Storage::vdisk_free($storecfg, $volid);
1997     });
1998
1999     if ($keep_empty_config) {
2000         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2001     } else {
2002         unlink $conffile;
2003     }
2004
2005     # also remove unused disk
2006     eval {
2007         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2008
2009         eval {
2010             PVE::Storage::foreach_volid($dl, sub {
2011                 my ($volid, $sid, $volname, $d) = @_;
2012                 PVE::Storage::vdisk_free($storecfg, $volid);
2013             });
2014         };
2015         warn $@ if $@;
2016
2017     };
2018     warn $@ if $@;
2019 }
2020
2021 sub load_config {
2022     my ($vmid, $node) = @_;
2023
2024     my $cfspath = cfs_config_path($vmid, $node);
2025
2026     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2027
2028     die "no such VM ('$vmid')\n" if !defined($conf);
2029
2030     return $conf;
2031 }
2032
2033 sub parse_vm_config {
2034     my ($filename, $raw) = @_;
2035
2036     return undef if !defined($raw);
2037
2038     my $res = {
2039         digest => Digest::SHA::sha1_hex($raw),
2040         snapshots => {},
2041         pending => {},
2042     };
2043
2044     $filename =~ m|/qemu-server/(\d+)\.conf$|
2045         || die "got strange filename '$filename'";
2046
2047     my $vmid = $1;
2048
2049     my $conf = $res;
2050     my $descr;
2051     my $section = '';
2052
2053     my @lines = split(/\n/, $raw);
2054     foreach my $line (@lines) {
2055         next if $line =~ m/^\s*$/;
2056
2057         if ($line =~ m/^\[PENDING\]\s*$/i) {
2058             $section = 'pending';
2059             if (defined($descr)) {
2060                 $descr =~ s/\s+$//;
2061                 $conf->{description} = $descr;
2062             }
2063             $descr = undef;
2064             $conf = $res->{$section} = {};
2065             next;
2066
2067         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2068             $section = $1;
2069             if (defined($descr)) {
2070                 $descr =~ s/\s+$//;
2071                 $conf->{description} = $descr;
2072             }
2073             $descr = undef;
2074             $conf = $res->{snapshots}->{$section} = {};
2075             next;
2076         }
2077
2078         if ($line =~ m/^\#(.*)\s*$/) {
2079             $descr = '' if !defined($descr);
2080             $descr .= PVE::Tools::decode_text($1) . "\n";
2081             next;
2082         }
2083
2084         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2085             $descr = '' if !defined($descr);
2086             $descr .= PVE::Tools::decode_text($2);
2087         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2088             $conf->{snapstate} = $1;
2089         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2090             my $key = $1;
2091             my $value = $2;
2092             $conf->{$key} = $value;
2093         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2094             my $value = $1;
2095             if ($section eq 'pending') {
2096                 $conf->{delete} = $value; # we parse this later
2097             } else {
2098                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2099             }
2100         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2101             my $key = $1;
2102             my $value = $2;
2103             eval { $value = check_type($key, $value); };
2104             if ($@) {
2105                 warn "vm $vmid - unable to parse value of '$key' - $@";
2106             } else {
2107                 my $fmt = $confdesc->{$key}->{format};
2108                 if ($fmt && $fmt eq 'pve-qm-drive') {
2109                     my $v = parse_drive($key, $value);
2110                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2111                         $v->{file} = $volid;
2112                         $value = print_drive($vmid, $v);
2113                     } else {
2114                         warn "vm $vmid - unable to parse value of '$key'\n";
2115                         next;
2116                     }
2117                 }
2118
2119                 if ($key eq 'cdrom') {
2120                     $conf->{ide2} = $value;
2121                 } else {
2122                     $conf->{$key} = $value;
2123                 }
2124             }
2125         }
2126     }
2127
2128     if (defined($descr)) {
2129         $descr =~ s/\s+$//;
2130         $conf->{description} = $descr;
2131     }
2132     delete $res->{snapstate}; # just to be sure
2133
2134     return $res;
2135 }
2136
2137 sub write_vm_config {
2138     my ($filename, $conf) = @_;
2139
2140     delete $conf->{snapstate}; # just to be sure
2141
2142     if ($conf->{cdrom}) {
2143         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2144         $conf->{ide2} = $conf->{cdrom};
2145         delete $conf->{cdrom};
2146     }
2147
2148     # we do not use 'smp' any longer
2149     if ($conf->{sockets}) {
2150         delete $conf->{smp};
2151     } elsif ($conf->{smp}) {
2152         $conf->{sockets} = $conf->{smp};
2153         delete $conf->{cores};
2154         delete $conf->{smp};
2155     }
2156
2157     my $used_volids = {};
2158
2159     my $cleanup_config = sub {
2160         my ($cref, $pending, $snapname) = @_;
2161
2162         foreach my $key (keys %$cref) {
2163             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2164                 $key eq 'snapstate' || $key eq 'pending';
2165             my $value = $cref->{$key};
2166             if ($key eq 'delete') {
2167                 die "propertry 'delete' is only allowed in [PENDING]\n"
2168                     if !$pending;
2169                 # fixme: check syntax?
2170                 next;
2171             }
2172             eval { $value = check_type($key, $value); };
2173             die "unable to parse value of '$key' - $@" if $@;
2174
2175             $cref->{$key} = $value;
2176
2177             if (!$snapname && valid_drivename($key)) {
2178                 my $drive = parse_drive($key, $value);
2179                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2180             }
2181         }
2182     };
2183
2184     &$cleanup_config($conf);
2185
2186     &$cleanup_config($conf->{pending}, 1);
2187
2188     foreach my $snapname (keys %{$conf->{snapshots}}) {
2189         die "internal error" if $snapname eq 'pending';
2190         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2191     }
2192
2193     # remove 'unusedX' settings if we re-add a volume
2194     foreach my $key (keys %$conf) {
2195         my $value = $conf->{$key};
2196         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2197             delete $conf->{$key};
2198         }
2199     }
2200
2201     my $generate_raw_config = sub {
2202         my ($conf, $pending) = @_;
2203
2204         my $raw = '';
2205
2206         # add description as comment to top of file
2207         if (defined(my $descr = $conf->{description})) {
2208             if ($descr) {
2209                 foreach my $cl (split(/\n/, $descr)) {
2210                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2211                 }
2212             } else {
2213                 $raw .= "#\n" if $pending;
2214             }
2215         }
2216
2217         foreach my $key (sort keys %$conf) {
2218             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2219             $raw .= "$key: $conf->{$key}\n";
2220         }
2221         return $raw;
2222     };
2223
2224     my $raw = &$generate_raw_config($conf);
2225
2226     if (scalar(keys %{$conf->{pending}})){
2227         $raw .= "\n[PENDING]\n";
2228         $raw .= &$generate_raw_config($conf->{pending}, 1);
2229     }
2230
2231     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2232         $raw .= "\n[$snapname]\n";
2233         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2234     }
2235
2236     return $raw;
2237 }
2238
2239 sub update_config_nolock {
2240     my ($vmid, $conf, $skiplock) = @_;
2241
2242     check_lock($conf) if !$skiplock;
2243
2244     my $cfspath = cfs_config_path($vmid);
2245
2246     PVE::Cluster::cfs_write_file($cfspath, $conf);
2247 }
2248
2249 sub update_config {
2250     my ($vmid, $conf, $skiplock) = @_;
2251
2252     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2253 }
2254
2255 sub load_defaults {
2256
2257     my $res = {};
2258
2259     # we use static defaults from our JSON schema configuration
2260     foreach my $key (keys %$confdesc) {
2261         if (defined(my $default = $confdesc->{$key}->{default})) {
2262             $res->{$key} = $default;
2263         }
2264     }
2265
2266     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2267     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2268
2269     return $res;
2270 }
2271
2272 sub config_list {
2273     my $vmlist = PVE::Cluster::get_vmlist();
2274     my $res = {};
2275     return $res if !$vmlist || !$vmlist->{ids};
2276     my $ids = $vmlist->{ids};
2277
2278     foreach my $vmid (keys %$ids) {
2279         my $d = $ids->{$vmid};
2280         next if !$d->{node} || $d->{node} ne $nodename;
2281         next if !$d->{type} || $d->{type} ne 'qemu';
2282         $res->{$vmid}->{exists} = 1;
2283     }
2284     return $res;
2285 }
2286
2287 # test if VM uses local resources (to prevent migration)
2288 sub check_local_resources {
2289     my ($conf, $noerr) = @_;
2290
2291     my $loc_res = 0;
2292
2293     $loc_res = 1 if $conf->{hostusb}; # old syntax
2294     $loc_res = 1 if $conf->{hostpci}; # old syntax
2295
2296     foreach my $k (keys %$conf) {
2297         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2298         # sockets are safe: they will recreated be on the target side post-migrate
2299         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2300         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2301     }
2302
2303     die "VM uses local resources\n" if $loc_res && !$noerr;
2304
2305     return $loc_res;
2306 }
2307
2308 # check if used storages are available on all nodes (use by migrate)
2309 sub check_storage_availability {
2310     my ($storecfg, $conf, $node) = @_;
2311
2312     foreach_drive($conf, sub {
2313         my ($ds, $drive) = @_;
2314
2315         my $volid = $drive->{file};
2316         return if !$volid;
2317
2318         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2319         return if !$sid;
2320
2321         # check if storage is available on both nodes
2322         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2323         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2324    });
2325 }
2326
2327 # list nodes where all VM images are available (used by has_feature API)
2328 sub shared_nodes {
2329     my ($conf, $storecfg) = @_;
2330
2331     my $nodelist = PVE::Cluster::get_nodelist();
2332     my $nodehash = { map { $_ => 1 } @$nodelist };
2333     my $nodename = PVE::INotify::nodename();
2334
2335     foreach_drive($conf, sub {
2336         my ($ds, $drive) = @_;
2337
2338         my $volid = $drive->{file};
2339         return if !$volid;
2340
2341         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2342         if ($storeid) {
2343             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2344             if ($scfg->{disable}) {
2345                 $nodehash = {};
2346             } elsif (my $avail = $scfg->{nodes}) {
2347                 foreach my $node (keys %$nodehash) {
2348                     delete $nodehash->{$node} if !$avail->{$node};
2349                 }
2350             } elsif (!$scfg->{shared}) {
2351                 foreach my $node (keys %$nodehash) {
2352                     delete $nodehash->{$node} if $node ne $nodename
2353                 }
2354             }
2355         }
2356     });
2357
2358     return $nodehash
2359 }
2360
2361 sub check_lock {
2362     my ($conf) = @_;
2363
2364     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2365 }
2366
2367 sub check_cmdline {
2368     my ($pidfile, $pid) = @_;
2369
2370     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2371     if (defined($fh)) {
2372         my $line = <$fh>;
2373         $fh->close;
2374         return undef if !$line;
2375         my @param = split(/\0/, $line);
2376
2377         my $cmd = $param[0];
2378         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2379
2380         for (my $i = 0; $i < scalar (@param); $i++) {
2381             my $p = $param[$i];
2382             next if !$p;
2383             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2384                 my $p = $param[$i+1];
2385                 return 1 if $p && ($p eq $pidfile);
2386                 return undef;
2387             }
2388         }
2389     }
2390     return undef;
2391 }
2392
2393 sub check_running {
2394     my ($vmid, $nocheck, $node) = @_;
2395
2396     my $filename = config_file($vmid, $node);
2397
2398     die "unable to find configuration file for VM $vmid - no such machine\n"
2399         if !$nocheck && ! -f $filename;
2400
2401     my $pidfile = pidfile_name($vmid);
2402
2403     if (my $fd = IO::File->new("<$pidfile")) {
2404         my $st = stat($fd);
2405         my $line = <$fd>;
2406         close($fd);
2407
2408         my $mtime = $st->mtime;
2409         if ($mtime > time()) {
2410             warn "file '$filename' modified in future\n";
2411         }
2412
2413         if ($line =~ m/^(\d+)$/) {
2414             my $pid = $1;
2415             if (check_cmdline($pidfile, $pid)) {
2416                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2417                     return $pid;
2418                 }
2419             }
2420         }
2421     }
2422
2423     return undef;
2424 }
2425
2426 sub vzlist {
2427
2428     my $vzlist = config_list();
2429
2430     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2431
2432     while (defined(my $de = $fd->read)) {
2433         next if $de !~ m/^(\d+)\.pid$/;
2434         my $vmid = $1;
2435         next if !defined($vzlist->{$vmid});
2436         if (my $pid = check_running($vmid)) {
2437             $vzlist->{$vmid}->{pid} = $pid;
2438         }
2439     }
2440
2441     return $vzlist;
2442 }
2443
2444 sub disksize {
2445     my ($storecfg, $conf) = @_;
2446
2447     my $bootdisk = $conf->{bootdisk};
2448     return undef if !$bootdisk;
2449     return undef if !valid_drivename($bootdisk);
2450
2451     return undef if !$conf->{$bootdisk};
2452
2453     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2454     return undef if !defined($drive);
2455
2456     return undef if drive_is_cdrom($drive);
2457
2458     my $volid = $drive->{file};
2459     return undef if !$volid;
2460
2461     return $drive->{size};
2462 }
2463
2464 my $last_proc_pid_stat;
2465
2466 # get VM status information
2467 # This must be fast and should not block ($full == false)
2468 # We only query KVM using QMP if $full == true (this can be slow)
2469 sub vmstatus {
2470     my ($opt_vmid, $full) = @_;
2471
2472     my $res = {};
2473
2474     my $storecfg = PVE::Storage::config();
2475
2476     my $list = vzlist();
2477     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2478
2479     my $cpucount = $cpuinfo->{cpus} || 1;
2480
2481     foreach my $vmid (keys %$list) {
2482         next if $opt_vmid && ($vmid ne $opt_vmid);
2483
2484         my $cfspath = cfs_config_path($vmid);
2485         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2486
2487         my $d = {};
2488         $d->{pid} = $list->{$vmid}->{pid};
2489
2490         # fixme: better status?
2491         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2492
2493         my $size = disksize($storecfg, $conf);
2494         if (defined($size)) {
2495             $d->{disk} = 0; # no info available
2496             $d->{maxdisk} = $size;
2497         } else {
2498             $d->{disk} = 0;
2499             $d->{maxdisk} = 0;
2500         }
2501
2502         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2503         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2504         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2505
2506         $d->{name} = $conf->{name} || "VM $vmid";
2507         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2508
2509         if ($conf->{balloon}) {
2510             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2511             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2512         }
2513
2514         $d->{uptime} = 0;
2515         $d->{cpu} = 0;
2516         $d->{mem} = 0;
2517
2518         $d->{netout} = 0;
2519         $d->{netin} = 0;
2520
2521         $d->{diskread} = 0;
2522         $d->{diskwrite} = 0;
2523
2524         $d->{template} = is_template($conf);
2525
2526         $res->{$vmid} = $d;
2527     }
2528
2529     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2530     foreach my $dev (keys %$netdev) {
2531         next if $dev !~ m/^tap([1-9]\d*)i/;
2532         my $vmid = $1;
2533         my $d = $res->{$vmid};
2534         next if !$d;
2535
2536         $d->{netout} += $netdev->{$dev}->{receive};
2537         $d->{netin} += $netdev->{$dev}->{transmit};
2538
2539         if ($full) {
2540             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2541             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2542         }
2543
2544     }
2545
2546     my $ctime = gettimeofday;
2547
2548     foreach my $vmid (keys %$list) {
2549
2550         my $d = $res->{$vmid};
2551         my $pid = $d->{pid};
2552         next if !$pid;
2553
2554         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2555         next if !$pstat; # not running
2556
2557         my $used = $pstat->{utime} + $pstat->{stime};
2558
2559         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2560
2561         if ($pstat->{vsize}) {
2562             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2563         }
2564
2565         my $old = $last_proc_pid_stat->{$pid};
2566         if (!$old) {
2567             $last_proc_pid_stat->{$pid} = {
2568                 time => $ctime,
2569                 used => $used,
2570                 cpu => 0,
2571             };
2572             next;
2573         }
2574
2575         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2576
2577         if ($dtime > 1000) {
2578             my $dutime = $used -  $old->{used};
2579
2580             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2581             $last_proc_pid_stat->{$pid} = {
2582                 time => $ctime,
2583                 used => $used,
2584                 cpu => $d->{cpu},
2585             };
2586         } else {
2587             $d->{cpu} = $old->{cpu};
2588         }
2589     }
2590
2591     return $res if !$full;
2592
2593     my $qmpclient = PVE::QMPClient->new();
2594
2595     my $ballooncb = sub {
2596         my ($vmid, $resp) = @_;
2597
2598         my $info = $resp->{'return'};
2599         return if !$info->{max_mem};
2600
2601         my $d = $res->{$vmid};
2602
2603         # use memory assigned to VM
2604         $d->{maxmem} = $info->{max_mem};
2605         $d->{balloon} = $info->{actual};
2606
2607         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2608             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2609             $d->{freemem} = $info->{free_mem};
2610         }
2611
2612         $d->{ballooninfo} = $info;
2613     };
2614
2615     my $blockstatscb = sub {
2616         my ($vmid, $resp) = @_;
2617         my $data = $resp->{'return'} || [];
2618         my $totalrdbytes = 0;
2619         my $totalwrbytes = 0;
2620
2621         for my $blockstat (@$data) {
2622             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2623             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2624
2625             $blockstat->{device} =~ s/drive-//;
2626             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2627         }
2628         $res->{$vmid}->{diskread} = $totalrdbytes;
2629         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2630     };
2631
2632     my $statuscb = sub {
2633         my ($vmid, $resp) = @_;
2634
2635         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2636         # this fails if ballon driver is not loaded, so this must be
2637         # the last commnand (following command are aborted if this fails).
2638         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2639
2640         my $status = 'unknown';
2641         if (!defined($status = $resp->{'return'}->{status})) {
2642             warn "unable to get VM status\n";
2643             return;
2644         }
2645
2646         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2647     };
2648
2649     foreach my $vmid (keys %$list) {
2650         next if $opt_vmid && ($vmid ne $opt_vmid);
2651         next if !$res->{$vmid}->{pid}; # not running
2652         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2653     }
2654
2655     $qmpclient->queue_execute(undef, 1);
2656
2657     foreach my $vmid (keys %$list) {
2658         next if $opt_vmid && ($vmid ne $opt_vmid);
2659         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2660     }
2661
2662     return $res;
2663 }
2664
2665 sub foreach_dimm {
2666     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2667
2668     my $dimm_id = 0;
2669     my $current_size = 1024;
2670     my $dimm_size = 512;
2671     return if $current_size == $memory;
2672
2673     for (my $j = 0; $j < 8; $j++) {
2674         for (my $i = 0; $i < 32; $i++) {
2675             my $name = "dimm${dimm_id}";
2676             $dimm_id++;
2677             my $numanode = $i % $sockets;
2678             $current_size += $dimm_size;
2679             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2680             return  $current_size if $current_size >= $memory;
2681         }
2682         $dimm_size *= 2;
2683     }
2684 }
2685
2686 sub foreach_reverse_dimm {
2687     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2688
2689     my $dimm_id = 253;
2690     my $current_size = 4177920;
2691     my $dimm_size = 65536;
2692     return if $current_size == $memory;
2693
2694     for (my $j = 0; $j < 8; $j++) {
2695         for (my $i = 0; $i < 32; $i++) {
2696             my $name = "dimm${dimm_id}";
2697             $dimm_id--;
2698             my $numanode = $i % $sockets;
2699             $current_size -= $dimm_size;
2700             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2701             return  $current_size if $current_size <= $memory;
2702         }
2703         $dimm_size /= 2;
2704     }
2705 }
2706
2707 sub foreach_drive {
2708     my ($conf, $func) = @_;
2709
2710     foreach my $ds (keys %$conf) {
2711         next if !valid_drivename($ds);
2712
2713         my $drive = parse_drive($ds, $conf->{$ds});
2714         next if !$drive;
2715
2716         &$func($ds, $drive);
2717     }
2718 }
2719
2720 sub foreach_volid {
2721     my ($conf, $func) = @_;
2722
2723     my $volhash = {};
2724
2725     my $test_volid = sub {
2726         my ($volid, $is_cdrom) = @_;
2727
2728         return if !$volid;
2729
2730         $volhash->{$volid} = $is_cdrom || 0;
2731     };
2732
2733     foreach_drive($conf, sub {
2734         my ($ds, $drive) = @_;
2735         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2736     });
2737
2738     foreach my $snapname (keys %{$conf->{snapshots}}) {
2739         my $snap = $conf->{snapshots}->{$snapname};
2740         &$test_volid($snap->{vmstate}, 0);
2741         foreach_drive($snap, sub {
2742             my ($ds, $drive) = @_;
2743             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2744         });
2745     }
2746
2747     foreach my $volid (keys %$volhash) {
2748         &$func($volid, $volhash->{$volid});
2749     }
2750 }
2751
2752 sub vga_conf_has_spice {
2753     my ($vga) = @_;
2754
2755     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2756
2757     return $1 || 1;
2758 }
2759
2760 sub config_to_command {
2761     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2762
2763     my $cmd = [];
2764     my $globalFlags = [];
2765     my $machineFlags = [];
2766     my $rtcFlags = [];
2767     my $cpuFlags = [];
2768     my $devices = [];
2769     my $pciaddr = '';
2770     my $bridges = {};
2771     my $kvmver = kvm_user_version();
2772     my $vernum = 0; # unknown
2773     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2774         $vernum = $1*1000000+$2*1000;
2775     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2776         $vernum = $1*1000000+$2*1000+$3;
2777     }
2778
2779     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2780
2781     my $have_ovz = -f '/proc/vz/vestat';
2782
2783     my $q35 = machine_type_is_q35($conf);
2784     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2785     my $machine_type = $forcemachine || $conf->{machine};
2786     my $use_old_bios_files = undef;
2787     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2788
2789     my $cpuunits = defined($conf->{cpuunits}) ?
2790             $conf->{cpuunits} : $defaults->{cpuunits};
2791
2792     push @$cmd, '/usr/bin/systemd-run';
2793     push @$cmd, '--scope';
2794     push @$cmd, '--slice', "qemu";
2795     push @$cmd, '--unit', $vmid;
2796     # set KillMode=none, so that systemd don't kill those scopes
2797     # at shutdown (pve-manager service should stop the VMs instead)
2798     push @$cmd, '-p', "KillMode=none";
2799     push @$cmd, '-p', "CPUShares=$cpuunits";
2800     if ($conf->{cpulimit}) {
2801         my $cpulimit = int($conf->{cpulimit} * 100);
2802         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2803     }
2804
2805     push @$cmd, '/usr/bin/kvm';
2806
2807     push @$cmd, '-id', $vmid;
2808
2809     my $use_virtio = 0;
2810
2811     my $qmpsocket = qmp_socket($vmid);
2812     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2813     push @$cmd, '-mon', "chardev=qmp,mode=control";
2814
2815     my $socket = vnc_socket($vmid);
2816     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2817
2818     push @$cmd, '-pidfile' , pidfile_name($vmid);
2819
2820     push @$cmd, '-daemonize';
2821
2822     if ($conf->{smbios1}) {
2823         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2824     }
2825
2826     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2827         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2828         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2829         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2830         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2831         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2832         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2833     }
2834
2835     if ($q35) {
2836         # the q35 chipset support native usb2, so we enable usb controller
2837         # by default for this machine type
2838         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2839     } else {
2840         $pciaddr = print_pci_addr("piix3", $bridges);
2841         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2842
2843         my $use_usb2 = 0;
2844         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2845             next if !$conf->{"usb$i"};
2846             $use_usb2 = 1;
2847         }
2848         # include usb device config
2849         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2850     }
2851
2852     my $vga = $conf->{vga};
2853
2854     my $qxlnum = vga_conf_has_spice($vga);
2855     $vga = 'qxl' if $qxlnum;
2856
2857     if (!$vga) {
2858         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2859                                 $conf->{ostype} eq 'win7' ||
2860                                 $conf->{ostype} eq 'w2k8')) {
2861             $vga = 'std';
2862         } else {
2863             $vga = 'cirrus';
2864         }
2865     }
2866
2867     # enable absolute mouse coordinates (needed by vnc)
2868     my $tablet;
2869     if (defined($conf->{tablet})) {
2870         $tablet = $conf->{tablet};
2871     } else {
2872         $tablet = $defaults->{tablet};
2873         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2874         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2875     }
2876
2877     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2878
2879     my $kvm_off = 0;
2880     my $nohyperv;
2881     # host pci devices
2882     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2883         my $d = parse_hostpci($conf->{"hostpci$i"});
2884         next if !$d;
2885
2886         my $pcie = $d->{pcie};
2887         if($pcie){
2888             die "q35 machine model is not enabled" if !$q35;
2889             $pciaddr = print_pcie_addr("hostpci$i");
2890         }else{
2891             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2892         }
2893
2894         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2895         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2896         if ($xvga && $xvga ne '') {
2897             $kvm_off = 1;
2898             $vga = 'none';
2899             $nohyperv = 1;
2900             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2901                 $xvga = "";
2902             }
2903         }
2904         my $pcidevices = $d->{pciid};
2905         my $multifunction = 1 if @$pcidevices > 1;
2906
2907         my $j=0;
2908         foreach my $pcidevice (@$pcidevices) {
2909
2910             my $id = "hostpci$i";
2911             $id .= ".$j" if $multifunction;
2912             my $addr = $pciaddr;
2913             $addr .= ".$j" if $multifunction;
2914             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2915
2916             if($j == 0){
2917                 $devicestr .= "$rombar$xvga";
2918                 $devicestr .= ",multifunction=on" if $multifunction;
2919             }
2920
2921             push @$devices, '-device', $devicestr;
2922             $j++;
2923         }
2924     }
2925
2926     # usb devices
2927     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2928         my $d = parse_usb_device($conf->{"usb$i"});
2929         next if !$d;
2930         if ($d->{vendorid} && $d->{productid}) {
2931             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2932         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2933             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2934         } elsif ($d->{spice}) {
2935             # usb redir support for spice
2936             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2937             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2938         }
2939     }
2940
2941     # serial devices
2942     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2943         if (my $path = $conf->{"serial$i"}) {
2944             if ($path eq 'socket') {
2945                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2946                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2947                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2948             } else {
2949                 die "no such serial device\n" if ! -c $path;
2950                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2951                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2952             }
2953         }
2954     }
2955
2956     # parallel devices
2957     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2958         if (my $path = $conf->{"parallel$i"}) {
2959             die "no such parallel device\n" if ! -c $path;
2960             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2961             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2962             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2963         }
2964     }
2965
2966     my $vmname = $conf->{name} || "vm$vmid";
2967
2968     push @$cmd, '-name', $vmname;
2969
2970     my $sockets = 1;
2971     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2972     $sockets = $conf->{sockets} if  $conf->{sockets};
2973
2974     my $cores = $conf->{cores} || 1;
2975
2976     my $maxcpus = $sockets * $cores;
2977
2978     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2979
2980     my $allowed_vcpus = $cpuinfo->{cpus};
2981
2982     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2983         if ($allowed_vcpus < $maxcpus);
2984
2985     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2986
2987     push @$cmd, '-nodefaults';
2988
2989     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2990
2991     my $bootindex_hash = {};
2992     my $i = 1;
2993     foreach my $o (split(//, $bootorder)) {
2994         $bootindex_hash->{$o} = $i*100;
2995         $i++;
2996     }
2997
2998     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2999
3000     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3001
3002     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3003
3004     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3005
3006     # time drift fix
3007     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3008
3009     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3010     my $useLocaltime = $conf->{localtime};
3011
3012     if (my $ost = $conf->{ostype}) {
3013         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3014
3015         if ($ost =~ m/^w/) { # windows
3016             $useLocaltime = 1 if !defined($conf->{localtime});
3017
3018             # use time drift fix when acpi is enabled
3019             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3020                 $tdf = 1 if !defined($conf->{tdf});
3021             }
3022         }
3023
3024         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
3025             $ost eq 'wvista') {
3026             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3027             push @$cmd, '-no-hpet';
3028             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3029                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
3030                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
3031                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
3032
3033             } else {
3034                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
3035             }
3036         }
3037
3038         if ($ost eq 'win7' || $ost eq 'win8') {
3039             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
3040         }
3041     }
3042
3043     push @$rtcFlags, 'driftfix=slew' if $tdf;
3044
3045     if ($nokvm) {
3046         push @$machineFlags, 'accel=tcg';
3047     } else {
3048         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3049     }
3050
3051     if ($machine_type) {
3052         push @$machineFlags, "type=${machine_type}";
3053     }
3054
3055     if ($conf->{startdate}) {
3056         push @$rtcFlags, "base=$conf->{startdate}";
3057     } elsif ($useLocaltime) {
3058         push @$rtcFlags, 'base=localtime';
3059     }
3060
3061     my $cpu = $nokvm ? "qemu64" : "kvm64";
3062     if (my $cputype = $conf->{cpu}) {
3063         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3064             or die "Cannot parse cpu description: $cputype\n";
3065         $cpu = $cpuconf->{cputype};
3066         $kvm_off = 1 if $cpuconf->{hidden};
3067     }
3068
3069     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3070
3071     push @$cpuFlags , '-x2apic'
3072         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3073
3074     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3075
3076     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3077
3078     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3079
3080         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3081         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3082     }
3083
3084     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3085
3086     push @$cpuFlags, 'kvm=off' if $kvm_off;
3087
3088     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3089
3090     push @$cmd, '-cpu', $cpu;
3091
3092     my $memory = $conf->{memory} || $defaults->{memory};
3093     my $static_memory = 0;
3094     my $dimm_memory = 0;
3095
3096     if ($hotplug_features->{memory}) {
3097         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3098         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3099         $static_memory = $STATICMEM;
3100         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3101         $dimm_memory = $memory - $static_memory;
3102         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3103
3104     } else {
3105
3106         $static_memory = $memory;
3107         push @$cmd, '-m', $static_memory;
3108     }
3109
3110     if ($conf->{numa}) {
3111
3112         my $numa_totalmemory = undef;
3113         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3114             next if !$conf->{"numa$i"};
3115             my $numa = parse_numa($conf->{"numa$i"});
3116             next if !$numa;
3117             # memory
3118             die "missing numa node$i memory value\n" if !$numa->{memory};
3119             my $numa_memory = $numa->{memory};
3120             $numa_totalmemory += $numa_memory;
3121             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3122
3123             # cpus
3124             my $cpus_start = $numa->{cpus}->{start};
3125             die "missing numa node$i cpus\n" if !defined($cpus_start);
3126             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3127             my $cpus = $cpus_start;
3128             if (defined($cpus_end)) {
3129                 $cpus .= "-$cpus_end";
3130                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3131             }
3132
3133             # hostnodes
3134             my $hostnodes_start = $numa->{hostnodes}->{start};
3135             if (defined($hostnodes_start)) {
3136                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3137                 my $hostnodes = $hostnodes_start;
3138                 if (defined($hostnodes_end)) {
3139                     $hostnodes .= "-$hostnodes_end";
3140                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3141                 }
3142
3143                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3144                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3145                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3146                 }
3147
3148                 # policy
3149                 my $policy = $numa->{policy};
3150                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3151                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3152             }
3153
3154             push @$cmd, '-object', $numa_object;
3155             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3156         }
3157
3158         die "total memory for NUMA nodes must be equal to vm static memory\n"
3159             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3160
3161         #if no custom tology, we split memory and cores across numa nodes
3162         if(!$numa_totalmemory) {
3163
3164             my $numa_memory = ($static_memory / $sockets) . "M";
3165
3166             for (my $i = 0; $i < $sockets; $i++)  {
3167
3168                 my $cpustart = ($cores * $i);
3169                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3170                 my $cpus = $cpustart;
3171                 $cpus .= "-$cpuend" if $cpuend;
3172
3173                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3174                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3175             }
3176         }
3177     }
3178
3179     if ($hotplug_features->{memory}) {
3180         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3181             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3182             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3183             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3184
3185             #if dimm_memory is not aligned to dimm map
3186             if($current_size > $memory) {
3187                  $conf->{memory} = $current_size;
3188                  update_config_nolock($vmid, $conf, 1);
3189             }
3190         });
3191     }
3192
3193     push @$cmd, '-S' if $conf->{freeze};
3194
3195     # set keyboard layout
3196     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3197     push @$cmd, '-k', $kb if $kb;
3198
3199     # enable sound
3200     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3201     #push @$cmd, '-soundhw', 'es1370';
3202     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3203
3204     if($conf->{agent}) {
3205         my $qgasocket = qmp_socket($vmid, 1);
3206         my $pciaddr = print_pci_addr("qga0", $bridges);
3207         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3208         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3209         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3210     }
3211
3212     my $spice_port;
3213
3214     if ($qxlnum) {
3215         if ($qxlnum > 1) {
3216             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3217                 for(my $i = 1; $i < $qxlnum; $i++){
3218                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3219                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3220                 }
3221             } else {
3222                 # assume other OS works like Linux
3223                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3224                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3225             }
3226         }
3227
3228         my $pciaddr = print_pci_addr("spice", $bridges);
3229
3230         my $nodename = PVE::INotify::nodename();
3231         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3232         $spice_port = PVE::Tools::next_spice_port($pfamily);
3233
3234         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3235
3236         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3237         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3238         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3239     }
3240
3241     # enable balloon by default, unless explicitly disabled
3242     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3243         $pciaddr = print_pci_addr("balloon0", $bridges);
3244         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3245     }
3246
3247     if ($conf->{watchdog}) {
3248         my $wdopts = parse_watchdog($conf->{watchdog});
3249         $pciaddr = print_pci_addr("watchdog", $bridges);
3250         my $watchdog = $wdopts->{model} || 'i6300esb';
3251         push @$devices, '-device', "$watchdog$pciaddr";
3252         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3253     }
3254
3255     my $vollist = [];
3256     my $scsicontroller = {};
3257     my $ahcicontroller = {};
3258     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3259
3260     # Add iscsi initiator name if available
3261     if (my $initiator = get_initiator_name()) {
3262         push @$devices, '-iscsi', "initiator-name=$initiator";
3263     }
3264
3265     foreach_drive($conf, sub {
3266         my ($ds, $drive) = @_;
3267
3268         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3269             push @$vollist, $drive->{file};
3270         }
3271
3272         $use_virtio = 1 if $ds =~ m/^virtio/;
3273
3274         if (drive_is_cdrom ($drive)) {
3275             if ($bootindex_hash->{d}) {
3276                 $drive->{bootindex} = $bootindex_hash->{d};
3277                 $bootindex_hash->{d} += 1;
3278             }
3279         } else {
3280             if ($bootindex_hash->{c}) {
3281                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3282                 $bootindex_hash->{c} += 1;
3283             }
3284         }
3285
3286         if($drive->{interface} eq 'virtio'){
3287            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3288         }
3289
3290         if ($drive->{interface} eq 'scsi') {
3291
3292             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3293
3294             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3295             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3296
3297             my $iothread = '';
3298             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3299                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3300                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3301             }
3302
3303             my $queues = '';
3304             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3305                 $queues = ",num_queues=$drive->{queues}";
3306             }
3307
3308             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3309             $scsicontroller->{$controller}=1;
3310         }
3311
3312         if ($drive->{interface} eq 'sata') {
3313            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3314            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3315            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3316            $ahcicontroller->{$controller}=1;
3317         }
3318
3319         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3320         push @$devices, '-drive',$drive_cmd;
3321         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3322     });
3323
3324     for (my $i = 0; $i < $MAX_NETS; $i++) {
3325          next if !$conf->{"net$i"};
3326          my $d = parse_net($conf->{"net$i"});
3327          next if !$d;
3328
3329          $use_virtio = 1 if $d->{model} eq 'virtio';
3330
3331          if ($bootindex_hash->{n}) {
3332             $d->{bootindex} = $bootindex_hash->{n};
3333             $bootindex_hash->{n} += 1;
3334          }
3335
3336          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3337          push @$devices, '-netdev', $netdevfull;
3338
3339          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3340          push @$devices, '-device', $netdevicefull;
3341     }
3342
3343     if (!$q35) {
3344         # add pci bridges
3345         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3346            $bridges->{1} = 1;
3347            $bridges->{2} = 1;
3348         }
3349
3350         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3351
3352         while (my ($k, $v) = each %$bridges) {
3353             $pciaddr = print_pci_addr("pci.$k");
3354             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3355         }
3356     }
3357
3358     # add custom args
3359     if ($conf->{args}) {
3360         my $aa = PVE::Tools::split_args($conf->{args});
3361         push @$cmd, @$aa;
3362     }
3363
3364     push @$cmd, @$devices;
3365     push @$cmd, '-rtc', join(',', @$rtcFlags)
3366         if scalar(@$rtcFlags);
3367     push @$cmd, '-machine', join(',', @$machineFlags)
3368         if scalar(@$machineFlags);
3369     push @$cmd, '-global', join(',', @$globalFlags)
3370         if scalar(@$globalFlags);
3371
3372     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3373 }
3374
3375 sub vnc_socket {
3376     my ($vmid) = @_;
3377     return "${var_run_tmpdir}/$vmid.vnc";
3378 }
3379
3380 sub spice_port {
3381     my ($vmid) = @_;
3382
3383     my $res = vm_mon_cmd($vmid, 'query-spice');
3384
3385     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3386 }
3387
3388 sub qmp_socket {
3389     my ($vmid, $qga) = @_;
3390     my $sockettype = $qga ? 'qga' : 'qmp';
3391     return "${var_run_tmpdir}/$vmid.$sockettype";
3392 }
3393
3394 sub pidfile_name {
3395     my ($vmid) = @_;
3396     return "${var_run_tmpdir}/$vmid.pid";
3397 }
3398
3399 sub vm_devices_list {
3400     my ($vmid) = @_;
3401
3402     my $res = vm_mon_cmd($vmid, 'query-pci');
3403     my $devices = {};
3404     foreach my $pcibus (@$res) {
3405         foreach my $device (@{$pcibus->{devices}}) {
3406             next if !$device->{'qdev_id'};
3407             if ($device->{'pci_bridge'}) {
3408                 $devices->{$device->{'qdev_id'}} = 1;
3409                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3410                     next if !$bridge_device->{'qdev_id'};
3411                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3412                     $devices->{$device->{'qdev_id'}}++;
3413                 }
3414             } else {
3415                 $devices->{$device->{'qdev_id'}} = 1;
3416             }
3417         }
3418     }
3419
3420     my $resblock = vm_mon_cmd($vmid, 'query-block');
3421     foreach my $block (@$resblock) {
3422         if($block->{device} =~ m/^drive-(\S+)/){
3423                 $devices->{$1} = 1;
3424         }
3425     }
3426
3427     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3428     foreach my $mice (@$resmice) {
3429         if ($mice->{name} eq 'QEMU HID Tablet') {
3430             $devices->{tablet} = 1;
3431             last;
3432         }
3433     }
3434
3435     return $devices;
3436 }
3437
3438 sub vm_deviceplug {
3439     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3440
3441     my $q35 = machine_type_is_q35($conf);
3442
3443     my $devices_list = vm_devices_list($vmid);
3444     return 1 if defined($devices_list->{$deviceid});
3445
3446     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3447
3448     if ($deviceid eq 'tablet') {
3449
3450         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3451
3452     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3453
3454         qemu_iothread_add($vmid, $deviceid, $device);
3455
3456         qemu_driveadd($storecfg, $vmid, $device);
3457         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3458
3459         qemu_deviceadd($vmid, $devicefull);
3460         eval { qemu_deviceaddverify($vmid, $deviceid); };
3461         if (my $err = $@) {
3462             eval { qemu_drivedel($vmid, $deviceid); };
3463             warn $@ if $@;
3464             die $err;
3465         }
3466
3467     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3468
3469
3470         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3471         my $pciaddr = print_pci_addr($deviceid);
3472         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3473
3474         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3475
3476         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3477             qemu_iothread_add($vmid, $deviceid, $device);
3478             $devicefull .= ",iothread=iothread-$deviceid";
3479         }
3480
3481         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3482             $devicefull .= ",num_queues=$device->{queues}";
3483         }
3484
3485         qemu_deviceadd($vmid, $devicefull);
3486         qemu_deviceaddverify($vmid, $deviceid);
3487
3488     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3489
3490         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3491         qemu_driveadd($storecfg, $vmid, $device);
3492
3493         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3494         eval { qemu_deviceadd($vmid, $devicefull); };
3495         if (my $err = $@) {
3496             eval { qemu_drivedel($vmid, $deviceid); };
3497             warn $@ if $@;
3498             die $err;
3499         }
3500
3501     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3502
3503         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3504
3505         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3506         my $use_old_bios_files = undef;
3507         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3508
3509         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3510         qemu_deviceadd($vmid, $netdevicefull);
3511         eval { qemu_deviceaddverify($vmid, $deviceid); };
3512         if (my $err = $@) {
3513             eval { qemu_netdevdel($vmid, $deviceid); };
3514             warn $@ if $@;
3515             die $err;
3516         }
3517
3518     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3519
3520         my $bridgeid = $2;
3521         my $pciaddr = print_pci_addr($deviceid);
3522         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3523
3524         qemu_deviceadd($vmid, $devicefull);
3525         qemu_deviceaddverify($vmid, $deviceid);
3526
3527     } else {
3528         die "can't hotplug device '$deviceid'\n";
3529     }
3530
3531     return 1;
3532 }
3533
3534 # fixme: this should raise exceptions on error!
3535 sub vm_deviceunplug {
3536     my ($vmid, $conf, $deviceid) = @_;
3537
3538     my $devices_list = vm_devices_list($vmid);
3539     return 1 if !defined($devices_list->{$deviceid});
3540
3541     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3542
3543     if ($deviceid eq 'tablet') {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546
3547     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3548
3549         qemu_devicedel($vmid, $deviceid);
3550         qemu_devicedelverify($vmid, $deviceid);
3551         qemu_drivedel($vmid, $deviceid);
3552         qemu_iothread_del($conf, $vmid, $deviceid);
3553
3554     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3555
3556         qemu_devicedel($vmid, $deviceid);
3557         qemu_devicedelverify($vmid, $deviceid);
3558         qemu_iothread_del($conf, $vmid, $deviceid);
3559
3560     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3561
3562         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3563         my $device = parse_drive($deviceid, $conf->{$deviceid});
3564         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3565
3566         qemu_devicedel($vmid, $deviceid);
3567         qemu_drivedel($vmid, $deviceid);
3568         qemu_deletescsihw($conf, $vmid, $deviceid);
3569
3570     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3571
3572         qemu_devicedel($vmid, $deviceid);
3573         qemu_devicedelverify($vmid, $deviceid);
3574         qemu_netdevdel($vmid, $deviceid);
3575
3576     } else {
3577         die "can't unplug device '$deviceid'\n";
3578     }
3579
3580     return 1;
3581 }
3582
3583 sub qemu_deviceadd {
3584     my ($vmid, $devicefull) = @_;
3585
3586     $devicefull = "driver=".$devicefull;
3587     my %options =  split(/[=,]/, $devicefull);
3588
3589     vm_mon_cmd($vmid, "device_add" , %options);
3590 }
3591
3592 sub qemu_devicedel {
3593     my ($vmid, $deviceid) = @_;
3594
3595     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3596 }
3597
3598 sub qemu_iothread_add {
3599     my($vmid, $deviceid, $device) = @_;
3600
3601     if ($device->{iothread}) {
3602         my $iothreads = vm_iothreads_list($vmid);
3603         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3604     }
3605 }
3606
3607 sub qemu_iothread_del {
3608     my($conf, $vmid, $deviceid) = @_;
3609
3610     my $device = parse_drive($deviceid, $conf->{$deviceid});
3611     if ($device->{iothread}) {
3612         my $iothreads = vm_iothreads_list($vmid);
3613         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3614     }
3615 }
3616
3617 sub qemu_objectadd {
3618     my($vmid, $objectid, $qomtype) = @_;
3619
3620     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3621
3622     return 1;
3623 }
3624
3625 sub qemu_objectdel {
3626     my($vmid, $objectid) = @_;
3627
3628     vm_mon_cmd($vmid, "object-del", id => $objectid);
3629
3630     return 1;
3631 }
3632
3633 sub qemu_driveadd {
3634     my ($storecfg, $vmid, $device) = @_;
3635
3636     my $drive = print_drive_full($storecfg, $vmid, $device);
3637     $drive =~ s/\\/\\\\/g;
3638     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3639
3640     # If the command succeeds qemu prints: "OK"
3641     return 1 if $ret =~ m/OK/s;
3642
3643     die "adding drive failed: $ret\n";
3644 }
3645
3646 sub qemu_drivedel {
3647     my($vmid, $deviceid) = @_;
3648
3649     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3650     $ret =~ s/^\s+//;
3651
3652     return 1 if $ret eq "";
3653
3654     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3655     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3656
3657     die "deleting drive $deviceid failed : $ret\n";
3658 }
3659
3660 sub qemu_deviceaddverify {
3661     my ($vmid, $deviceid) = @_;
3662
3663     for (my $i = 0; $i <= 5; $i++) {
3664          my $devices_list = vm_devices_list($vmid);
3665          return 1 if defined($devices_list->{$deviceid});
3666          sleep 1;
3667     }
3668
3669     die "error on hotplug device '$deviceid'\n";
3670 }
3671
3672
3673 sub qemu_devicedelverify {
3674     my ($vmid, $deviceid) = @_;
3675
3676     # need to verify that the device is correctly removed as device_del
3677     # is async and empty return is not reliable
3678
3679     for (my $i = 0; $i <= 5; $i++) {
3680          my $devices_list = vm_devices_list($vmid);
3681          return 1 if !defined($devices_list->{$deviceid});
3682          sleep 1;
3683     }
3684
3685     die "error on hot-unplugging device '$deviceid'\n";
3686 }
3687
3688 sub qemu_findorcreatescsihw {
3689     my ($storecfg, $conf, $vmid, $device) = @_;
3690
3691     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3692
3693     my $scsihwid="$controller_prefix$controller";
3694     my $devices_list = vm_devices_list($vmid);
3695
3696     if(!defined($devices_list->{$scsihwid})) {
3697         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3698     }
3699
3700     return 1;
3701 }
3702
3703 sub qemu_deletescsihw {
3704     my ($conf, $vmid, $opt) = @_;
3705
3706     my $device = parse_drive($opt, $conf->{$opt});
3707
3708     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3709         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3710         return 1;
3711     }
3712
3713     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3714
3715     my $devices_list = vm_devices_list($vmid);
3716     foreach my $opt (keys %{$devices_list}) {
3717         if (PVE::QemuServer::valid_drivename($opt)) {
3718             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3719             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3720                 return 1;
3721             }
3722         }
3723     }
3724
3725     my $scsihwid="scsihw$controller";
3726
3727     vm_deviceunplug($vmid, $conf, $scsihwid);
3728
3729     return 1;
3730 }
3731
3732 sub qemu_add_pci_bridge {
3733     my ($storecfg, $conf, $vmid, $device) = @_;
3734
3735     my $bridges = {};
3736
3737     my $bridgeid;
3738
3739     print_pci_addr($device, $bridges);
3740
3741     while (my ($k, $v) = each %$bridges) {
3742         $bridgeid = $k;
3743     }
3744     return 1 if !defined($bridgeid) || $bridgeid < 1;
3745
3746     my $bridge = "pci.$bridgeid";
3747     my $devices_list = vm_devices_list($vmid);
3748
3749     if (!defined($devices_list->{$bridge})) {
3750         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3751     }
3752
3753     return 1;
3754 }
3755
3756 sub qemu_set_link_status {
3757     my ($vmid, $device, $up) = @_;
3758
3759     vm_mon_cmd($vmid, "set_link", name => $device,
3760                up => $up ? JSON::true : JSON::false);
3761 }
3762
3763 sub qemu_netdevadd {
3764     my ($vmid, $conf, $device, $deviceid) = @_;
3765
3766     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3767     my %options =  split(/[=,]/, $netdev);
3768
3769     vm_mon_cmd($vmid, "netdev_add",  %options);
3770     return 1;
3771 }
3772
3773 sub qemu_netdevdel {
3774     my ($vmid, $deviceid) = @_;
3775
3776     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3777 }
3778
3779 sub qemu_cpu_hotplug {
3780     my ($vmid, $conf, $vcpus) = @_;
3781
3782     my $sockets = 1;
3783     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3784     $sockets = $conf->{sockets} if  $conf->{sockets};
3785     my $cores = $conf->{cores} || 1;
3786     my $maxcpus = $sockets * $cores;
3787
3788     $vcpus = $maxcpus if !$vcpus;
3789
3790     die "you can't add more vcpus than maxcpus\n"
3791         if $vcpus > $maxcpus;
3792
3793     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3794     die "online cpu unplug is not yet possible\n"
3795         if $vcpus < $currentvcpus;
3796
3797     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3798     die "vcpus in running vm is different than configuration\n"
3799         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3800
3801     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3802         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3803     }
3804 }
3805
3806 sub qemu_memory_hotplug {
3807     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3808
3809     return $value if !check_running($vmid);
3810
3811     my $memory = $conf->{memory} || $defaults->{memory};
3812     $value = $defaults->{memory} if !$value;
3813     return $value if $value == $memory;
3814
3815     my $static_memory = $STATICMEM;
3816     my $dimm_memory = $memory - $static_memory;
3817
3818     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3819     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3820
3821
3822     my $sockets = 1;
3823     $sockets = $conf->{sockets} if $conf->{sockets};
3824
3825     if($value > $memory) {
3826
3827         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3828             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3829
3830                 return if $current_size <= $conf->{memory};
3831
3832                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3833                 if (my $err = $@) {
3834                     eval { qemu_objectdel($vmid, "mem-$name"); };
3835                     die $err;
3836                 }
3837
3838                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3839                 if (my $err = $@) {
3840                     eval { qemu_objectdel($vmid, "mem-$name"); };
3841                     die $err;
3842                 }
3843                 #update conf after each succesful module hotplug
3844                 $conf->{memory} = $current_size;
3845                 update_config_nolock($vmid, $conf, 1);
3846         });
3847
3848     } else {
3849
3850         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3851             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3852
3853                 return if $current_size >= $conf->{memory};
3854                 print "try to unplug memory dimm $name\n";
3855
3856                 my $retry = 0;
3857                 while (1) {
3858                     eval { qemu_devicedel($vmid, $name) };
3859                     sleep 3;
3860                     my $dimm_list = qemu_dimm_list($vmid);
3861                     last if !$dimm_list->{$name};
3862                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3863                     $retry++;
3864                 }
3865
3866                 #update conf after each succesful module unplug
3867                 $conf->{memory} = $current_size;
3868
3869                 eval { qemu_objectdel($vmid, "mem-$name"); };
3870                 update_config_nolock($vmid, $conf, 1);
3871         });
3872     }
3873 }
3874
3875 sub qemu_dimm_list {
3876     my ($vmid) = @_;
3877
3878     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3879     my $dimms = {};
3880
3881     foreach my $dimm (@$dimmarray) {
3882
3883         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3884         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3885         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3886         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3887         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3888     }
3889     return $dimms;
3890 }
3891
3892 sub qemu_block_set_io_throttle {
3893     my ($vmid, $deviceid,
3894         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3895         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3896
3897     return if !check_running($vmid) ;
3898
3899     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3900         bps => int($bps),
3901         bps_rd => int($bps_rd),
3902         bps_wr => int($bps_wr),
3903         iops => int($iops),
3904         iops_rd => int($iops_rd),
3905         iops_wr => int($iops_wr),
3906         bps_max => int($bps_max),
3907         bps_rd_max => int($bps_rd_max),
3908         bps_wr_max => int($bps_wr_max),
3909         iops_max => int($iops_max),
3910         iops_rd_max => int($iops_rd_max),
3911         iops_wr_max => int($iops_wr_max)
3912     );
3913
3914 }
3915
3916 # old code, only used to shutdown old VM after update
3917 sub __read_avail {
3918     my ($fh, $timeout) = @_;
3919
3920     my $sel = new IO::Select;
3921     $sel->add($fh);
3922
3923     my $res = '';
3924     my $buf;
3925
3926     my @ready;
3927     while (scalar (@ready = $sel->can_read($timeout))) {
3928         my $count;
3929         if ($count = $fh->sysread($buf, 8192)) {
3930             if ($buf =~ /^(.*)\(qemu\) $/s) {
3931                 $res .= $1;
3932                 last;
3933             } else {
3934                 $res .= $buf;
3935             }
3936         } else {
3937             if (!defined($count)) {
3938                 die "$!\n";
3939             }
3940             last;
3941         }
3942     }
3943
3944     die "monitor read timeout\n" if !scalar(@ready);
3945
3946     return $res;
3947 }
3948
3949 # old code, only used to shutdown old VM after update
3950 sub vm_monitor_command {
3951     my ($vmid, $cmdstr, $nocheck) = @_;
3952
3953     my $res;
3954
3955     eval {
3956         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3957
3958         my $sname = "${var_run_tmpdir}/$vmid.mon";
3959
3960         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3961             die "unable to connect to VM $vmid socket - $!\n";
3962
3963         my $timeout = 3;
3964
3965         # hack: migrate sometime blocks the monitor (when migrate_downtime
3966         # is set)
3967         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3968             $timeout = 60*60; # 1 hour
3969         }
3970
3971         # read banner;
3972         my $data = __read_avail($sock, $timeout);
3973
3974         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3975             die "got unexpected qemu monitor banner\n";
3976         }
3977
3978         my $sel = new IO::Select;
3979         $sel->add($sock);
3980
3981         if (!scalar(my @ready = $sel->can_write($timeout))) {
3982             die "monitor write error - timeout";
3983         }
3984
3985         my $fullcmd = "$cmdstr\r";
3986
3987         # syslog('info', "VM $vmid monitor command: $cmdstr");
3988
3989         my $b;
3990         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3991             die "monitor write error - $!";
3992         }
3993
3994         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3995
3996         $timeout = 20;
3997
3998         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3999             $timeout = 60*60; # 1 hour
4000         } elsif ($cmdstr =~ m/^(eject|change)/) {
4001             $timeout = 60; # note: cdrom mount command is slow
4002         }
4003         if ($res = __read_avail($sock, $timeout)) {
4004
4005             my @lines = split("\r?\n", $res);
4006
4007             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4008
4009             $res = join("\n", @lines);
4010             $res .= "\n";
4011         }
4012     };
4013
4014     my $err = $@;
4015
4016     if ($err) {
4017         syslog("err", "VM $vmid monitor command failed - $err");
4018         die $err;
4019     }
4020
4021     return $res;
4022 }
4023
4024 sub qemu_block_resize {
4025     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4026
4027     my $running = check_running($vmid);
4028
4029     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4030
4031     return if !$running;
4032
4033     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4034
4035 }
4036
4037 sub qemu_volume_snapshot {
4038     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4039
4040     my $running = check_running($vmid);
4041
4042     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4043         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4044     } else {
4045         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4046     }
4047 }
4048
4049 sub qemu_volume_snapshot_delete {
4050     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4051
4052     my $running = check_running($vmid);
4053
4054     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4055
4056     return if !$running;
4057
4058     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4059 }
4060
4061 sub set_migration_caps {
4062     my ($vmid) = @_;
4063
4064     my $cap_ref = [];
4065
4066     my $enabled_cap = {
4067         "auto-converge" => 1,
4068         "xbzrle" => 1,
4069         "x-rdma-pin-all" => 0,
4070         "zero-blocks" => 0,
4071         "compress" => 0
4072     };
4073
4074     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4075
4076     for my $supported_capability (@$supported_capabilities) {
4077         push @$cap_ref, {
4078             capability => $supported_capability->{capability},
4079             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4080         };
4081     }
4082
4083     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4084 }
4085
4086 my $fast_plug_option = {
4087     'lock' => 1,
4088     'name' => 1,
4089     'onboot' => 1,
4090     'shares' => 1,
4091     'startup' => 1,
4092     'description' => 1,
4093 };
4094
4095 # hotplug changes in [PENDING]
4096 # $selection hash can be used to only apply specified options, for
4097 # example: { cores => 1 } (only apply changed 'cores')
4098 # $errors ref is used to return error messages
4099 sub vmconfig_hotplug_pending {
4100     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4101
4102     my $defaults = load_defaults();
4103
4104     # commit values which do not have any impact on running VM first
4105     # Note: those option cannot raise errors, we we do not care about
4106     # $selection and always apply them.
4107
4108     my $add_error = sub {
4109         my ($opt, $msg) = @_;
4110         $errors->{$opt} = "hotplug problem - $msg";
4111     };
4112
4113     my $changes = 0;
4114     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4115         if ($fast_plug_option->{$opt}) {
4116             $conf->{$opt} = $conf->{pending}->{$opt};
4117             delete $conf->{pending}->{$opt};
4118             $changes = 1;
4119         }
4120     }
4121
4122     if ($changes) {
4123         update_config_nolock($vmid, $conf, 1);
4124         $conf = load_config($vmid); # update/reload
4125     }
4126
4127     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4128
4129     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4130     while (my ($opt, $force) = each %$pending_delete_hash) {
4131         next if $selection && !$selection->{$opt};
4132         eval {
4133             if ($opt eq 'hotplug') {
4134                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4135             } elsif ($opt eq 'tablet') {
4136                 die "skip\n" if !$hotplug_features->{usb};
4137                 if ($defaults->{tablet}) {
4138                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4139                 } else {
4140                     vm_deviceunplug($vmid, $conf, $opt);
4141                 }
4142             } elsif ($opt eq 'vcpus') {
4143                 die "skip\n" if !$hotplug_features->{cpu};
4144                 qemu_cpu_hotplug($vmid, $conf, undef);
4145             } elsif ($opt eq 'balloon') {
4146                 # enable balloon device is not hotpluggable
4147                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4148             } elsif ($fast_plug_option->{$opt}) {
4149                 # do nothing
4150             } elsif ($opt =~ m/^net(\d+)$/) {
4151                 die "skip\n" if !$hotplug_features->{network};
4152                 vm_deviceunplug($vmid, $conf, $opt);
4153             } elsif (valid_drivename($opt)) {
4154                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4155                 vm_deviceunplug($vmid, $conf, $opt);
4156                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4157             } elsif ($opt =~ m/^memory$/) {
4158                 die "skip\n" if !$hotplug_features->{memory};
4159                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4160             } elsif ($opt eq 'cpuunits') {
4161                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4162             } elsif ($opt eq 'cpulimit') {
4163                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4164             } else {
4165                 die "skip\n";
4166             }
4167         };
4168         if (my $err = $@) {
4169             &$add_error($opt, $err) if $err ne "skip\n";
4170         } else {
4171             # save new config if hotplug was successful
4172             delete $conf->{$opt};
4173             vmconfig_undelete_pending_option($conf, $opt);
4174             update_config_nolock($vmid, $conf, 1);
4175             $conf = load_config($vmid); # update/reload
4176         }
4177     }
4178
4179     foreach my $opt (keys %{$conf->{pending}}) {
4180         next if $selection && !$selection->{$opt};
4181         my $value = $conf->{pending}->{$opt};
4182         eval {
4183             if ($opt eq 'hotplug') {
4184                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4185             } elsif ($opt eq 'tablet') {
4186                 die "skip\n" if !$hotplug_features->{usb};
4187                 if ($value == 1) {
4188                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4189                 } elsif ($value == 0) {
4190                     vm_deviceunplug($vmid, $conf, $opt);
4191                 }
4192             } elsif ($opt eq 'vcpus') {
4193                 die "skip\n" if !$hotplug_features->{cpu};
4194                 qemu_cpu_hotplug($vmid, $conf, $value);
4195             } elsif ($opt eq 'balloon') {
4196                 # enable/disable balloning device is not hotpluggable
4197                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4198                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4199                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4200
4201                 # allow manual ballooning if shares is set to zero
4202                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4203                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4204                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4205                 }
4206             } elsif ($opt =~ m/^net(\d+)$/) {
4207                 # some changes can be done without hotplug
4208                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4209                                     $vmid, $opt, $value);
4210             } elsif (valid_drivename($opt)) {
4211                 # some changes can be done without hotplug
4212                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4213                                      $vmid, $opt, $value, 1);
4214             } elsif ($opt =~ m/^memory$/) { #dimms
4215                 die "skip\n" if !$hotplug_features->{memory};
4216                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4217             } elsif ($opt eq 'cpuunits') {
4218                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4219             } elsif ($opt eq 'cpulimit') {
4220                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4221                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4222             } else {
4223                 die "skip\n";  # skip non-hot-pluggable options
4224             }
4225         };
4226         if (my $err = $@) {
4227             &$add_error($opt, $err) if $err ne "skip\n";
4228         } else {
4229             # save new config if hotplug was successful
4230             $conf->{$opt} = $value;
4231             delete $conf->{pending}->{$opt};
4232             update_config_nolock($vmid, $conf, 1);
4233             $conf = load_config($vmid); # update/reload
4234         }
4235     }
4236 }
4237
4238 sub try_deallocate_drive {
4239     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4240
4241     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4242         my $volid = $drive->{file};
4243         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4244             my $sid = PVE::Storage::parse_volume_id($volid);
4245             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4246
4247             # check if the disk is really unused
4248             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4249                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4250             PVE::Storage::vdisk_free($storecfg, $volid);
4251             return 1;
4252         } else {
4253             # If vm is not owner of this disk remove from config
4254             return 1;
4255         }
4256     }
4257
4258     return undef;
4259 }
4260
4261 sub vmconfig_delete_or_detach_drive {
4262     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4263
4264     my $drive = parse_drive($opt, $conf->{$opt});
4265
4266     my $rpcenv = PVE::RPCEnvironment::get();
4267     my $authuser = $rpcenv->get_user();
4268
4269     if ($force) {
4270         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4271         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4272     } else {
4273         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4274     }
4275 }
4276
4277 sub vmconfig_apply_pending {
4278     my ($vmid, $conf, $storecfg) = @_;
4279
4280     # cold plug
4281
4282     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4283     while (my ($opt, $force) = each %$pending_delete_hash) {
4284         die "internal error" if $opt =~ m/^unused/;
4285         $conf = load_config($vmid); # update/reload
4286         if (!defined($conf->{$opt})) {
4287             vmconfig_undelete_pending_option($conf, $opt);
4288             update_config_nolock($vmid, $conf, 1);
4289         } elsif (valid_drivename($opt)) {
4290             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4291             vmconfig_undelete_pending_option($conf, $opt);
4292             delete $conf->{$opt};
4293             update_config_nolock($vmid, $conf, 1);
4294         } else {
4295             vmconfig_undelete_pending_option($conf, $opt);
4296             delete $conf->{$opt};
4297             update_config_nolock($vmid, $conf, 1);
4298         }
4299     }
4300
4301     $conf = load_config($vmid); # update/reload
4302
4303     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4304         $conf = load_config($vmid); # update/reload
4305
4306         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4307             # skip if nothing changed
4308         } elsif (valid_drivename($opt)) {
4309             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4310                 if defined($conf->{$opt});
4311             $conf->{$opt} = $conf->{pending}->{$opt};
4312         } else {
4313             $conf->{$opt} = $conf->{pending}->{$opt};
4314         }
4315
4316         delete $conf->{pending}->{$opt};
4317         update_config_nolock($vmid, $conf, 1);
4318     }
4319 }
4320
4321 my $safe_num_ne = sub {
4322     my ($a, $b) = @_;
4323
4324     return 0 if !defined($a) && !defined($b);
4325     return 1 if !defined($a);
4326     return 1 if !defined($b);
4327
4328     return $a != $b;
4329 };
4330
4331 my $safe_string_ne = sub {
4332     my ($a, $b) = @_;
4333
4334     return 0 if !defined($a) && !defined($b);
4335     return 1 if !defined($a);
4336     return 1 if !defined($b);
4337
4338     return $a ne $b;
4339 };
4340
4341 sub vmconfig_update_net {
4342     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4343
4344     my $newnet = parse_net($value);
4345
4346     if ($conf->{$opt}) {
4347         my $oldnet = parse_net($conf->{$opt});
4348
4349         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4350             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4351             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4352             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4353
4354             # for non online change, we try to hot-unplug
4355             die "skip\n" if !$hotplug;
4356             vm_deviceunplug($vmid, $conf, $opt);
4357         } else {
4358
4359             die "internal error" if $opt !~ m/net(\d+)/;
4360             my $iface = "tap${vmid}i$1";
4361
4362             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4363                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4364             }
4365
4366             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4367                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4368                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4369                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4370                 PVE::Network::tap_unplug($iface);
4371                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4372             }
4373
4374             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4375                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4376             }
4377
4378             return 1;
4379         }
4380     }
4381
4382     if ($hotplug) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4384     } else {
4385         die "skip\n";
4386     }
4387 }
4388
4389 sub vmconfig_update_disk {
4390     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4391
4392     # fixme: do we need force?
4393
4394     my $drive = parse_drive($opt, $value);
4395
4396     if ($conf->{$opt}) {
4397
4398         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4399
4400             my $media = $drive->{media} || 'disk';
4401             my $oldmedia = $old_drive->{media} || 'disk';
4402             die "unable to change media type\n" if $media ne $oldmedia;
4403
4404             if (!drive_is_cdrom($old_drive)) {
4405
4406                 if ($drive->{file} ne $old_drive->{file}) {
4407
4408                     die "skip\n" if !$hotplug;
4409
4410                     # unplug and register as unused
4411                     vm_deviceunplug($vmid, $conf, $opt);
4412                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4413
4414                 } else {
4415                     # update existing disk
4416
4417                     # skip non hotpluggable value
4418                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4419                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4420                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4421                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4422                         die "skip\n";
4423                     }
4424
4425                     # apply throttle
4426                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4427                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4428                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4429                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4430                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4431                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4432                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4433                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4434                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4435                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4436                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4437                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4438
4439                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4440                                                    ($drive->{mbps} || 0)*1024*1024,
4441                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4442                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4443                                                    $drive->{iops} || 0,
4444                                                    $drive->{iops_rd} || 0,
4445                                                    $drive->{iops_wr} || 0,
4446                                                    ($drive->{mbps_max} || 0)*1024*1024,
4447                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4448                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4449                                                    $drive->{iops_max} || 0,
4450                                                    $drive->{iops_rd_max} || 0,
4451                                                    $drive->{iops_wr_max} || 0);
4452
4453                     }
4454
4455                     return 1;
4456                 }
4457
4458             } else { # cdrom
4459
4460                 if ($drive->{file} eq 'none') {
4461                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4462                 } else {
4463                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4464                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4465                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4466                 }
4467
4468                 return 1;
4469             }
4470         }
4471     }
4472
4473     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4474     # hotplug new disks
4475     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4476     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4477 }
4478
4479 sub vm_start {
4480     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4481         $forcemachine, $spice_ticket) = @_;
4482
4483     lock_config($vmid, sub {
4484         my $conf = load_config($vmid, $migratedfrom);
4485
4486         die "you can't start a vm if it's a template\n" if is_template($conf);
4487
4488         check_lock($conf) if !$skiplock;
4489
4490         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4491
4492         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4493             vmconfig_apply_pending($vmid, $conf, $storecfg);
4494             $conf = load_config($vmid); # update/reload
4495         }
4496
4497         my $defaults = load_defaults();
4498
4499         # set environment variable useful inside network script
4500         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4501
4502         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4503
4504         my $migrate_port = 0;
4505         my $migrate_uri;
4506         if ($statefile) {
4507             if ($statefile eq 'tcp') {
4508                 my $localip = "localhost";
4509                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4510                 my $nodename = PVE::INotify::nodename();
4511                 if ($datacenterconf->{migration_unsecure}) {
4512                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4513                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4514                 }
4515                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4516                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4517                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4518                 push @$cmd, '-incoming', $migrate_uri;
4519                 push @$cmd, '-S';
4520             } else {
4521                 push @$cmd, '-loadstate', $statefile;
4522             }
4523         } elsif ($paused) {
4524             push @$cmd, '-S';
4525         }
4526
4527         # host pci devices
4528         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4529           my $d = parse_hostpci($conf->{"hostpci$i"});
4530           next if !$d;
4531           my $pcidevices = $d->{pciid};
4532           foreach my $pcidevice (@$pcidevices) {
4533                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4534
4535                 my $info = pci_device_info("0000:$pciid");
4536                 die "IOMMU not present\n" if !check_iommu_support();
4537                 die "no pci device info for device '$pciid'\n" if !$info;
4538                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4539                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4540           }
4541         }
4542
4543         PVE::Storage::activate_volumes($storecfg, $vollist);
4544
4545         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4546                     umask => 0077); };
4547         my $err = $@;
4548         die "start failed: $err" if $err;
4549
4550         print "migration listens on $migrate_uri\n" if $migrate_uri;
4551
4552         if ($statefile && $statefile ne 'tcp')  {
4553             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4554             warn $@ if $@;
4555         }
4556
4557         if ($migratedfrom) {
4558
4559             eval {
4560                 set_migration_caps($vmid);
4561             };
4562             warn $@ if $@;
4563
4564             if ($spice_port) {
4565                 print "spice listens on port $spice_port\n";
4566                 if ($spice_ticket) {
4567                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4568                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4569                 }
4570             }
4571
4572         } else {
4573
4574             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4575                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4576                     if $conf->{balloon};
4577             }
4578
4579             foreach my $opt (keys %$conf) {
4580                 next if $opt !~  m/^net\d+$/;
4581                 my $nicconf = parse_net($conf->{$opt});
4582                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4583             }
4584         }
4585
4586         vm_mon_cmd_nocheck($vmid, 'qom-set',
4587                     path => "machine/peripheral/balloon0",
4588                     property => "guest-stats-polling-interval",
4589                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4590
4591     });
4592 }
4593
4594 sub vm_mon_cmd {
4595     my ($vmid, $execute, %params) = @_;
4596
4597     my $cmd = { execute => $execute, arguments => \%params };
4598     vm_qmp_command($vmid, $cmd);
4599 }
4600
4601 sub vm_mon_cmd_nocheck {
4602     my ($vmid, $execute, %params) = @_;
4603
4604     my $cmd = { execute => $execute, arguments => \%params };
4605     vm_qmp_command($vmid, $cmd, 1);
4606 }
4607
4608 sub vm_qmp_command {
4609     my ($vmid, $cmd, $nocheck) = @_;
4610
4611     my $res;
4612
4613     my $timeout;
4614     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4615         $timeout = $cmd->{arguments}->{timeout};
4616         delete $cmd->{arguments}->{timeout};
4617     }
4618
4619     eval {
4620         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4621         my $sname = qmp_socket($vmid);
4622         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4623             my $qmpclient = PVE::QMPClient->new();
4624
4625             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4626         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4627             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4628                 if scalar(%{$cmd->{arguments}});
4629             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4630         } else {
4631             die "unable to open monitor socket\n";
4632         }
4633     };
4634     if (my $err = $@) {
4635         syslog("err", "VM $vmid qmp command failed - $err");
4636         die $err;
4637     }
4638
4639     return $res;
4640 }
4641
4642 sub vm_human_monitor_command {
4643     my ($vmid, $cmdline) = @_;
4644
4645     my $res;
4646
4647     my $cmd = {
4648         execute => 'human-monitor-command',
4649         arguments => { 'command-line' => $cmdline},
4650     };
4651
4652     return vm_qmp_command($vmid, $cmd);
4653 }
4654
4655 sub vm_commandline {
4656     my ($storecfg, $vmid) = @_;
4657
4658     my $conf = load_config($vmid);
4659
4660     my $defaults = load_defaults();
4661
4662     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4663
4664     return join(' ', @$cmd);
4665 }
4666
4667 sub vm_reset {
4668     my ($vmid, $skiplock) = @_;
4669
4670     lock_config($vmid, sub {
4671
4672         my $conf = load_config($vmid);
4673
4674         check_lock($conf) if !$skiplock;
4675
4676         vm_mon_cmd($vmid, "system_reset");
4677     });
4678 }
4679
4680 sub get_vm_volumes {
4681     my ($conf) = @_;
4682
4683     my $vollist = [];
4684     foreach_volid($conf, sub {
4685         my ($volid, $is_cdrom) = @_;
4686
4687         return if $volid =~ m|^/|;
4688
4689         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4690         return if !$sid;
4691
4692         push @$vollist, $volid;
4693     });
4694
4695     return $vollist;
4696 }
4697
4698 sub vm_stop_cleanup {
4699     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4700
4701     eval {
4702
4703         if (!$keepActive) {
4704             my $vollist = get_vm_volumes($conf);
4705             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4706         }
4707
4708         foreach my $ext (qw(mon qmp pid vnc qga)) {
4709             unlink "/var/run/qemu-server/${vmid}.$ext";
4710         }
4711
4712         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4713     };
4714     warn $@ if $@; # avoid errors - just warn
4715 }
4716
4717 # Note: use $nockeck to skip tests if VM configuration file exists.
4718 # We need that when migration VMs to other nodes (files already moved)
4719 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4720 sub vm_stop {
4721     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4722
4723     $force = 1 if !defined($force) && !$shutdown;
4724
4725     if ($migratedfrom){
4726         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4727         kill 15, $pid if $pid;
4728         my $conf = load_config($vmid, $migratedfrom);
4729         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4730         return;
4731     }
4732
4733     lock_config($vmid, sub {
4734
4735         my $pid = check_running($vmid, $nocheck);
4736         return if !$pid;
4737
4738         my $conf;
4739         if (!$nocheck) {
4740             $conf = load_config($vmid);
4741             check_lock($conf) if !$skiplock;
4742             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4743                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4744                 $timeout = $opts->{down} if $opts->{down};
4745             }
4746         }
4747
4748         $timeout = 60 if !defined($timeout);
4749
4750         eval {
4751             if ($shutdown) {
4752                 if (defined($conf) && $conf->{agent}) {
4753                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4754                 } else {
4755                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4756                 }
4757             } else {
4758                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4759             }
4760         };
4761         my $err = $@;
4762
4763         if (!$err) {
4764             my $count = 0;
4765             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4766                 $count++;
4767                 sleep 1;
4768             }
4769
4770             if ($count >= $timeout) {
4771                 if ($force) {
4772                     warn "VM still running - terminating now with SIGTERM\n";
4773                     kill 15, $pid;
4774                 } else {
4775                     die "VM quit/powerdown failed - got timeout\n";
4776                 }
4777             } else {
4778                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4779                 return;
4780             }
4781         } else {
4782             if ($force) {
4783                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4784                 kill 15, $pid;
4785             } else {
4786                 die "VM quit/powerdown failed\n";
4787             }
4788         }
4789
4790         # wait again
4791         $timeout = 10;
4792
4793         my $count = 0;
4794         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4795             $count++;
4796             sleep 1;
4797         }
4798
4799         if ($count >= $timeout) {
4800             warn "VM still running - terminating now with SIGKILL\n";
4801             kill 9, $pid;
4802             sleep 1;
4803         }
4804
4805         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4806    });
4807 }
4808
4809 sub vm_suspend {
4810     my ($vmid, $skiplock) = @_;
4811
4812     lock_config($vmid, sub {
4813
4814         my $conf = load_config($vmid);
4815
4816         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4817
4818         vm_mon_cmd($vmid, "stop");
4819     });
4820 }
4821
4822 sub vm_resume {
4823     my ($vmid, $skiplock, $nocheck) = @_;
4824
4825     lock_config($vmid, sub {
4826
4827         if (!$nocheck) {
4828
4829             my $conf = load_config($vmid);
4830
4831             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4832
4833             vm_mon_cmd($vmid, "cont");
4834
4835         } else {
4836             vm_mon_cmd_nocheck($vmid, "cont");
4837         }
4838     });
4839 }
4840
4841 sub vm_sendkey {
4842     my ($vmid, $skiplock, $key) = @_;
4843
4844     lock_config($vmid, sub {
4845
4846         my $conf = load_config($vmid);
4847
4848         # there is no qmp command, so we use the human monitor command
4849         vm_human_monitor_command($vmid, "sendkey $key");
4850     });
4851 }
4852
4853 sub vm_destroy {
4854     my ($storecfg, $vmid, $skiplock) = @_;
4855
4856     lock_config($vmid, sub {
4857
4858         my $conf = load_config($vmid);
4859
4860         if (!check_running($vmid)) {
4861             destroy_vm($storecfg, $vmid, undef, $skiplock);
4862         } else {
4863             die "VM $vmid is running - destroy failed\n";
4864         }
4865     });
4866 }
4867
4868 # pci helpers
4869
4870 sub file_write {
4871     my ($filename, $buf) = @_;
4872
4873     my $fh = IO::File->new($filename, "w");
4874     return undef if !$fh;
4875
4876     my $res = print $fh $buf;
4877
4878     $fh->close();
4879
4880     return $res;
4881 }
4882
4883 sub pci_device_info {
4884     my ($name) = @_;
4885
4886     my $res;
4887
4888     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4889     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4890
4891     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4892     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4893
4894     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4895     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4896
4897     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4898     return undef if !defined($product) || $product !~ s/^0x//;
4899
4900     $res = {
4901         name => $name,
4902         vendor => $vendor,
4903         product => $product,
4904         domain => $domain,
4905         bus => $bus,
4906         slot => $slot,
4907         func => $func,
4908         irq => $irq,
4909         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4910     };
4911
4912     return $res;
4913 }
4914
4915 sub pci_dev_reset {
4916     my ($dev) = @_;
4917
4918     my $name = $dev->{name};
4919
4920     my $fn = "$pcisysfs/devices/$name/reset";
4921
4922     return file_write($fn, "1");
4923 }
4924
4925 sub pci_dev_bind_to_vfio {
4926     my ($dev) = @_;
4927
4928     my $name = $dev->{name};
4929
4930     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4931
4932     if (!-d $vfio_basedir) {
4933         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4934     }
4935     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4936
4937     my $testdir = "$vfio_basedir/$name";
4938     return 1 if -d $testdir;
4939
4940     my $data = "$dev->{vendor} $dev->{product}";
4941     return undef if !file_write("$vfio_basedir/new_id", $data);
4942
4943     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4944     if (!file_write($fn, $name)) {
4945         return undef if -f $fn;
4946     }
4947
4948     $fn = "$vfio_basedir/bind";
4949     if (! -d $testdir) {
4950         return undef if !file_write($fn, $name);
4951     }
4952
4953     return -d $testdir;
4954 }
4955
4956 sub pci_dev_group_bind_to_vfio {
4957     my ($pciid) = @_;
4958
4959     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4960
4961     if (!-d $vfio_basedir) {
4962         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4963     }
4964     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4965
4966     # get IOMMU group devices
4967     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4968       my @devs = grep /^0000:/, readdir($D);
4969     closedir($D);
4970
4971     foreach my $pciid (@devs) {
4972         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4973
4974         # pci bridges, switches or root ports are not supported
4975         # they have a pci_bus subdirectory so skip them
4976         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4977
4978         my $info = pci_device_info($1);
4979         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4980     }
4981
4982     return 1;
4983 }
4984
4985 sub print_pci_addr {
4986     my ($id, $bridges) = @_;
4987
4988     my $res = '';
4989     my $devices = {
4990         piix3 => { bus => 0, addr => 1 },
4991         #addr2 : first videocard
4992         balloon0 => { bus => 0, addr => 3 },
4993         watchdog => { bus => 0, addr => 4 },
4994         scsihw0 => { bus => 0, addr => 5 },
4995         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4996         scsihw1 => { bus => 0, addr => 6 },
4997         ahci0 => { bus => 0, addr => 7 },
4998         qga0 => { bus => 0, addr => 8 },
4999         spice => { bus => 0, addr => 9 },
5000         virtio0 => { bus => 0, addr => 10 },
5001         virtio1 => { bus => 0, addr => 11 },
5002         virtio2 => { bus => 0, addr => 12 },
5003         virtio3 => { bus => 0, addr => 13 },
5004         virtio4 => { bus => 0, addr => 14 },
5005         virtio5 => { bus => 0, addr => 15 },
5006         hostpci0 => { bus => 0, addr => 16 },
5007         hostpci1 => { bus => 0, addr => 17 },
5008         net0 => { bus => 0, addr => 18 },
5009         net1 => { bus => 0, addr => 19 },
5010         net2 => { bus => 0, addr => 20 },
5011         net3 => { bus => 0, addr => 21 },
5012         net4 => { bus => 0, addr => 22 },
5013         net5 => { bus => 0, addr => 23 },
5014         vga1 => { bus => 0, addr => 24 },
5015         vga2 => { bus => 0, addr => 25 },
5016         vga3 => { bus => 0, addr => 26 },
5017         hostpci2 => { bus => 0, addr => 27 },
5018         hostpci3 => { bus => 0, addr => 28 },
5019         #addr29 : usb-host (pve-usb.cfg)
5020         'pci.1' => { bus => 0, addr => 30 },
5021         'pci.2' => { bus => 0, addr => 31 },
5022         'net6' => { bus => 1, addr => 1 },
5023         'net7' => { bus => 1, addr => 2 },
5024         'net8' => { bus => 1, addr => 3 },
5025         'net9' => { bus => 1, addr => 4 },
5026         'net10' => { bus => 1, addr => 5 },
5027         'net11' => { bus => 1, addr => 6 },
5028         'net12' => { bus => 1, addr => 7 },
5029         'net13' => { bus => 1, addr => 8 },
5030         'net14' => { bus => 1, addr => 9 },
5031         'net15' => { bus => 1, addr => 10 },
5032         'net16' => { bus => 1, addr => 11 },
5033         'net17' => { bus => 1, addr => 12 },
5034         'net18' => { bus => 1, addr => 13 },
5035         'net19' => { bus => 1, addr => 14 },
5036         'net20' => { bus => 1, addr => 15 },
5037         'net21' => { bus => 1, addr => 16 },
5038         'net22' => { bus => 1, addr => 17 },
5039         'net23' => { bus => 1, addr => 18 },
5040         'net24' => { bus => 1, addr => 19 },
5041         'net25' => { bus => 1, addr => 20 },
5042         'net26' => { bus => 1, addr => 21 },
5043         'net27' => { bus => 1, addr => 22 },
5044         'net28' => { bus => 1, addr => 23 },
5045         'net29' => { bus => 1, addr => 24 },
5046         'net30' => { bus => 1, addr => 25 },
5047         'net31' => { bus => 1, addr => 26 },
5048         'virtio6' => { bus => 2, addr => 1 },
5049         'virtio7' => { bus => 2, addr => 2 },
5050         'virtio8' => { bus => 2, addr => 3 },
5051         'virtio9' => { bus => 2, addr => 4 },
5052         'virtio10' => { bus => 2, addr => 5 },
5053         'virtio11' => { bus => 2, addr => 6 },
5054         'virtio12' => { bus => 2, addr => 7 },
5055         'virtio13' => { bus => 2, addr => 8 },
5056         'virtio14' => { bus => 2, addr => 9 },
5057         'virtio15' => { bus => 2, addr => 10 },
5058         'virtioscsi0' => { bus => 3, addr => 1 },
5059         'virtioscsi1' => { bus => 3, addr => 2 },
5060         'virtioscsi2' => { bus => 3, addr => 3 },
5061         'virtioscsi3' => { bus => 3, addr => 4 },
5062         'virtioscsi4' => { bus => 3, addr => 5 },
5063         'virtioscsi5' => { bus => 3, addr => 6 },
5064         'virtioscsi6' => { bus => 3, addr => 7 },
5065         'virtioscsi7' => { bus => 3, addr => 8 },
5066         'virtioscsi8' => { bus => 3, addr => 9 },
5067         'virtioscsi9' => { bus => 3, addr => 10 },
5068         'virtioscsi10' => { bus => 3, addr => 11 },
5069         'virtioscsi11' => { bus => 3, addr => 12 },
5070         'virtioscsi12' => { bus => 3, addr => 13 },
5071         'virtioscsi13' => { bus => 3, addr => 14 },
5072         'virtioscsi14' => { bus => 3, addr => 15 },
5073         'virtioscsi15' => { bus => 3, addr => 16 },
5074         'virtioscsi16' => { bus => 3, addr => 17 },
5075         'virtioscsi17' => { bus => 3, addr => 18 },
5076         'virtioscsi18' => { bus => 3, addr => 19 },
5077         'virtioscsi19' => { bus => 3, addr => 20 },
5078         'virtioscsi20' => { bus => 3, addr => 21 },
5079         'virtioscsi21' => { bus => 3, addr => 22 },
5080         'virtioscsi22' => { bus => 3, addr => 23 },
5081         'virtioscsi23' => { bus => 3, addr => 24 },
5082         'virtioscsi24' => { bus => 3, addr => 25 },
5083         'virtioscsi25' => { bus => 3, addr => 26 },
5084         'virtioscsi26' => { bus => 3, addr => 27 },
5085         'virtioscsi27' => { bus => 3, addr => 28 },
5086         'virtioscsi28' => { bus => 3, addr => 29 },
5087         'virtioscsi29' => { bus => 3, addr => 30 },
5088         'virtioscsi30' => { bus => 3, addr => 31 },
5089
5090     };
5091
5092     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5093            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5094            my $bus = $devices->{$id}->{bus};
5095            $res = ",bus=pci.$bus,addr=$addr";
5096            $bridges->{$bus} = 1 if $bridges;
5097     }
5098     return $res;
5099
5100 }
5101
5102 sub print_pcie_addr {
5103     my ($id) = @_;
5104
5105     my $res = '';
5106     my $devices = {
5107         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5108         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5109         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5110         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5111     };
5112
5113     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5114            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5115            my $bus = $devices->{$id}->{bus};
5116            $res = ",bus=$bus,addr=$addr";
5117     }
5118     return $res;
5119
5120 }
5121
5122 # vzdump restore implementaion
5123
5124 sub tar_archive_read_firstfile {
5125     my $archive = shift;
5126
5127     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5128
5129     # try to detect archive type first
5130     my $pid = open (TMP, "tar tf '$archive'|") ||
5131         die "unable to open file '$archive'\n";
5132     my $firstfile = <TMP>;
5133     kill 15, $pid;
5134     close TMP;
5135
5136     die "ERROR: archive contaions no data\n" if !$firstfile;
5137     chomp $firstfile;
5138
5139     return $firstfile;
5140 }
5141
5142 sub tar_restore_cleanup {
5143     my ($storecfg, $statfile) = @_;
5144
5145     print STDERR "starting cleanup\n";
5146
5147     if (my $fd = IO::File->new($statfile, "r")) {
5148         while (defined(my $line = <$fd>)) {
5149             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5150                 my $volid = $2;
5151                 eval {
5152                     if ($volid =~ m|^/|) {
5153                         unlink $volid || die 'unlink failed\n';
5154                     } else {
5155                         PVE::Storage::vdisk_free($storecfg, $volid);
5156                     }
5157                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5158                 };
5159                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5160             } else {
5161                 print STDERR "unable to parse line in statfile - $line";
5162             }
5163         }
5164         $fd->close();
5165     }
5166 }
5167
5168 sub restore_archive {
5169     my ($archive, $vmid, $user, $opts) = @_;
5170
5171     my $format = $opts->{format};
5172     my $comp;
5173
5174     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5175         $format = 'tar' if !$format;
5176         $comp = 'gzip';
5177     } elsif ($archive =~ m/\.tar$/) {
5178         $format = 'tar' if !$format;
5179     } elsif ($archive =~ m/.tar.lzo$/) {
5180         $format = 'tar' if !$format;
5181         $comp = 'lzop';
5182     } elsif ($archive =~ m/\.vma$/) {
5183         $format = 'vma' if !$format;
5184     } elsif ($archive =~ m/\.vma\.gz$/) {
5185         $format = 'vma' if !$format;
5186         $comp = 'gzip';
5187     } elsif ($archive =~ m/\.vma\.lzo$/) {
5188         $format = 'vma' if !$format;
5189         $comp = 'lzop';
5190     } else {
5191         $format = 'vma' if !$format; # default
5192     }
5193
5194     # try to detect archive format
5195     if ($format eq 'tar') {
5196         return restore_tar_archive($archive, $vmid, $user, $opts);
5197     } else {
5198         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5199     }
5200 }
5201
5202 sub restore_update_config_line {
5203     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5204
5205     return if $line =~ m/^\#qmdump\#/;
5206     return if $line =~ m/^\#vzdump\#/;
5207     return if $line =~ m/^lock:/;
5208     return if $line =~ m/^unused\d+:/;
5209     return if $line =~ m/^parent:/;
5210     return if $line =~ m/^template:/; # restored VM is never a template
5211
5212     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5213         # try to convert old 1.X settings
5214         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5215         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5216             my ($model, $macaddr) = split(/\=/, $devconfig);
5217             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5218             my $net = {
5219                 model => $model,
5220                 bridge => "vmbr$ind",
5221                 macaddr => $macaddr,
5222             };
5223             my $netstr = print_net($net);
5224
5225             print $outfd "net$cookie->{netcount}: $netstr\n";
5226             $cookie->{netcount}++;
5227         }
5228     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5229         my ($id, $netstr) = ($1, $2);
5230         my $net = parse_net($netstr);
5231         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5232         $netstr = print_net($net);
5233         print $outfd "$id: $netstr\n";
5234     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5235         my $virtdev = $1;
5236         my $value = $3;
5237         if ($line =~ m/backup=no/) {
5238             print $outfd "#$line";
5239         } elsif ($virtdev && $map->{$virtdev}) {
5240             my $di = parse_drive($virtdev, $value);
5241             delete $di->{format}; # format can change on restore
5242             $di->{file} = $map->{$virtdev};
5243             $value = print_drive($vmid, $di);
5244             print $outfd "$virtdev: $value\n";
5245         } else {
5246             print $outfd $line;
5247         }
5248     } else {
5249         print $outfd $line;
5250     }
5251 }
5252
5253 sub scan_volids {
5254     my ($cfg, $vmid) = @_;
5255
5256     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5257
5258     my $volid_hash = {};
5259     foreach my $storeid (keys %$info) {
5260         foreach my $item (@{$info->{$storeid}}) {
5261             next if !($item->{volid} && $item->{size});
5262             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5263             $volid_hash->{$item->{volid}} = $item;
5264         }
5265     }
5266
5267     return $volid_hash;
5268 }
5269
5270 sub is_volume_in_use {
5271     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5272
5273     my $path = PVE::Storage::path($storecfg, $volid);
5274
5275     my $scan_config = sub {
5276         my ($cref, $snapname) = @_;
5277
5278         foreach my $key (keys %$cref) {
5279             my $value = $cref->{$key};
5280             if (valid_drivename($key)) {
5281                 next if $skip_drive && $key eq $skip_drive;
5282                 my $drive = parse_drive($key, $value);
5283                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5284                 return 1 if $volid eq $drive->{file};
5285                 if ($drive->{file} =~ m!^/!) {
5286                     return 1 if $drive->{file} eq $path;
5287                 } else {
5288                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5289                     next if !$storeid;
5290                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5291                     next if !$scfg;
5292                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5293                 }
5294             }
5295         }
5296
5297         return 0;
5298     };
5299
5300     return 1 if &$scan_config($conf);
5301
5302     undef $skip_drive;
5303
5304     foreach my $snapname (keys %{$conf->{snapshots}}) {
5305         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5306     }
5307
5308     return 0;
5309 }
5310
5311 sub update_disksize {
5312     my ($vmid, $conf, $volid_hash) = @_;
5313
5314     my $changes;
5315
5316     my $used = {};
5317
5318     # Note: it is allowed to define multiple storages with same path (alias), so
5319     # we need to check both 'volid' and real 'path' (two different volid can point
5320     # to the same path).
5321
5322     my $usedpath = {};
5323
5324     # update size info
5325     foreach my $opt (keys %$conf) {
5326         if (valid_drivename($opt)) {
5327             my $drive = parse_drive($opt, $conf->{$opt});
5328             my $volid = $drive->{file};
5329             next if !$volid;
5330
5331             $used->{$volid} = 1;
5332             if ($volid_hash->{$volid} &&
5333                 (my $path = $volid_hash->{$volid}->{path})) {
5334                 $usedpath->{$path} = 1;
5335             }
5336
5337             next if drive_is_cdrom($drive);
5338             next if !$volid_hash->{$volid};
5339
5340             $drive->{size} = $volid_hash->{$volid}->{size};
5341             my $new = print_drive($vmid, $drive);
5342             if ($new ne $conf->{$opt}) {
5343                 $changes = 1;
5344                 $conf->{$opt} = $new;
5345             }
5346         }
5347     }
5348
5349     # remove 'unusedX' entry if volume is used
5350     foreach my $opt (keys %$conf) {
5351         next if $opt !~ m/^unused\d+$/;
5352         my $volid = $conf->{$opt};
5353         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5354         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5355             $changes = 1;
5356             delete $conf->{$opt};
5357         }
5358     }
5359
5360     foreach my $volid (sort keys %$volid_hash) {
5361         next if $volid =~ m/vm-$vmid-state-/;
5362         next if $used->{$volid};
5363         my $path = $volid_hash->{$volid}->{path};
5364         next if !$path; # just to be sure
5365         next if $usedpath->{$path};
5366         $changes = 1;
5367         add_unused_volume($conf, $volid);
5368         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5369     }
5370
5371     return $changes;
5372 }
5373
5374 sub rescan {
5375     my ($vmid, $nolock) = @_;
5376
5377     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5378
5379     my $volid_hash = scan_volids($cfg, $vmid);
5380
5381     my $updatefn =  sub {
5382         my ($vmid) = @_;
5383
5384         my $conf = load_config($vmid);
5385
5386         check_lock($conf);
5387
5388         my $vm_volids = {};
5389         foreach my $volid (keys %$volid_hash) {
5390             my $info = $volid_hash->{$volid};
5391             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5392         }
5393
5394         my $changes = update_disksize($vmid, $conf, $vm_volids);
5395
5396         update_config_nolock($vmid, $conf, 1) if $changes;
5397     };
5398
5399     if (defined($vmid)) {
5400         if ($nolock) {
5401             &$updatefn($vmid);
5402         } else {
5403             lock_config($vmid, $updatefn, $vmid);
5404         }
5405     } else {
5406         my $vmlist = config_list();
5407         foreach my $vmid (keys %$vmlist) {
5408             if ($nolock) {
5409                 &$updatefn($vmid);
5410             } else {
5411                 lock_config($vmid, $updatefn, $vmid);
5412             }
5413         }
5414     }
5415 }
5416
5417 sub restore_vma_archive {
5418     my ($archive, $vmid, $user, $opts, $comp) = @_;
5419
5420     my $input = $archive eq '-' ? "<&STDIN" : undef;
5421     my $readfrom = $archive;
5422
5423     my $uncomp = '';
5424     if ($comp) {
5425         $readfrom = '-';
5426         my $qarchive = PVE::Tools::shellquote($archive);
5427         if ($comp eq 'gzip') {
5428             $uncomp = "zcat $qarchive|";
5429         } elsif ($comp eq 'lzop') {
5430             $uncomp = "lzop -d -c $qarchive|";
5431         } else {
5432             die "unknown compression method '$comp'\n";
5433         }
5434
5435     }
5436
5437     my $tmpdir = "/var/tmp/vzdumptmp$$";
5438     rmtree $tmpdir;
5439
5440     # disable interrupts (always do cleanups)
5441     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5442         warn "got interrupt - ignored\n";
5443     };
5444
5445     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5446     POSIX::mkfifo($mapfifo, 0600);
5447     my $fifofh;
5448
5449     my $openfifo = sub {
5450         open($fifofh, '>', $mapfifo) || die $!;
5451     };
5452
5453     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5454
5455     my $oldtimeout;
5456     my $timeout = 5;
5457
5458     my $devinfo = {};
5459
5460     my $rpcenv = PVE::RPCEnvironment::get();
5461
5462     my $conffile = config_file($vmid);
5463     my $tmpfn = "$conffile.$$.tmp";
5464
5465     # Note: $oldconf is undef if VM does not exists
5466     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5467
5468     my $print_devmap = sub {
5469         my $virtdev_hash = {};
5470
5471         my $cfgfn = "$tmpdir/qemu-server.conf";
5472
5473         # we can read the config - that is already extracted
5474         my $fh = IO::File->new($cfgfn, "r") ||
5475             "unable to read qemu-server.conf - $!\n";
5476
5477         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5478         if (-f $fwcfgfn) {
5479             my $pve_firewall_dir = '/etc/pve/firewall';
5480             mkdir $pve_firewall_dir; # make sure the dir exists
5481             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5482         }
5483
5484         while (defined(my $line = <$fh>)) {
5485             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5486                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5487                 die "archive does not contain data for drive '$virtdev'\n"
5488                     if !$devinfo->{$devname};
5489                 if (defined($opts->{storage})) {
5490                     $storeid = $opts->{storage} || 'local';
5491                 } elsif (!$storeid) {
5492                     $storeid = 'local';
5493                 }
5494                 $format = 'raw' if !$format;
5495                 $devinfo->{$devname}->{devname} = $devname;
5496                 $devinfo->{$devname}->{virtdev} = $virtdev;
5497                 $devinfo->{$devname}->{format} = $format;
5498                 $devinfo->{$devname}->{storeid} = $storeid;
5499
5500                 # check permission on storage
5501                 my $pool = $opts->{pool}; # todo: do we need that?
5502                 if ($user ne 'root@pam') {
5503                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5504                 }
5505
5506                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5507             }
5508         }
5509
5510         foreach my $devname (keys %$devinfo) {
5511             die "found no device mapping information for device '$devname'\n"
5512                 if !$devinfo->{$devname}->{virtdev};
5513         }
5514
5515         my $cfg = cfs_read_file('storage.cfg');
5516
5517         # create empty/temp config
5518         if ($oldconf) {
5519             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5520             foreach_drive($oldconf, sub {
5521                 my ($ds, $drive) = @_;
5522
5523                 return if drive_is_cdrom($drive);
5524
5525                 my $volid = $drive->{file};
5526
5527                 return if !$volid || $volid =~ m|^/|;
5528
5529                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5530                 return if !$path || !$owner || ($owner != $vmid);
5531
5532                 # Note: only delete disk we want to restore
5533                 # other volumes will become unused
5534                 if ($virtdev_hash->{$ds}) {
5535                     PVE::Storage::vdisk_free($cfg, $volid);
5536                 }
5537             });
5538         }
5539
5540         my $map = {};
5541         foreach my $virtdev (sort keys %$virtdev_hash) {
5542             my $d = $virtdev_hash->{$virtdev};
5543             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5544             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5545
5546             # test if requested format is supported
5547             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5548             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5549             $d->{format} = $defFormat if !$supported;
5550
5551             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5552                                                   $d->{format}, undef, $alloc_size);
5553             print STDERR "new volume ID is '$volid'\n";
5554             $d->{volid} = $volid;
5555             my $path = PVE::Storage::path($cfg, $volid);
5556
5557             PVE::Storage::activate_volumes($cfg,[$volid]);
5558
5559             my $write_zeros = 1;
5560             # fixme: what other storages types initialize volumes with zero?
5561             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5562                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5563                 $write_zeros = 0;
5564             }
5565
5566             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5567
5568             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5569             $map->{$virtdev} = $volid;
5570         }
5571
5572         $fh->seek(0, 0) || die "seek failed - $!\n";
5573
5574         my $outfd = new IO::File ($tmpfn, "w") ||
5575             die "unable to write config for VM $vmid\n";
5576
5577         my $cookie = { netcount => 0 };
5578         while (defined(my $line = <$fh>)) {
5579             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5580         }
5581
5582         $fh->close();
5583         $outfd->close();
5584     };
5585
5586     eval {
5587         # enable interrupts
5588         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5589             die "interrupted by signal\n";
5590         };
5591         local $SIG{ALRM} = sub { die "got timeout\n"; };
5592
5593         $oldtimeout = alarm($timeout);
5594
5595         my $parser = sub {
5596             my $line = shift;
5597
5598             print "$line\n";
5599
5600             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5601                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5602                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5603             } elsif ($line =~ m/^CTIME: /) {
5604                 # we correctly received the vma config, so we can disable
5605                 # the timeout now for disk allocation (set to 10 minutes, so
5606                 # that we always timeout if something goes wrong)
5607                 alarm(600);
5608                 &$print_devmap();
5609                 print $fifofh "done\n";
5610                 my $tmp = $oldtimeout || 0;
5611                 $oldtimeout = undef;
5612                 alarm($tmp);
5613                 close($fifofh);
5614             }
5615         };
5616
5617         print "restore vma archive: $cmd\n";
5618         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5619     };
5620     my $err = $@;
5621
5622     alarm($oldtimeout) if $oldtimeout;
5623
5624     my $vollist = [];
5625     foreach my $devname (keys %$devinfo) {
5626         my $volid = $devinfo->{$devname}->{volid};
5627         push @$vollist, $volid if $volid;
5628     }
5629
5630     my $cfg = cfs_read_file('storage.cfg');
5631     PVE::Storage::deactivate_volumes($cfg, $vollist);
5632
5633     unlink $mapfifo;
5634
5635     if ($err) {
5636         rmtree $tmpdir;
5637         unlink $tmpfn;
5638
5639         foreach my $devname (keys %$devinfo) {
5640             my $volid = $devinfo->{$devname}->{volid};
5641             next if !$volid;
5642             eval {
5643                 if ($volid =~ m|^/|) {
5644                     unlink $volid || die 'unlink failed\n';
5645                 } else {
5646                     PVE::Storage::vdisk_free($cfg, $volid);
5647                 }
5648                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5649             };
5650             print STDERR "unable to cleanup '$volid' - $@" if $@;
5651         }
5652         die $err;
5653     }
5654
5655     rmtree $tmpdir;
5656
5657     rename($tmpfn, $conffile) ||
5658         die "unable to commit configuration file '$conffile'\n";
5659
5660     PVE::Cluster::cfs_update(); # make sure we read new file
5661
5662     eval { rescan($vmid, 1); };
5663     warn $@ if $@;
5664 }
5665
5666 sub restore_tar_archive {
5667     my ($archive, $vmid, $user, $opts) = @_;
5668
5669     if ($archive ne '-') {
5670         my $firstfile = tar_archive_read_firstfile($archive);
5671         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5672             if $firstfile ne 'qemu-server.conf';
5673     }
5674
5675     my $storecfg = cfs_read_file('storage.cfg');
5676
5677     # destroy existing data - keep empty config
5678     my $vmcfgfn = config_file($vmid);
5679     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5680
5681     my $tocmd = "/usr/lib/qemu-server/qmextract";
5682
5683     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5684     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5685     $tocmd .= ' --prealloc' if $opts->{prealloc};
5686     $tocmd .= ' --info' if $opts->{info};
5687
5688     # tar option "xf" does not autodetect compression when read from STDIN,
5689     # so we pipe to zcat
5690     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5691         PVE::Tools::shellquote("--to-command=$tocmd");
5692
5693     my $tmpdir = "/var/tmp/vzdumptmp$$";
5694     mkpath $tmpdir;
5695
5696     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5697     local $ENV{VZDUMP_VMID} = $vmid;
5698     local $ENV{VZDUMP_USER} = $user;
5699
5700     my $conffile = config_file($vmid);
5701     my $tmpfn = "$conffile.$$.tmp";
5702
5703     # disable interrupts (always do cleanups)
5704     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5705         print STDERR "got interrupt - ignored\n";
5706     };
5707
5708     eval {
5709         # enable interrupts
5710         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5711             die "interrupted by signal\n";
5712         };
5713
5714         if ($archive eq '-') {
5715             print "extracting archive from STDIN\n";
5716             run_command($cmd, input => "<&STDIN");
5717         } else {
5718             print "extracting archive '$archive'\n";
5719             run_command($cmd);
5720         }
5721
5722         return if $opts->{info};
5723
5724         # read new mapping
5725         my $map = {};
5726         my $statfile = "$tmpdir/qmrestore.stat";
5727         if (my $fd = IO::File->new($statfile, "r")) {
5728             while (defined (my $line = <$fd>)) {
5729                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5730                     $map->{$1} = $2 if $1;
5731                 } else {
5732                     print STDERR "unable to parse line in statfile - $line\n";
5733                 }
5734             }
5735             $fd->close();
5736         }
5737
5738         my $confsrc = "$tmpdir/qemu-server.conf";
5739
5740         my $srcfd = new IO::File($confsrc, "r") ||
5741             die "unable to open file '$confsrc'\n";
5742
5743         my $outfd = new IO::File ($tmpfn, "w") ||
5744             die "unable to write config for VM $vmid\n";
5745
5746         my $cookie = { netcount => 0 };
5747         while (defined (my $line = <$srcfd>)) {
5748             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5749         }
5750
5751         $srcfd->close();
5752         $outfd->close();
5753     };
5754     my $err = $@;
5755
5756     if ($err) {
5757
5758         unlink $tmpfn;
5759
5760         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5761
5762         die $err;
5763     }
5764
5765     rmtree $tmpdir;
5766
5767     rename $tmpfn, $conffile ||
5768         die "unable to commit configuration file '$conffile'\n";
5769
5770     PVE::Cluster::cfs_update(); # make sure we read new file
5771
5772     eval { rescan($vmid, 1); };
5773     warn $@ if $@;
5774 };
5775
5776
5777 # Internal snapshots
5778
5779 # NOTE: Snapshot create/delete involves several non-atomic
5780 # action, and can take a long time.
5781 # So we try to avoid locking the file and use 'lock' variable
5782 # inside the config file instead.
5783
5784 my $snapshot_copy_config = sub {
5785     my ($source, $dest) = @_;
5786
5787     foreach my $k (keys %$source) {
5788         next if $k eq 'snapshots';
5789         next if $k eq 'snapstate';
5790         next if $k eq 'snaptime';
5791         next if $k eq 'vmstate';
5792         next if $k eq 'lock';
5793         next if $k eq 'digest';
5794         next if $k eq 'description';
5795         next if $k =~ m/^unused\d+$/;
5796
5797         $dest->{$k} = $source->{$k};
5798     }
5799 };
5800
5801 my $snapshot_apply_config = sub {
5802     my ($conf, $snap) = @_;
5803
5804     # copy snapshot list
5805     my $newconf = {
5806         snapshots => $conf->{snapshots},
5807     };
5808
5809     # keep description and list of unused disks
5810     foreach my $k (keys %$conf) {
5811         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5812         $newconf->{$k} = $conf->{$k};
5813     }
5814
5815     &$snapshot_copy_config($snap, $newconf);
5816
5817     return $newconf;
5818 };
5819
5820 sub foreach_writable_storage {
5821     my ($conf, $func) = @_;
5822
5823     my $sidhash = {};
5824
5825     foreach my $ds (keys %$conf) {
5826         next if !valid_drivename($ds);
5827
5828         my $drive = parse_drive($ds, $conf->{$ds});
5829         next if !$drive;
5830         next if drive_is_cdrom($drive);
5831
5832         my $volid = $drive->{file};
5833
5834         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5835         $sidhash->{$sid} = $sid if $sid;
5836     }
5837
5838     foreach my $sid (sort keys %$sidhash) {
5839         &$func($sid);
5840     }
5841 }
5842
5843 my $alloc_vmstate_volid = sub {
5844     my ($storecfg, $vmid, $conf, $snapname) = @_;
5845
5846     # Note: we try to be smart when selecting a $target storage
5847
5848     my $target;
5849
5850     # search shared storage first
5851     foreach_writable_storage($conf, sub {
5852         my ($sid) = @_;
5853         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5854         return if !$scfg->{shared};
5855
5856         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5857     });
5858
5859     if (!$target) {
5860         # now search local storage
5861         foreach_writable_storage($conf, sub {
5862             my ($sid) = @_;
5863             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5864             return if $scfg->{shared};
5865
5866             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5867         });
5868     }
5869
5870     $target = 'local' if !$target;
5871
5872     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5873     # we abort live save after $conf->{memory}, so we need at max twice that space
5874     my $size = $conf->{memory}*2 + $driver_state_size;
5875
5876     my $name = "vm-$vmid-state-$snapname";
5877     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5878     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5879     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5880
5881     return $volid;
5882 };
5883
5884 my $snapshot_prepare = sub {
5885     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5886
5887     my $snap;
5888
5889     my $updatefn =  sub {
5890
5891         my $conf = load_config($vmid);
5892
5893         die "you can't take a snapshot if it's a template\n"
5894             if is_template($conf);
5895
5896         check_lock($conf);
5897
5898         $conf->{lock} = 'snapshot';
5899
5900         die "snapshot name '$snapname' already used\n"
5901             if defined($conf->{snapshots}->{$snapname});
5902
5903         my $storecfg = PVE::Storage::config();
5904         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5905
5906         $snap = $conf->{snapshots}->{$snapname} = {};
5907
5908         if ($save_vmstate && check_running($vmid)) {
5909             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5910         }
5911
5912         &$snapshot_copy_config($conf, $snap);
5913
5914         $snap->{snapstate} = "prepare";
5915         $snap->{snaptime} = time();
5916         $snap->{description} = $comment if $comment;
5917
5918         # always overwrite machine if we save vmstate. This makes sure we
5919         # can restore it later using correct machine type
5920         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5921
5922         update_config_nolock($vmid, $conf, 1);
5923     };
5924
5925     lock_config($vmid, $updatefn);
5926
5927     return $snap;
5928 };
5929
5930 my $snapshot_commit = sub {
5931     my ($vmid, $snapname) = @_;
5932
5933     my $updatefn = sub {
5934
5935         my $conf = load_config($vmid);
5936
5937         die "missing snapshot lock\n"
5938             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5939
5940         my $has_machine_config = defined($conf->{machine});
5941
5942         my $snap = $conf->{snapshots}->{$snapname};
5943
5944         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5945
5946         die "wrong snapshot state\n"
5947             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5948
5949         delete $snap->{snapstate};
5950         delete $conf->{lock};
5951
5952         my $newconf = &$snapshot_apply_config($conf, $snap);
5953
5954         delete $newconf->{machine} if !$has_machine_config;
5955
5956         $newconf->{parent} = $snapname;
5957
5958         update_config_nolock($vmid, $newconf, 1);
5959     };
5960
5961     lock_config($vmid, $updatefn);
5962 };
5963
5964 sub snapshot_rollback {
5965     my ($vmid, $snapname) = @_;
5966
5967     my $prepare = 1;
5968
5969     my $storecfg = PVE::Storage::config();
5970
5971     my $conf = load_config($vmid);
5972
5973     my $get_snapshot_config = sub {
5974
5975         die "you can't rollback if vm is a template\n" if is_template($conf);
5976
5977         my $res = $conf->{snapshots}->{$snapname};
5978
5979         die "snapshot '$snapname' does not exist\n" if !defined($res);
5980
5981         return $res;
5982     };
5983
5984     my $snap = &$get_snapshot_config();
5985
5986     foreach_drive($snap, sub {
5987         my ($ds, $drive) = @_;
5988
5989         return if drive_is_cdrom($drive);
5990
5991         my $volid = $drive->{file};
5992
5993         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5994     });
5995
5996     my $updatefn = sub {
5997
5998         $conf = load_config($vmid);
5999
6000         $snap = &$get_snapshot_config();
6001
6002         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6003             if $snap->{snapstate};
6004
6005         if ($prepare) {
6006             check_lock($conf);
6007             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6008         }
6009
6010         die "unable to rollback vm $vmid: vm is running\n"
6011             if check_running($vmid);
6012
6013         if ($prepare) {
6014             $conf->{lock} = 'rollback';
6015         } else {
6016             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6017             delete $conf->{lock};
6018         }
6019
6020         my $forcemachine;
6021
6022         if (!$prepare) {
6023             my $has_machine_config = defined($conf->{machine});
6024
6025             # copy snapshot config to current config
6026             $conf = &$snapshot_apply_config($conf, $snap);
6027             $conf->{parent} = $snapname;
6028
6029             # Note: old code did not store 'machine', so we try to be smart
6030             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6031             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6032             # we remove the 'machine' configuration if not explicitly specified
6033             # in the original config.
6034             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6035         }
6036
6037         update_config_nolock($vmid, $conf, 1);
6038
6039         if (!$prepare && $snap->{vmstate}) {
6040             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6041             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6042         }
6043     };
6044
6045     lock_config($vmid, $updatefn);
6046
6047     foreach_drive($snap, sub {
6048         my ($ds, $drive) = @_;
6049
6050         return if drive_is_cdrom($drive);
6051
6052         my $volid = $drive->{file};
6053         my $device = "drive-$ds";
6054
6055         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6056     });
6057
6058     $prepare = 0;
6059     lock_config($vmid, $updatefn);
6060 }
6061
6062 my $savevm_wait = sub {
6063     my ($vmid) = @_;
6064
6065     for(;;) {
6066         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6067         if (!$stat->{status}) {
6068             die "savevm not active\n";
6069         } elsif ($stat->{status} eq 'active') {
6070             sleep(1);
6071             next;
6072         } elsif ($stat->{status} eq 'completed') {
6073             last;
6074         } else {
6075             die "query-savevm returned status '$stat->{status}'\n";
6076         }
6077     }
6078 };
6079
6080 sub do_snapshots_with_qemu {
6081     my ($storecfg, $volid) = @_;
6082
6083     my $storage_name = PVE::Storage::parse_volume_id($volid);
6084
6085     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6086         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6087         return 1;
6088     }
6089
6090     if ($volid =~ m/\.(qcow2|qed)$/){
6091         return 1;
6092     }
6093
6094     return undef;
6095 }
6096
6097 sub snapshot_create {
6098     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6099
6100     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6101
6102     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6103
6104     my $config = load_config($vmid);
6105
6106     my $running = check_running($vmid);
6107
6108     my $freezefs = $running && $config->{agent};
6109     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6110
6111     my $drivehash = {};
6112
6113     if ($freezefs) {
6114         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6115         warn "guest-fsfreeze-freeze problems - $@" if $@;
6116     }
6117
6118     eval {
6119         # create internal snapshots of all drives
6120
6121         my $storecfg = PVE::Storage::config();
6122
6123         if ($running) {
6124             if ($snap->{vmstate}) {
6125                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6126                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6127                 &$savevm_wait($vmid);
6128             } else {
6129                 vm_mon_cmd($vmid, "savevm-start");
6130             }
6131         };
6132
6133         foreach_drive($snap, sub {
6134             my ($ds, $drive) = @_;
6135
6136             return if drive_is_cdrom($drive);
6137
6138             my $volid = $drive->{file};
6139             my $device = "drive-$ds";
6140
6141             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6142             $drivehash->{$ds} = 1;
6143        });
6144     };
6145     my $err = $@;
6146
6147     if ($running) {
6148         eval { vm_mon_cmd($vmid, "savevm-end")  };
6149         warn $@ if $@;
6150
6151         if ($freezefs) {
6152             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6153             warn "guest-fsfreeze-thaw problems - $@" if $@;
6154         }
6155
6156         # savevm-end is async, we need to wait
6157         for (;;) {
6158             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6159             if (!$stat->{bytes}) {
6160                 last;
6161             } else {
6162                 print "savevm not yet finished\n";
6163                 sleep(1);
6164                 next;
6165             }
6166         }
6167     }
6168
6169     if ($err) {
6170         warn "snapshot create failed: starting cleanup\n";
6171         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6172         warn $@ if $@;
6173         die $err;
6174     }
6175
6176     &$snapshot_commit($vmid, $snapname);
6177 }
6178
6179 # Note: $drivehash is only set when called from snapshot_create.
6180 sub snapshot_delete {
6181     my ($vmid, $snapname, $force, $drivehash) = @_;
6182
6183     my $prepare = 1;
6184
6185     my $snap;
6186     my $unused = [];
6187
6188     my $unlink_parent = sub {
6189         my ($confref, $new_parent) = @_;
6190
6191         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6192             if ($new_parent) {
6193                 $confref->{parent} = $new_parent;
6194             } else {
6195                 delete $confref->{parent};
6196             }
6197         }
6198     };
6199
6200     my $updatefn =  sub {
6201         my ($remove_drive) = @_;
6202
6203         my $conf = load_config($vmid);
6204
6205         if (!$drivehash) {
6206             check_lock($conf);
6207             die "you can't delete a snapshot if vm is a template\n"
6208                 if is_template($conf);
6209         }
6210
6211         $snap = $conf->{snapshots}->{$snapname};
6212
6213         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6214
6215         # remove parent refs
6216         if (!$prepare) {
6217             &$unlink_parent($conf, $snap->{parent});
6218             foreach my $sn (keys %{$conf->{snapshots}}) {
6219                 next if $sn eq $snapname;
6220                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6221             }
6222         }
6223
6224         if ($remove_drive) {
6225             if ($remove_drive eq 'vmstate') {
6226                 delete $snap->{$remove_drive};
6227             } else {
6228                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6229                 my $volid = $drive->{file};
6230                 delete $snap->{$remove_drive};
6231                 add_unused_volume($conf, $volid);
6232             }
6233         }
6234
6235         if ($prepare) {
6236             $snap->{snapstate} = 'delete';
6237         } else {
6238             delete $conf->{snapshots}->{$snapname};
6239             delete $conf->{lock} if $drivehash;
6240             foreach my $volid (@$unused) {
6241                 add_unused_volume($conf, $volid);
6242             }
6243         }
6244
6245         update_config_nolock($vmid, $conf, 1);
6246     };
6247
6248     lock_config($vmid, $updatefn);
6249
6250     # now remove vmstate file
6251
6252     my $storecfg = PVE::Storage::config();
6253
6254     if ($snap->{vmstate}) {
6255         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6256         if (my $err = $@) {
6257             die $err if !$force;
6258             warn $err;
6259         }
6260         # save changes (remove vmstate from snapshot)
6261         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6262     };
6263
6264     # now remove all internal snapshots
6265     foreach_drive($snap, sub {
6266         my ($ds, $drive) = @_;
6267
6268         return if drive_is_cdrom($drive);
6269
6270         my $volid = $drive->{file};
6271         my $device = "drive-$ds";
6272
6273         if (!$drivehash || $drivehash->{$ds}) {
6274             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6275             if (my $err = $@) {
6276                 die $err if !$force;
6277                 warn $err;
6278             }
6279         }
6280
6281         # save changes (remove drive fron snapshot)
6282         lock_config($vmid, $updatefn, $ds) if !$force;
6283         push @$unused, $volid;
6284     });
6285
6286     # now cleanup config
6287     $prepare = 0;
6288     lock_config($vmid, $updatefn);
6289 }
6290
6291 sub has_feature {
6292     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6293
6294     my $err;
6295     foreach_drive($conf, sub {
6296         my ($ds, $drive) = @_;
6297
6298         return if drive_is_cdrom($drive);
6299         my $volid = $drive->{file};
6300         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6301     });
6302
6303     return $err ? 0 : 1;
6304 }
6305
6306 sub template_create {
6307     my ($vmid, $conf, $disk) = @_;
6308
6309     my $storecfg = PVE::Storage::config();
6310
6311     foreach_drive($conf, sub {
6312         my ($ds, $drive) = @_;
6313
6314         return if drive_is_cdrom($drive);
6315         return if $disk && $ds ne $disk;
6316
6317         my $volid = $drive->{file};
6318         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6319
6320         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6321         $drive->{file} = $voliddst;
6322         $conf->{$ds} = print_drive($vmid, $drive);
6323         update_config_nolock($vmid, $conf, 1);
6324     });
6325 }
6326
6327 sub is_template {
6328     my ($conf) = @_;
6329
6330     return 1 if defined $conf->{template} && $conf->{template} == 1;
6331 }
6332
6333 sub qemu_img_convert {
6334     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6335
6336     my $storecfg = PVE::Storage::config();
6337     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6338     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6339
6340     if ($src_storeid && $dst_storeid) {
6341
6342         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6343
6344         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6345         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6346
6347         my $src_format = qemu_img_format($src_scfg, $src_volname);
6348         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6349
6350         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6351         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6352
6353         my $cmd = [];
6354         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6355         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6356         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6357
6358         my $parser = sub {
6359             my $line = shift;
6360             if($line =~ m/\((\S+)\/100\%\)/){
6361                 my $percent = $1;
6362                 my $transferred = int($size * $percent / 100);
6363                 my $remaining = $size - $transferred;
6364
6365                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6366             }
6367
6368         };
6369
6370         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6371         my $err = $@;
6372         die "copy failed: $err" if $err;
6373     }
6374 }
6375
6376 sub qemu_img_format {
6377     my ($scfg, $volname) = @_;
6378
6379     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6380         return $1;
6381     } else {
6382         return "raw";
6383     }
6384 }
6385
6386 sub qemu_drive_mirror {
6387     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6388
6389     my $storecfg = PVE::Storage::config();
6390     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6391
6392     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6393
6394     my $format = qemu_img_format($dst_scfg, $dst_volname);
6395
6396     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6397
6398     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6399     $opts->{format} = $format if $format;
6400
6401     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6402
6403     eval {
6404     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6405         while (1) {
6406             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6407             my $stat = @$stats[0];
6408             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6409             die "error job is not mirroring" if $stat->{type} ne "mirror";
6410
6411             my $busy = $stat->{busy};
6412             my $ready = $stat->{ready};
6413
6414             if (my $total = $stat->{len}) {
6415                 my $transferred = $stat->{offset} || 0;
6416                 my $remaining = $total - $transferred;
6417                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6418
6419                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6420             }
6421
6422
6423             if ($stat->{ready} eq 'true') {
6424
6425                 last if $vmiddst != $vmid;
6426
6427                 # try to switch the disk if source and destination are on the same guest
6428                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6429                 last if !$@;
6430                 die $@ if $@ !~ m/cannot be completed/;
6431             }
6432             sleep 1;
6433         }
6434
6435
6436     };
6437     my $err = $@;
6438
6439     my $cancel_job = sub {
6440         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6441         while (1) {
6442             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6443             my $stat = @$stats[0];
6444             last if !$stat;
6445             sleep 1;
6446         }
6447     };
6448
6449     if ($err) {
6450         eval { &$cancel_job(); };
6451         die "mirroring error: $err";
6452     }
6453
6454     if ($vmiddst != $vmid) {
6455         # if we clone a disk for a new target vm, we don't switch the disk
6456         &$cancel_job(); # so we call block-job-cancel
6457     }
6458 }
6459
6460 sub clone_disk {
6461     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6462         $newvmid, $storage, $format, $full, $newvollist) = @_;
6463
6464     my $newvolid;
6465
6466     if (!$full) {
6467         print "create linked clone of drive $drivename ($drive->{file})\n";
6468         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6469         push @$newvollist, $newvolid;
6470     } else {
6471         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6472         $storeid = $storage if $storage;
6473
6474         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6475         if (!$format) {
6476             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6477             $format = qemu_img_format($scfg, $volname);
6478         }
6479
6480         # test if requested format is supported - else use default
6481         my $supported = grep { $_ eq $format } @$validFormats;
6482         $format = $defFormat if !$supported;
6483
6484         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6485
6486         print "create full clone of drive $drivename ($drive->{file})\n";
6487         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6488         push @$newvollist, $newvolid;
6489
6490         PVE::Storage::activate_volumes($storecfg, $newvollist);
6491
6492         if (!$running || $snapname) {
6493             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6494         } else {
6495             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6496         }
6497     }
6498
6499     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6500
6501     my $disk = $drive;
6502     $disk->{format} = undef;
6503     $disk->{file} = $newvolid;
6504     $disk->{size} = $size;
6505
6506     return $disk;
6507 }
6508
6509 # this only works if VM is running
6510 sub get_current_qemu_machine {
6511     my ($vmid) = @_;
6512
6513     my $cmd = { execute => 'query-machines', arguments => {} };
6514     my $res = vm_qmp_command($vmid, $cmd);
6515
6516     my ($current, $default);
6517     foreach my $e (@$res) {
6518         $default = $e->{name} if $e->{'is-default'};
6519         $current = $e->{name} if $e->{'is-current'};
6520     }
6521
6522     # fallback to the default machine if current is not supported by qemu
6523     return $current || $default || 'pc';
6524 }
6525
6526 sub qemu_machine_feature_enabled {
6527     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6528
6529     my $current_major;
6530     my $current_minor;
6531
6532     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6533
6534         $current_major = $3;
6535         $current_minor = $4;
6536
6537     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6538
6539         $current_major = $1;
6540         $current_minor = $2;
6541     }
6542
6543     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6544
6545
6546 }
6547
6548 sub qemu_machine_pxe {
6549     my ($vmid, $conf, $machine) = @_;
6550
6551     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6552
6553     foreach my $opt (keys %$conf) {
6554         next if $opt !~ m/^net(\d+)$/;
6555         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6556         next if !$net;
6557         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6558         return $machine.".pxe" if $romfile =~ m/pxe/;
6559         last;
6560     }
6561
6562 }
6563
6564 sub qemu_use_old_bios_files {
6565     my ($machine_type) = @_;
6566
6567     return if !$machine_type;
6568
6569     my $use_old_bios_files = undef;
6570
6571     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6572         $machine_type = $1;
6573         $use_old_bios_files = 1;
6574     } else {
6575         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6576         # load new efi bios files on migration. So this hack is required to allow
6577         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6578         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6579         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6580     }
6581
6582     return ($use_old_bios_files, $machine_type);
6583 }
6584
6585 sub lspci {
6586
6587     my $devices = {};
6588
6589     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6590             my (undef, $id, $function) = @_;
6591             my $res = { id => $id, function => $function};
6592             push @{$devices->{$id}}, $res;
6593     });
6594
6595     return $devices;
6596 }
6597
6598 sub vm_iothreads_list {
6599     my ($vmid) = @_;
6600
6601     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6602
6603     my $iothreads = {};
6604     foreach my $iothread (@$res) {
6605         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6606     }
6607
6608     return $iothreads;
6609 }
6610
6611 sub scsihw_infos {
6612     my ($conf, $drive) = @_;
6613
6614     my $maxdev = 0;
6615
6616     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6617         $maxdev = 7;
6618     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6619         $maxdev = 1;
6620     } else {
6621         $maxdev = 256;
6622     }
6623
6624     my $controller = int($drive->{index} / $maxdev);
6625     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6626
6627     return ($maxdev, $controller, $controller_prefix);
6628 }
6629
6630 # bash completion helper
6631
6632 sub complete_backup_archives {
6633     my ($cmdname, $pname, $cvalue) = @_;
6634
6635     my $cfg = PVE::Storage::config();
6636
6637     my $storeid;
6638
6639     if ($cvalue =~ m/^([^:]+):/) {
6640         $storeid = $1;
6641     }
6642
6643     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6644
6645     my $res = [];
6646     foreach my $id (keys %$data) {
6647         foreach my $item (@{$data->{$id}}) {
6648             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6649             push @$res, $item->{volid} if defined($item->{volid});
6650         }
6651     }
6652
6653     return $res;
6654 }
6655
6656 my $complete_vmid_full = sub {
6657     my ($running) = @_;
6658
6659     my $idlist = vmstatus();
6660
6661     my $res = [];
6662
6663     foreach my $id (keys %$idlist) {
6664         my $d = $idlist->{$id};
6665         if (defined($running)) {
6666             next if $d->{template};
6667             next if $running && $d->{status} ne 'running';
6668             next if !$running && $d->{status} eq 'running';
6669         }
6670         push @$res, $id;
6671
6672     }
6673     return $res;
6674 };
6675
6676 sub complete_vmid {
6677     return &$complete_vmid_full();
6678 }
6679
6680 sub complete_vmid_stopped {
6681     return &$complete_vmid_full(0);
6682 }
6683
6684 sub complete_vmid_running {
6685     return &$complete_vmid_full(1);
6686 }
6687
6688 sub complete_storage {
6689
6690     my $cfg = PVE::Storage::config();
6691     my $ids = $cfg->{ids};
6692
6693     my $res = [];
6694     foreach my $sid (keys %$ids) {
6695         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6696         next if !$ids->{$sid}->{content}->{images};
6697         push @$res, $sid;
6698     }
6699
6700     return $res;
6701 }
6702
6703 1;