PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbformat = {
 731     host => {
 732         default_key => 1,
 733         type => 'string', format => 'pve-qm-usb-device',
 734         format_description => 'HOSTUSBDEVICE|spice',
 735         description => 'The Host USB device or port or the value spice',
 736     },
 737     usb3 => {
 738         optional => 1,
 739         type => 'boolean',
 740         format_description => 'yes|no',
 741         description => 'Specifies whether if given host option is a USB3 device or port',
 742     },
 743 };
 744
 745 my $usbdesc = {
 746     optional => 1,
 747     type => 'string', format => $usbformat,
 748     description => <<EODESCR,
 749 Configure an USB device (n is 0 to 4). This can be used to
 750 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 751
 752 'bus-port(.port)*' (decimal numbers) or
 753 'vendor_id:product_id' (hexadeciaml numbers) or
 754 'spice'
 755
 756 You can use the 'lsusb -t' command to list existing usb devices.
 757
 758 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 759
 760 The value 'spice' can be used to add a usb redirection devices for spice.
 761
 762 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 763
 764 EODESCR
 765 };
 766 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 767
 768 my $hostpcidesc = {
 769         optional => 1,
 770         type => 'string', format => 'pve-qm-hostpci',
 771         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 772         description => <<EODESCR,
 773 Map host pci devices. HOSTPCIDEVICE syntax is:
 774
 775 'bus:dev.func' (hexadecimal numbers)
 776
 777 You can us the 'lspci' command to list existing pci devices.
 778
 779 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 780
 781 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 782
 783 Experimental: user reported problems with this option.
 784 EODESCR
 785 };
 786 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 787
 788 my $serialdesc = {
 789         optional => 1,
 790         type => 'string',
 791         pattern => '(/dev/.+|socket)',
 792         description =>  <<EODESCR,
 793 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 794
 795 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 796
 797 Experimental: user reported problems with this option.
 798 EODESCR
 799 };
 800
 801 my $paralleldesc= {
 802         optional => 1,
 803         type => 'string',
 804         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 805         description =>  <<EODESCR,
 806 Map host parallel devices (n is 0 to 2).
 807
 808 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 809
 810 Experimental: user reported problems with this option.
 811 EODESCR
 812 };
 813
 814 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 815     $confdesc->{"parallel$i"} = $paralleldesc;
 816 }
 817
 818 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 819     $confdesc->{"serial$i"} = $serialdesc;
 820 }
 821
 822 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 823     $confdesc->{"hostpci$i"} = $hostpcidesc;
 824 }
 825
 826 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 827     $drivename_hash->{"ide$i"} = 1;
 828     $confdesc->{"ide$i"} = $idedesc;
 829 }
 830
 831 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 832     $drivename_hash->{"sata$i"} = 1;
 833     $confdesc->{"sata$i"} = $satadesc;
 834 }
 835
 836 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 837     $drivename_hash->{"scsi$i"} = 1;
 838     $confdesc->{"scsi$i"} = $scsidesc ;
 839 }
 840
 841 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 842     $drivename_hash->{"virtio$i"} = 1;
 843     $confdesc->{"virtio$i"} = $virtiodesc;
 844 }
 845
 846 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 847     $confdesc->{"usb$i"} = $usbdesc;
 848 }
 849
 850 my $unuseddesc = {
 851     optional => 1,
 852     type => 'string', format => 'pve-volume-id',
 853     description => "Reference to unused volumes.",
 854 };
 855
 856 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 857     $confdesc->{"unused$i"} = $unuseddesc;
 858 }
 859
 860 my $kvm_api_version = 0;
 861
 862 sub kvm_version {
 863
 864     return $kvm_api_version if $kvm_api_version;
 865
 866     my $fh = IO::File->new("</dev/kvm") ||
 867         return 0;
 868
 869     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 870         $kvm_api_version = $v;
 871     }
 872
 873     $fh->close();
 874
 875     return  $kvm_api_version;
 876 }
 877
 878 my $kvm_user_version;
 879
 880 sub kvm_user_version {
 881
 882     return $kvm_user_version if $kvm_user_version;
 883
 884     $kvm_user_version = 'unknown';
 885
 886     my $tmp = `kvm -help 2>/dev/null`;
 887
 888     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 889         $kvm_user_version = $2;
 890     }
 891
 892     return $kvm_user_version;
 893
 894 }
 895
 896 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 897
 898 sub disknames {
 899     # order is important - used to autoselect boot disk
 900     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 901             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 902             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 903             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 904 }
 905
 906 sub valid_drivename {
 907     my $dev = shift;
 908
 909     return defined($drivename_hash->{$dev});
 910 }
 911
 912 sub option_exists {
 913     my $key = shift;
 914     return defined($confdesc->{$key});
 915 }
 916
 917 sub nic_models {
 918     return $nic_model_list;
 919 }
 920
 921 sub os_list_description {
 922
 923     return {
 924         other => 'Other',
 925         wxp => 'Windows XP',
 926         w2k => 'Windows 2000',
 927         w2k3 =>, 'Windows 2003',
 928         w2k8 => 'Windows 2008',
 929         wvista => 'Windows Vista',
 930         win7 => 'Windows 7',
 931         win8 => 'Windows 8/2012',
 932         l24 => 'Linux 2.4',
 933         l26 => 'Linux 2.6',
 934     };
 935 }
 936
 937 my $cdrom_path;
 938
 939 sub get_cdrom_path {
 940
 941     return  $cdrom_path if $cdrom_path;
 942
 943     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 944     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 945     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 946 }
 947
 948 sub get_iso_path {
 949     my ($storecfg, $vmid, $cdrom) = @_;
 950
 951     if ($cdrom eq 'cdrom') {
 952         return get_cdrom_path();
 953     } elsif ($cdrom eq 'none') {
 954         return '';
 955     } elsif ($cdrom =~ m|^/|) {
 956         return $cdrom;
 957     } else {
 958         return PVE::Storage::path($storecfg, $cdrom);
 959     }
 960 }
 961
 962 # try to convert old style file names to volume IDs
 963 sub filename_to_volume_id {
 964     my ($vmid, $file, $media) = @_;
 965
 966     if (!($file eq 'none' || $file eq 'cdrom' ||
 967           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 968
 969         return undef if $file =~ m|/|;
 970
 971         if ($media && $media eq 'cdrom') {
 972             $file = "local:iso/$file";
 973         } else {
 974             $file = "local:$vmid/$file";
 975         }
 976     }
 977
 978     return $file;
 979 }
 980
 981 sub verify_media_type {
 982     my ($opt, $vtype, $media) = @_;
 983
 984     return if !$media;
 985
 986     my $etype;
 987     if ($media eq 'disk') {
 988         $etype = 'images';
 989     } elsif ($media eq 'cdrom') {
 990         $etype = 'iso';
 991     } else {
 992         die "internal error";
 993     }
 994
 995     return if ($vtype eq $etype);
 996
 997     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 998 }
 999
1000 sub cleanup_drive_path {
1001     my ($opt, $storecfg, $drive) = @_;
1002
1003     # try to convert filesystem paths to volume IDs
1004
1005     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1006         ($drive->{file} !~ m|^/dev/.+|) &&
1007         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1008         ($drive->{file} !~ m/^\d+$/)) {
1009         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1010         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1011         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1012         verify_media_type($opt, $vtype, $drive->{media});
1013         $drive->{file} = $volid;
1014     }
1015
1016     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1017 }
1018
1019 sub create_conf_nolock {
1020     my ($vmid, $settings) = @_;
1021
1022     my $filename = config_file($vmid);
1023
1024     die "configuration file '$filename' already exists\n" if -f $filename;
1025
1026     my $defaults = load_defaults();
1027
1028     $settings->{name} = "vm$vmid" if !$settings->{name};
1029     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
1030
1031     my $data = '';
1032     foreach my $opt (keys %$settings) {
1033         next if !$confdesc->{$opt};
1034
1035         my $value = $settings->{$opt};
1036         next if !$value;
1037
1038         $data .= "$opt: $value\n";
1039     }
1040
1041     PVE::Tools::file_set_contents($filename, $data);
1042 }
1043
1044 sub parse_hotplug_features {
1045     my ($data) = @_;
1046
1047     my $res = {};
1048
1049     return $res if $data eq '0';
1050
1051     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1052
1053     foreach my $feature (PVE::Tools::split_list($data)) {
1054         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1055             $res->{$1} = 1;
1056         } else {
1057             warn "ignoring unknown hotplug feature '$feature'\n";
1058         }
1059     }
1060     return $res;
1061 }
1062
1063 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1064 sub pve_verify_hotplug_features {
1065     my ($value, $noerr) = @_;
1066
1067     return $value if parse_hotplug_features($value);
1068
1069     return undef if $noerr;
1070
1071     die "unable to parse hotplug option\n";
1072 }
1073
1074 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1075 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1076 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1077 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1078 #        [,iothread=on][,serial=serial][,model=model]
1079
1080 sub parse_drive {
1081     my ($key, $data) = @_;
1082
1083     my ($interface, $index);
1084
1085     if ($key =~ m/^([^\d]+)(\d+)$/) {
1086         $interface = $1;
1087         $index = $2;
1088     } else {
1089         return undef;
1090     }
1091
1092     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1093                                      : $confdesc->{$key}->{format};
1094     if (!$desc) {
1095         warn "invalid drive key: $key\n";
1096         return undef;
1097     }
1098     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1099     return undef if !$res;
1100     $res->{interface} = $interface;
1101     $res->{index} = $index;
1102
1103     my $error = 0;
1104     foreach my $opt (qw(bps bps_rd bps_wr)) {
1105         if (my $bps = defined(delete $res->{$opt})) {
1106             if (defined($res->{"m$opt"})) {
1107                 warn "both $opt and m$opt specified\n";
1108                 ++$error;
1109                 next;
1110             }
1111             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1112         }
1113     }
1114     return undef if $error;
1115
1116     return undef if $res->{mbps_rd} && $res->{mbps};
1117     return undef if $res->{mbps_wr} && $res->{mbps};
1118     return undef if $res->{iops_rd} && $res->{iops};
1119     return undef if $res->{iops_wr} && $res->{iops};
1120
1121     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1122         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1123         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1124         return undef if $res->{interface} eq 'virtio';
1125     }
1126
1127     if (my $size = $res->{size}) {
1128         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1129     }
1130
1131     return $res;
1132 }
1133
1134 sub print_drive {
1135     my ($vmid, $drive) = @_;
1136     my $data = { %$drive };
1137     delete $data->{$_} for qw(index interface);
1138     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1139 }
1140
1141 sub scsi_inquiry {
1142     my($fh, $noerr) = @_;
1143
1144     my $SG_IO = 0x2285;
1145     my $SG_GET_VERSION_NUM = 0x2282;
1146
1147     my $versionbuf = "\x00" x 8;
1148     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1149     if (!$ret) {
1150         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1151         return undef;
1152     }
1153     my $version = unpack("I", $versionbuf);
1154     if ($version < 30000) {
1155         die "scsi generic interface too old\n"  if !$noerr;
1156         return undef;
1157     }
1158
1159     my $buf = "\x00" x 36;
1160     my $sensebuf = "\x00" x 8;
1161     my $cmd = pack("C x3 C x1", 0x12, 36);
1162
1163     # see /usr/include/scsi/sg.h
1164     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1165
1166     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1167                       length($sensebuf), 0, length($buf), $buf,
1168                       $cmd, $sensebuf, 6000);
1169
1170     $ret = ioctl($fh, $SG_IO, $packet);
1171     if (!$ret) {
1172         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1173         return undef;
1174     }
1175
1176     my @res = unpack($sg_io_hdr_t, $packet);
1177     if ($res[17] || $res[18]) {
1178         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1179         return undef;
1180     }
1181
1182     my $res = {};
1183     (my $byte0, my $byte1, $res->{vendor},
1184      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1185
1186     $res->{removable} = $byte1 & 128 ? 1 : 0;
1187     $res->{type} = $byte0 & 31;
1188
1189     return $res;
1190 }
1191
1192 sub path_is_scsi {
1193     my ($path) = @_;
1194
1195     my $fh = IO::File->new("+<$path") || return undef;
1196     my $res = scsi_inquiry($fh, 1);
1197     close($fh);
1198
1199     return $res;
1200 }
1201
1202 sub machine_type_is_q35 {
1203     my ($conf) = @_;
1204
1205     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1206 }
1207
1208 sub print_tabletdevice_full {
1209     my ($conf) = @_;
1210
1211     my $q35 = machine_type_is_q35($conf);
1212
1213     # we use uhci for old VMs because tablet driver was buggy in older qemu
1214     my $usbbus = $q35 ? "ehci" : "uhci";
1215
1216     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1217 }
1218
1219 sub print_drivedevice_full {
1220     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1221
1222     my $device = '';
1223     my $maxdev = 0;
1224
1225     if ($drive->{interface} eq 'virtio') {
1226         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1227         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1228         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1229     } elsif ($drive->{interface} eq 'scsi') {
1230
1231         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1232         my $unit = $drive->{index} % $maxdev;
1233         my $devicetype = 'hd';
1234         my $path = '';
1235         if (drive_is_cdrom($drive)) {
1236               $devicetype = 'cd';
1237         } else {
1238               if ($drive->{file} =~ m|^/|) {
1239                   $path = $drive->{file};
1240               } else {
1241                   $path = PVE::Storage::path($storecfg, $drive->{file});
1242               }
1243
1244               if($path =~ m/^iscsi\:\/\//){
1245                   $devicetype = 'generic';
1246               } else {
1247                   if (my $info = path_is_scsi($path)) {
1248                       if ($info->{type} == 0) {
1249                           $devicetype = 'block';
1250                       } elsif ($info->{type} == 1) { # tape
1251                           $devicetype = 'generic';
1252                       }
1253                   }
1254               }
1255          }
1256
1257         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1258             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1259         } else {
1260             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1261         }
1262
1263     } elsif ($drive->{interface} eq 'ide'){
1264         $maxdev = 2;
1265         my $controller = int($drive->{index} / $maxdev);
1266         my $unit = $drive->{index} % $maxdev;
1267         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1268
1269         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1270         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1271             $device .= ",model=$model";
1272         }
1273     } elsif ($drive->{interface} eq 'sata'){
1274         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1275         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1276         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1277     } elsif ($drive->{interface} eq 'usb') {
1278         die "implement me";
1279         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1280     } else {
1281         die "unsupported interface type";
1282     }
1283
1284     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1285
1286     return $device;
1287 }
1288
1289 sub get_initiator_name {
1290     my $initiator;
1291
1292     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1293     while (defined(my $line = <$fh>)) {
1294         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1295         $initiator = $1;
1296         last;
1297     }
1298     $fh->close();
1299
1300     return $initiator;
1301 }
1302
1303 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1304 sub print_drive_full {
1305     my ($storecfg, $vmid, $drive) = @_;
1306
1307     my $path;
1308     my $volid = $drive->{file};
1309     my $format;
1310
1311     if (drive_is_cdrom($drive)) {
1312         $path = get_iso_path($storecfg, $vmid, $volid);
1313     } else {
1314         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1315         if ($storeid) {
1316             $path = PVE::Storage::path($storecfg, $volid);
1317             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1318             $format = qemu_img_format($scfg, $volname);
1319         } else {
1320             $path = $volid;
1321             $format = "raw";
1322         }
1323    }
1324
1325     my $opts = '';
1326     foreach my $o (@qemu_drive_options) {
1327         next if $o eq 'bootindex';
1328         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1329     }
1330
1331     $opts .= ",format=$format" if $format && !$drive->{format};
1332
1333     foreach my $o (qw(bps bps_rd bps_wr)) {
1334         my $v = $drive->{"m$o"};
1335         $opts .= ",$o=" . int($v*1024*1024) if $v;
1336     }
1337
1338     my $cache_direct = 0;
1339
1340     if (my $cache = $drive->{cache}) {
1341         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1342     } elsif (!drive_is_cdrom($drive)) {
1343         $opts .= ",cache=none";
1344         $cache_direct = 1;
1345     }
1346
1347     # aio native works only with O_DIRECT
1348     if (!$drive->{aio}) {
1349         if($cache_direct) {
1350             $opts .= ",aio=native";
1351         } else {
1352             $opts .= ",aio=threads";
1353         }
1354     }
1355
1356     if (!drive_is_cdrom($drive)) {
1357         my $detectzeroes;
1358         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1359             $detectzeroes = 'off';
1360         } elsif ($drive->{discard}) {
1361             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1362         } else {
1363             # This used to be our default with discard not being specified:
1364             $detectzeroes = 'on';
1365         }
1366         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1367     }
1368
1369     my $pathinfo = $path ? "file=$path," : '';
1370
1371     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1372 }
1373
1374 sub print_netdevice_full {
1375     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1376
1377     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1378
1379     my $device = $net->{model};
1380     if ($net->{model} eq 'virtio') {
1381          $device = 'virtio-net-pci';
1382      };
1383
1384     my $pciaddr = print_pci_addr("$netid", $bridges);
1385     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1386     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1387         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1388         my $vectors = $net->{queues} * 2 + 2;
1389         $tmpstr .= ",vectors=$vectors,mq=on";
1390     }
1391     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1392
1393     if ($use_old_bios_files) {
1394         my $romfile;
1395         if ($device eq 'virtio-net-pci') {
1396             $romfile = 'pxe-virtio.rom';
1397         } elsif ($device eq 'e1000') {
1398             $romfile = 'pxe-e1000.rom';
1399         } elsif ($device eq 'ne2k') {
1400             $romfile = 'pxe-ne2k_pci.rom';
1401         } elsif ($device eq 'pcnet') {
1402             $romfile = 'pxe-pcnet.rom';
1403         } elsif ($device eq 'rtl8139') {
1404             $romfile = 'pxe-rtl8139.rom';
1405         }
1406         $tmpstr .= ",romfile=$romfile" if $romfile;
1407     }
1408
1409     return $tmpstr;
1410 }
1411
1412 sub print_netdev_full {
1413     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1414
1415     my $i = '';
1416     if ($netid =~ m/^net(\d+)$/) {
1417         $i = int($1);
1418     }
1419
1420     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1421
1422     my $ifname = "tap${vmid}i$i";
1423
1424     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1425     die "interface name '$ifname' is too long (max 15 character)\n"
1426         if length($ifname) >= 16;
1427
1428     my $vhostparam = '';
1429     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1430
1431     my $vmname = $conf->{name} || "vm$vmid";
1432
1433     my $netdev = "";
1434     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1435
1436     if ($net->{bridge}) {
1437         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1438     } else {
1439         $netdev = "type=user,id=$netid,hostname=$vmname";
1440     }
1441
1442     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1443
1444     return $netdev;
1445 }
1446
1447 sub drive_is_cdrom {
1448     my ($drive) = @_;
1449
1450     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1451
1452 }
1453
1454 sub parse_numa {
1455     my ($data) = @_;
1456
1457     my $res = {};
1458
1459     foreach my $kvp (split(/,/, $data)) {
1460
1461         if ($kvp =~ m/^memory=(\S+)$/) {
1462             $res->{memory} = $1;
1463         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1464             $res->{policy} = $1;
1465         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1466             $res->{cpus}->{start} = $1;
1467             $res->{cpus}->{end} = $3;
1468         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1469             $res->{hostnodes}->{start} = $1;
1470             $res->{hostnodes}->{end} = $3;
1471         } else {
1472             return undef;
1473         }
1474     }
1475
1476     return $res;
1477 }
1478
1479 sub parse_hostpci {
1480     my ($value) = @_;
1481
1482     return undef if !$value;
1483
1484
1485     my @list = split(/,/, $value);
1486     my $found;
1487
1488     my $res = {};
1489     foreach my $kv (@list) {
1490
1491         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1492             $found = 1;
1493             if(defined($4)){
1494                 push @{$res->{pciid}}, { id => $2 , function => $4};
1495
1496             }else{
1497                 my $pcidevices = lspci($2);
1498                 $res->{pciid} = $pcidevices->{$2};
1499             }
1500         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1501             $res->{rombar} = $1;
1502         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1503             $res->{'x-vga'} = $1;
1504         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1505             $res->{pcie} = 1 if $1 == 1;
1506         } else {
1507             warn "unknown hostpci setting '$kv'\n";
1508         }
1509     }
1510
1511     return undef if !$found;
1512
1513     return $res;
1514 }
1515
1516 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1517 sub parse_net {
1518     my ($data) = @_;
1519
1520     my $res = {};
1521
1522     foreach my $kvp (split(/,/, $data)) {
1523
1524         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1525             my $model = lc($1);
1526             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1527             $res->{model} = $model;
1528             $res->{macaddr} = $mac;
1529         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1530             $res->{bridge} = $1;
1531         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1532             $res->{queues} = $1;
1533         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1534             $res->{rate} = $1;
1535         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1536             $res->{tag} = $1;
1537         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1538             $res->{trunks} = $1;
1539         } elsif ($kvp =~ m/^firewall=([01])$/) {
1540             $res->{firewall} = $1;
1541         } elsif ($kvp =~ m/^link_down=([01])$/) {
1542             $res->{link_down} = $1;
1543         } else {
1544             return undef;
1545         }
1546
1547     }
1548
1549     return undef if !$res->{model};
1550
1551     return $res;
1552 }
1553
1554 sub print_net {
1555     my $net = shift;
1556
1557     my $res = "$net->{model}";
1558     $res .= "=$net->{macaddr}" if $net->{macaddr};
1559     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1560     $res .= ",rate=$net->{rate}" if $net->{rate};
1561     $res .= ",tag=$net->{tag}" if $net->{tag};
1562     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1563     $res .= ",firewall=1" if $net->{firewall};
1564     $res .= ",link_down=1" if $net->{link_down};
1565     $res .= ",queues=$net->{queues}" if $net->{queues};
1566
1567     return $res;
1568 }
1569
1570 sub add_random_macs {
1571     my ($settings) = @_;
1572
1573     foreach my $opt (keys %$settings) {
1574         next if $opt !~ m/^net(\d+)$/;
1575         my $net = parse_net($settings->{$opt});
1576         next if !$net;
1577         $settings->{$opt} = print_net($net);
1578     }
1579 }
1580
1581 sub add_unused_volume {
1582     my ($config, $volid) = @_;
1583
1584     my $key;
1585     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1586         my $test = "unused$ind";
1587         if (my $vid = $config->{$test}) {
1588             return if $vid eq $volid; # do not add duplicates
1589         } else {
1590             $key = $test;
1591         }
1592     }
1593
1594     die "To many unused volume - please delete them first.\n" if !$key;
1595
1596     $config->{$key} = $volid;
1597
1598     return $key;
1599 }
1600
1601 sub vm_is_volid_owner {
1602     my ($storecfg, $vmid, $volid) = @_;
1603
1604     if ($volid !~  m|^/|) {
1605         my ($path, $owner);
1606         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1607         if ($owner && ($owner == $vmid)) {
1608             return 1;
1609         }
1610     }
1611
1612     return undef;
1613 }
1614
1615 sub split_flagged_list {
1616     my $text = shift || '';
1617     $text =~ s/[,;]/ /g;
1618     $text =~ s/^\s+//;
1619     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1620 }
1621
1622 sub join_flagged_list {
1623     my ($how, $lst) = @_;
1624     join $how, map { $lst->{$_} . $_ } keys %$lst;
1625 }
1626
1627 sub vmconfig_delete_pending_option {
1628     my ($conf, $key, $force) = @_;
1629
1630     delete $conf->{pending}->{$key};
1631     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1632     $pending_delete_hash->{$key} = $force ? '!' : '';
1633     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1634 }
1635
1636 sub vmconfig_undelete_pending_option {
1637     my ($conf, $key) = @_;
1638
1639     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1640     delete $pending_delete_hash->{$key};
1641
1642     if (%$pending_delete_hash) {
1643         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1644     } else {
1645         delete $conf->{pending}->{delete};
1646     }
1647 }
1648
1649 sub vmconfig_register_unused_drive {
1650     my ($storecfg, $vmid, $conf, $drive) = @_;
1651
1652     if (!drive_is_cdrom($drive)) {
1653         my $volid = $drive->{file};
1654         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1655             add_unused_volume($conf, $volid, $vmid);
1656         }
1657     }
1658 }
1659
1660 sub vmconfig_cleanup_pending {
1661     my ($conf) = @_;
1662
1663     # remove pending changes when nothing changed
1664     my $changes;
1665     foreach my $opt (keys %{$conf->{pending}}) {
1666         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1667             $changes = 1;
1668             delete $conf->{pending}->{$opt};
1669         }
1670     }
1671
1672     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1673     my $pending_delete_hash = {};
1674     while (my ($opt, $force) = each %$current_delete_hash) {
1675         if (defined($conf->{$opt})) {
1676             $pending_delete_hash->{$opt} = $force;
1677         } else {
1678             $changes = 1;
1679         }
1680     }
1681
1682     if (%$pending_delete_hash) {
1683         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1684     } else {
1685         delete $conf->{pending}->{delete};
1686     }
1687
1688     return $changes;
1689 }
1690
1691 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1692 my $smbios1_desc = {
1693     uuid => {
1694         type => 'string',
1695         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1696         format_description => 'UUID',
1697         optional => 1,
1698     },
1699     version => {
1700         type => 'string',
1701         pattern => '\S+',
1702         format_description => 'str',
1703         optional => 1,
1704     },
1705     serial => {
1706         type => 'string',
1707         pattern => '\S+',
1708         format_description => 'str',
1709         optional => 1,
1710     },
1711     manufacturer => {
1712         type => 'string',
1713         pattern => '\S+',
1714         format_description => 'name',
1715         optional => 1,
1716     },
1717     product => {
1718         type => 'string',
1719         pattern => '\S+',
1720         format_description => 'name',
1721         optional => 1,
1722     },
1723     sku => {
1724         type => 'string',
1725         pattern => '\S+',
1726         format_description => 'str',
1727         optional => 1,
1728     },
1729     family => {
1730         type => 'string',
1731         pattern => '\S+',
1732         format_description => 'str',
1733         optional => 1,
1734     },
1735 };
1736
1737 sub parse_smbios1 {
1738     my ($data) = @_;
1739
1740     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1741     warn $@ if $@;
1742     return $res;
1743 }
1744
1745 sub print_smbios1 {
1746     my ($smbios1) = @_;
1747     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1748 }
1749
1750 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1751
1752 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1753 sub verify_bootdisk {
1754     my ($value, $noerr) = @_;
1755
1756     return $value if valid_drivename($value);
1757
1758     return undef if $noerr;
1759
1760     die "invalid boot disk '$value'\n";
1761 }
1762
1763 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1764 sub verify_numa {
1765     my ($value, $noerr) = @_;
1766
1767     return $value if parse_numa($value);
1768
1769     return undef if $noerr;
1770
1771     die "unable to parse numa options\n";
1772 }
1773
1774 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1775 sub verify_net {
1776     my ($value, $noerr) = @_;
1777
1778     return $value if parse_net($value);
1779
1780     return undef if $noerr;
1781
1782     die "unable to parse network options\n";
1783 }
1784
1785 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1786 sub verify_hostpci {
1787     my ($value, $noerr) = @_;
1788
1789     return $value if parse_hostpci($value);
1790
1791     return undef if $noerr;
1792
1793     die "unable to parse pci id\n";
1794 }
1795
1796 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1797 sub verify_watchdog {
1798     my ($value, $noerr) = @_;
1799
1800     return $value if parse_watchdog($value);
1801
1802     return undef if $noerr;
1803
1804     die "unable to parse watchdog options\n";
1805 }
1806
1807 sub parse_watchdog {
1808     my ($value) = @_;
1809
1810     return undef if !$value;
1811
1812     my $res = {};
1813
1814     foreach my $p (split(/,/, $value)) {
1815         next if $p =~ m/^\s*$/;
1816
1817         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1818             $res->{model} = $2;
1819         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1820             $res->{action} = $2;
1821         } else {
1822             return undef;
1823         }
1824     }
1825
1826     return $res;
1827 }
1828
1829 sub parse_usb_device {
1830     my ($value) = @_;
1831
1832     return undef if !$value;
1833
1834     my $res = {};
1835     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1836         $res->{vendorid} = $2;
1837         $res->{productid} = $4;
1838     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1839         $res->{hostbus} = $1;
1840         $res->{hostport} = $2;
1841     } elsif ($value =~ m/^spice$/i) {
1842         $res->{spice} = 1;
1843     } else {
1844         return undef;
1845     }
1846
1847     return $res;
1848 }
1849
1850 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1851 sub verify_usb_device {
1852     my ($value, $noerr) = @_;
1853
1854     return $value if parse_usb_device($value);
1855
1856     return undef if $noerr;
1857
1858     die "unable to parse usb device\n";
1859 }
1860
1861 # add JSON properties for create and set function
1862 sub json_config_properties {
1863     my $prop = shift;
1864
1865     foreach my $opt (keys %$confdesc) {
1866         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1867         $prop->{$opt} = $confdesc->{$opt};
1868     }
1869
1870     return $prop;
1871 }
1872
1873 sub check_type {
1874     my ($key, $value) = @_;
1875
1876     die "unknown setting '$key'\n" if !$confdesc->{$key};
1877
1878     my $type = $confdesc->{$key}->{type};
1879
1880     if (!defined($value)) {
1881         die "got undefined value\n";
1882     }
1883
1884     if ($value =~ m/[\n\r]/) {
1885         die "property contains a line feed\n";
1886     }
1887
1888     if ($type eq 'boolean') {
1889         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1890         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1891         die "type check ('boolean') failed - got '$value'\n";
1892     } elsif ($type eq 'integer') {
1893         return int($1) if $value =~ m/^(\d+)$/;
1894         die "type check ('integer') failed - got '$value'\n";
1895     } elsif ($type eq 'number') {
1896         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1897         die "type check ('number') failed - got '$value'\n";
1898     } elsif ($type eq 'string') {
1899         if (my $fmt = $confdesc->{$key}->{format}) {
1900             if ($fmt eq 'pve-qm-drive') {
1901                 # special case - we need to pass $key to parse_drive()
1902                 my $drive = parse_drive($key, $value);
1903                 return $value if $drive;
1904                 die "unable to parse drive options\n";
1905             }
1906             PVE::JSONSchema::check_format($fmt, $value);
1907             return $value;
1908         }
1909         $value =~ s/^\"(.*)\"$/$1/;
1910         return $value;
1911     } else {
1912         die "internal error"
1913     }
1914 }
1915
1916 sub lock_config_full {
1917     my ($vmid, $timeout, $code, @param) = @_;
1918
1919     my $filename = config_file_lock($vmid);
1920
1921     my $res = lock_file($filename, $timeout, $code, @param);
1922
1923     die $@ if $@;
1924
1925     return $res;
1926 }
1927
1928 sub lock_config_mode {
1929     my ($vmid, $timeout, $shared, $code, @param) = @_;
1930
1931     my $filename = config_file_lock($vmid);
1932
1933     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1934
1935     die $@ if $@;
1936
1937     return $res;
1938 }
1939
1940 sub lock_config {
1941     my ($vmid, $code, @param) = @_;
1942
1943     return lock_config_full($vmid, 10, $code, @param);
1944 }
1945
1946 sub cfs_config_path {
1947     my ($vmid, $node) = @_;
1948
1949     $node = $nodename if !$node;
1950     return "nodes/$node/qemu-server/$vmid.conf";
1951 }
1952
1953 sub check_iommu_support{
1954     #fixme : need to check IOMMU support
1955     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1956
1957     my $iommu=1;
1958     return $iommu;
1959
1960 }
1961
1962 sub config_file {
1963     my ($vmid, $node) = @_;
1964
1965     my $cfspath = cfs_config_path($vmid, $node);
1966     return "/etc/pve/$cfspath";
1967 }
1968
1969 sub config_file_lock {
1970     my ($vmid) = @_;
1971
1972     return "$lock_dir/lock-$vmid.conf";
1973 }
1974
1975 sub touch_config {
1976     my ($vmid) = @_;
1977
1978     my $conf = config_file($vmid);
1979     utime undef, undef, $conf;
1980 }
1981
1982 sub destroy_vm {
1983     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1984
1985     my $conffile = config_file($vmid);
1986
1987     my $conf = load_config($vmid);
1988
1989     check_lock($conf) if !$skiplock;
1990
1991     # only remove disks owned by this VM
1992     foreach_drive($conf, sub {
1993         my ($ds, $drive) = @_;
1994
1995         return if drive_is_cdrom($drive);
1996
1997         my $volid = $drive->{file};
1998
1999         return if !$volid || $volid =~ m|^/|;
2000
2001         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2002         return if !$path || !$owner || ($owner != $vmid);
2003
2004         PVE::Storage::vdisk_free($storecfg, $volid);
2005     });
2006
2007     if ($keep_empty_config) {
2008         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2009     } else {
2010         unlink $conffile;
2011     }
2012
2013     # also remove unused disk
2014     eval {
2015         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2016
2017         eval {
2018             PVE::Storage::foreach_volid($dl, sub {
2019                 my ($volid, $sid, $volname, $d) = @_;
2020                 PVE::Storage::vdisk_free($storecfg, $volid);
2021             });
2022         };
2023         warn $@ if $@;
2024
2025     };
2026     warn $@ if $@;
2027 }
2028
2029 sub load_config {
2030     my ($vmid, $node) = @_;
2031
2032     my $cfspath = cfs_config_path($vmid, $node);
2033
2034     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2035
2036     die "no such VM ('$vmid')\n" if !defined($conf);
2037
2038     return $conf;
2039 }
2040
2041 sub parse_vm_config {
2042     my ($filename, $raw) = @_;
2043
2044     return undef if !defined($raw);
2045
2046     my $res = {
2047         digest => Digest::SHA::sha1_hex($raw),
2048         snapshots => {},
2049         pending => {},
2050     };
2051
2052     $filename =~ m|/qemu-server/(\d+)\.conf$|
2053         || die "got strange filename '$filename'";
2054
2055     my $vmid = $1;
2056
2057     my $conf = $res;
2058     my $descr;
2059     my $section = '';
2060
2061     my @lines = split(/\n/, $raw);
2062     foreach my $line (@lines) {
2063         next if $line =~ m/^\s*$/;
2064
2065         if ($line =~ m/^\[PENDING\]\s*$/i) {
2066             $section = 'pending';
2067             if (defined($descr)) {
2068                 $descr =~ s/\s+$//;
2069                 $conf->{description} = $descr;
2070             }
2071             $descr = undef;
2072             $conf = $res->{$section} = {};
2073             next;
2074
2075         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2076             $section = $1;
2077             if (defined($descr)) {
2078                 $descr =~ s/\s+$//;
2079                 $conf->{description} = $descr;
2080             }
2081             $descr = undef;
2082             $conf = $res->{snapshots}->{$section} = {};
2083             next;
2084         }
2085
2086         if ($line =~ m/^\#(.*)\s*$/) {
2087             $descr = '' if !defined($descr);
2088             $descr .= PVE::Tools::decode_text($1) . "\n";
2089             next;
2090         }
2091
2092         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2093             $descr = '' if !defined($descr);
2094             $descr .= PVE::Tools::decode_text($2);
2095         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2096             $conf->{snapstate} = $1;
2097         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2098             my $key = $1;
2099             my $value = $2;
2100             $conf->{$key} = $value;
2101         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2102             my $value = $1;
2103             if ($section eq 'pending') {
2104                 $conf->{delete} = $value; # we parse this later
2105             } else {
2106                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2107             }
2108         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2109             my $key = $1;
2110             my $value = $2;
2111             eval { $value = check_type($key, $value); };
2112             if ($@) {
2113                 warn "vm $vmid - unable to parse value of '$key' - $@";
2114             } else {
2115                 my $fmt = $confdesc->{$key}->{format};
2116                 if ($fmt && $fmt eq 'pve-qm-drive') {
2117                     my $v = parse_drive($key, $value);
2118                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2119                         $v->{file} = $volid;
2120                         $value = print_drive($vmid, $v);
2121                     } else {
2122                         warn "vm $vmid - unable to parse value of '$key'\n";
2123                         next;
2124                     }
2125                 }
2126
2127                 if ($key eq 'cdrom') {
2128                     $conf->{ide2} = $value;
2129                 } else {
2130                     $conf->{$key} = $value;
2131                 }
2132             }
2133         }
2134     }
2135
2136     if (defined($descr)) {
2137         $descr =~ s/\s+$//;
2138         $conf->{description} = $descr;
2139     }
2140     delete $res->{snapstate}; # just to be sure
2141
2142     return $res;
2143 }
2144
2145 sub write_vm_config {
2146     my ($filename, $conf) = @_;
2147
2148     delete $conf->{snapstate}; # just to be sure
2149
2150     if ($conf->{cdrom}) {
2151         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2152         $conf->{ide2} = $conf->{cdrom};
2153         delete $conf->{cdrom};
2154     }
2155
2156     # we do not use 'smp' any longer
2157     if ($conf->{sockets}) {
2158         delete $conf->{smp};
2159     } elsif ($conf->{smp}) {
2160         $conf->{sockets} = $conf->{smp};
2161         delete $conf->{cores};
2162         delete $conf->{smp};
2163     }
2164
2165     my $used_volids = {};
2166
2167     my $cleanup_config = sub {
2168         my ($cref, $pending, $snapname) = @_;
2169
2170         foreach my $key (keys %$cref) {
2171             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2172                 $key eq 'snapstate' || $key eq 'pending';
2173             my $value = $cref->{$key};
2174             if ($key eq 'delete') {
2175                 die "propertry 'delete' is only allowed in [PENDING]\n"
2176                     if !$pending;
2177                 # fixme: check syntax?
2178                 next;
2179             }
2180             eval { $value = check_type($key, $value); };
2181             die "unable to parse value of '$key' - $@" if $@;
2182
2183             $cref->{$key} = $value;
2184
2185             if (!$snapname && valid_drivename($key)) {
2186                 my $drive = parse_drive($key, $value);
2187                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2188             }
2189         }
2190     };
2191
2192     &$cleanup_config($conf);
2193
2194     &$cleanup_config($conf->{pending}, 1);
2195
2196     foreach my $snapname (keys %{$conf->{snapshots}}) {
2197         die "internal error" if $snapname eq 'pending';
2198         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2199     }
2200
2201     # remove 'unusedX' settings if we re-add a volume
2202     foreach my $key (keys %$conf) {
2203         my $value = $conf->{$key};
2204         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2205             delete $conf->{$key};
2206         }
2207     }
2208
2209     my $generate_raw_config = sub {
2210         my ($conf, $pending) = @_;
2211
2212         my $raw = '';
2213
2214         # add description as comment to top of file
2215         if (defined(my $descr = $conf->{description})) {
2216             if ($descr) {
2217                 foreach my $cl (split(/\n/, $descr)) {
2218                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2219                 }
2220             } else {
2221                 $raw .= "#\n" if $pending;
2222             }
2223         }
2224
2225         foreach my $key (sort keys %$conf) {
2226             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2227             $raw .= "$key: $conf->{$key}\n";
2228         }
2229         return $raw;
2230     };
2231
2232     my $raw = &$generate_raw_config($conf);
2233
2234     if (scalar(keys %{$conf->{pending}})){
2235         $raw .= "\n[PENDING]\n";
2236         $raw .= &$generate_raw_config($conf->{pending}, 1);
2237     }
2238
2239     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2240         $raw .= "\n[$snapname]\n";
2241         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2242     }
2243
2244     return $raw;
2245 }
2246
2247 sub write_config {
2248     my ($vmid, $conf) = @_;
2249
2250     my $cfspath = cfs_config_path($vmid);
2251
2252     PVE::Cluster::cfs_write_file($cfspath, $conf);
2253 }
2254
2255 sub load_defaults {
2256
2257     my $res = {};
2258
2259     # we use static defaults from our JSON schema configuration
2260     foreach my $key (keys %$confdesc) {
2261         if (defined(my $default = $confdesc->{$key}->{default})) {
2262             $res->{$key} = $default;
2263         }
2264     }
2265
2266     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2267     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2268
2269     return $res;
2270 }
2271
2272 sub config_list {
2273     my $vmlist = PVE::Cluster::get_vmlist();
2274     my $res = {};
2275     return $res if !$vmlist || !$vmlist->{ids};
2276     my $ids = $vmlist->{ids};
2277
2278     foreach my $vmid (keys %$ids) {
2279         my $d = $ids->{$vmid};
2280         next if !$d->{node} || $d->{node} ne $nodename;
2281         next if !$d->{type} || $d->{type} ne 'qemu';
2282         $res->{$vmid}->{exists} = 1;
2283     }
2284     return $res;
2285 }
2286
2287 # test if VM uses local resources (to prevent migration)
2288 sub check_local_resources {
2289     my ($conf, $noerr) = @_;
2290
2291     my $loc_res = 0;
2292
2293     $loc_res = 1 if $conf->{hostusb}; # old syntax
2294     $loc_res = 1 if $conf->{hostpci}; # old syntax
2295
2296     foreach my $k (keys %$conf) {
2297         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2298         # sockets are safe: they will recreated be on the target side post-migrate
2299         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2300         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2301     }
2302
2303     die "VM uses local resources\n" if $loc_res && !$noerr;
2304
2305     return $loc_res;
2306 }
2307
2308 # check if used storages are available on all nodes (use by migrate)
2309 sub check_storage_availability {
2310     my ($storecfg, $conf, $node) = @_;
2311
2312     foreach_drive($conf, sub {
2313         my ($ds, $drive) = @_;
2314
2315         my $volid = $drive->{file};
2316         return if !$volid;
2317
2318         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2319         return if !$sid;
2320
2321         # check if storage is available on both nodes
2322         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2323         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2324    });
2325 }
2326
2327 # list nodes where all VM images are available (used by has_feature API)
2328 sub shared_nodes {
2329     my ($conf, $storecfg) = @_;
2330
2331     my $nodelist = PVE::Cluster::get_nodelist();
2332     my $nodehash = { map { $_ => 1 } @$nodelist };
2333     my $nodename = PVE::INotify::nodename();
2334
2335     foreach_drive($conf, sub {
2336         my ($ds, $drive) = @_;
2337
2338         my $volid = $drive->{file};
2339         return if !$volid;
2340
2341         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2342         if ($storeid) {
2343             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2344             if ($scfg->{disable}) {
2345                 $nodehash = {};
2346             } elsif (my $avail = $scfg->{nodes}) {
2347                 foreach my $node (keys %$nodehash) {
2348                     delete $nodehash->{$node} if !$avail->{$node};
2349                 }
2350             } elsif (!$scfg->{shared}) {
2351                 foreach my $node (keys %$nodehash) {
2352                     delete $nodehash->{$node} if $node ne $nodename
2353                 }
2354             }
2355         }
2356     });
2357
2358     return $nodehash
2359 }
2360
2361 sub check_lock {
2362     my ($conf) = @_;
2363
2364     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2365 }
2366
2367 sub check_cmdline {
2368     my ($pidfile, $pid) = @_;
2369
2370     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2371     if (defined($fh)) {
2372         my $line = <$fh>;
2373         $fh->close;
2374         return undef if !$line;
2375         my @param = split(/\0/, $line);
2376
2377         my $cmd = $param[0];
2378         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2379
2380         for (my $i = 0; $i < scalar (@param); $i++) {
2381             my $p = $param[$i];
2382             next if !$p;
2383             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2384                 my $p = $param[$i+1];
2385                 return 1 if $p && ($p eq $pidfile);
2386                 return undef;
2387             }
2388         }
2389     }
2390     return undef;
2391 }
2392
2393 sub check_running {
2394     my ($vmid, $nocheck, $node) = @_;
2395
2396     my $filename = config_file($vmid, $node);
2397
2398     die "unable to find configuration file for VM $vmid - no such machine\n"
2399         if !$nocheck && ! -f $filename;
2400
2401     my $pidfile = pidfile_name($vmid);
2402
2403     if (my $fd = IO::File->new("<$pidfile")) {
2404         my $st = stat($fd);
2405         my $line = <$fd>;
2406         close($fd);
2407
2408         my $mtime = $st->mtime;
2409         if ($mtime > time()) {
2410             warn "file '$filename' modified in future\n";
2411         }
2412
2413         if ($line =~ m/^(\d+)$/) {
2414             my $pid = $1;
2415             if (check_cmdline($pidfile, $pid)) {
2416                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2417                     return $pid;
2418                 }
2419             }
2420         }
2421     }
2422
2423     return undef;
2424 }
2425
2426 sub vzlist {
2427
2428     my $vzlist = config_list();
2429
2430     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2431
2432     while (defined(my $de = $fd->read)) {
2433         next if $de !~ m/^(\d+)\.pid$/;
2434         my $vmid = $1;
2435         next if !defined($vzlist->{$vmid});
2436         if (my $pid = check_running($vmid)) {
2437             $vzlist->{$vmid}->{pid} = $pid;
2438         }
2439     }
2440
2441     return $vzlist;
2442 }
2443
2444 sub disksize {
2445     my ($storecfg, $conf) = @_;
2446
2447     my $bootdisk = $conf->{bootdisk};
2448     return undef if !$bootdisk;
2449     return undef if !valid_drivename($bootdisk);
2450
2451     return undef if !$conf->{$bootdisk};
2452
2453     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2454     return undef if !defined($drive);
2455
2456     return undef if drive_is_cdrom($drive);
2457
2458     my $volid = $drive->{file};
2459     return undef if !$volid;
2460
2461     return $drive->{size};
2462 }
2463
2464 my $last_proc_pid_stat;
2465
2466 # get VM status information
2467 # This must be fast and should not block ($full == false)
2468 # We only query KVM using QMP if $full == true (this can be slow)
2469 sub vmstatus {
2470     my ($opt_vmid, $full) = @_;
2471
2472     my $res = {};
2473
2474     my $storecfg = PVE::Storage::config();
2475
2476     my $list = vzlist();
2477     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2478
2479     my $cpucount = $cpuinfo->{cpus} || 1;
2480
2481     foreach my $vmid (keys %$list) {
2482         next if $opt_vmid && ($vmid ne $opt_vmid);
2483
2484         my $cfspath = cfs_config_path($vmid);
2485         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2486
2487         my $d = {};
2488         $d->{pid} = $list->{$vmid}->{pid};
2489
2490         # fixme: better status?
2491         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2492
2493         my $size = disksize($storecfg, $conf);
2494         if (defined($size)) {
2495             $d->{disk} = 0; # no info available
2496             $d->{maxdisk} = $size;
2497         } else {
2498             $d->{disk} = 0;
2499             $d->{maxdisk} = 0;
2500         }
2501
2502         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2503         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2504         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2505
2506         $d->{name} = $conf->{name} || "VM $vmid";
2507         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2508
2509         if ($conf->{balloon}) {
2510             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2511             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2512         }
2513
2514         $d->{uptime} = 0;
2515         $d->{cpu} = 0;
2516         $d->{mem} = 0;
2517
2518         $d->{netout} = 0;
2519         $d->{netin} = 0;
2520
2521         $d->{diskread} = 0;
2522         $d->{diskwrite} = 0;
2523
2524         $d->{template} = is_template($conf);
2525
2526         $res->{$vmid} = $d;
2527     }
2528
2529     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2530     foreach my $dev (keys %$netdev) {
2531         next if $dev !~ m/^tap([1-9]\d*)i/;
2532         my $vmid = $1;
2533         my $d = $res->{$vmid};
2534         next if !$d;
2535
2536         $d->{netout} += $netdev->{$dev}->{receive};
2537         $d->{netin} += $netdev->{$dev}->{transmit};
2538
2539         if ($full) {
2540             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2541             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2542         }
2543
2544     }
2545
2546     my $ctime = gettimeofday;
2547
2548     foreach my $vmid (keys %$list) {
2549
2550         my $d = $res->{$vmid};
2551         my $pid = $d->{pid};
2552         next if !$pid;
2553
2554         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2555         next if !$pstat; # not running
2556
2557         my $used = $pstat->{utime} + $pstat->{stime};
2558
2559         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2560
2561         if ($pstat->{vsize}) {
2562             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2563         }
2564
2565         my $old = $last_proc_pid_stat->{$pid};
2566         if (!$old) {
2567             $last_proc_pid_stat->{$pid} = {
2568                 time => $ctime,
2569                 used => $used,
2570                 cpu => 0,
2571             };
2572             next;
2573         }
2574
2575         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2576
2577         if ($dtime > 1000) {
2578             my $dutime = $used -  $old->{used};
2579
2580             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2581             $last_proc_pid_stat->{$pid} = {
2582                 time => $ctime,
2583                 used => $used,
2584                 cpu => $d->{cpu},
2585             };
2586         } else {
2587             $d->{cpu} = $old->{cpu};
2588         }
2589     }
2590
2591     return $res if !$full;
2592
2593     my $qmpclient = PVE::QMPClient->new();
2594
2595     my $ballooncb = sub {
2596         my ($vmid, $resp) = @_;
2597
2598         my $info = $resp->{'return'};
2599         return if !$info->{max_mem};
2600
2601         my $d = $res->{$vmid};
2602
2603         # use memory assigned to VM
2604         $d->{maxmem} = $info->{max_mem};
2605         $d->{balloon} = $info->{actual};
2606
2607         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2608             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2609             $d->{freemem} = $info->{free_mem};
2610         }
2611
2612         $d->{ballooninfo} = $info;
2613     };
2614
2615     my $blockstatscb = sub {
2616         my ($vmid, $resp) = @_;
2617         my $data = $resp->{'return'} || [];
2618         my $totalrdbytes = 0;
2619         my $totalwrbytes = 0;
2620
2621         for my $blockstat (@$data) {
2622             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2623             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2624
2625             $blockstat->{device} =~ s/drive-//;
2626             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2627         }
2628         $res->{$vmid}->{diskread} = $totalrdbytes;
2629         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2630     };
2631
2632     my $statuscb = sub {
2633         my ($vmid, $resp) = @_;
2634
2635         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2636         # this fails if ballon driver is not loaded, so this must be
2637         # the last commnand (following command are aborted if this fails).
2638         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2639
2640         my $status = 'unknown';
2641         if (!defined($status = $resp->{'return'}->{status})) {
2642             warn "unable to get VM status\n";
2643             return;
2644         }
2645
2646         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2647     };
2648
2649     foreach my $vmid (keys %$list) {
2650         next if $opt_vmid && ($vmid ne $opt_vmid);
2651         next if !$res->{$vmid}->{pid}; # not running
2652         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2653     }
2654
2655     $qmpclient->queue_execute(undef, 1);
2656
2657     foreach my $vmid (keys %$list) {
2658         next if $opt_vmid && ($vmid ne $opt_vmid);
2659         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2660     }
2661
2662     return $res;
2663 }
2664
2665 sub foreach_dimm {
2666     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2667
2668     my $dimm_id = 0;
2669     my $current_size = 1024;
2670     my $dimm_size = 512;
2671     return if $current_size == $memory;
2672
2673     for (my $j = 0; $j < 8; $j++) {
2674         for (my $i = 0; $i < 32; $i++) {
2675             my $name = "dimm${dimm_id}";
2676             $dimm_id++;
2677             my $numanode = $i % $sockets;
2678             $current_size += $dimm_size;
2679             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2680             return  $current_size if $current_size >= $memory;
2681         }
2682         $dimm_size *= 2;
2683     }
2684 }
2685
2686 sub foreach_reverse_dimm {
2687     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2688
2689     my $dimm_id = 253;
2690     my $current_size = 4177920;
2691     my $dimm_size = 65536;
2692     return if $current_size == $memory;
2693
2694     for (my $j = 0; $j < 8; $j++) {
2695         for (my $i = 0; $i < 32; $i++) {
2696             my $name = "dimm${dimm_id}";
2697             $dimm_id--;
2698             my $numanode = $i % $sockets;
2699             $current_size -= $dimm_size;
2700             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2701             return  $current_size if $current_size <= $memory;
2702         }
2703         $dimm_size /= 2;
2704     }
2705 }
2706
2707 sub foreach_drive {
2708     my ($conf, $func) = @_;
2709
2710     foreach my $ds (keys %$conf) {
2711         next if !valid_drivename($ds);
2712
2713         my $drive = parse_drive($ds, $conf->{$ds});
2714         next if !$drive;
2715
2716         &$func($ds, $drive);
2717     }
2718 }
2719
2720 sub foreach_volid {
2721     my ($conf, $func) = @_;
2722
2723     my $volhash = {};
2724
2725     my $test_volid = sub {
2726         my ($volid, $is_cdrom) = @_;
2727
2728         return if !$volid;
2729
2730         $volhash->{$volid} = $is_cdrom || 0;
2731     };
2732
2733     foreach_drive($conf, sub {
2734         my ($ds, $drive) = @_;
2735         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2736     });
2737
2738     foreach my $snapname (keys %{$conf->{snapshots}}) {
2739         my $snap = $conf->{snapshots}->{$snapname};
2740         &$test_volid($snap->{vmstate}, 0);
2741         foreach_drive($snap, sub {
2742             my ($ds, $drive) = @_;
2743             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2744         });
2745     }
2746
2747     foreach my $volid (keys %$volhash) {
2748         &$func($volid, $volhash->{$volid});
2749     }
2750 }
2751
2752 sub vga_conf_has_spice {
2753     my ($vga) = @_;
2754
2755     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2756
2757     return $1 || 1;
2758 }
2759
2760 sub config_to_command {
2761     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2762
2763     my $cmd = [];
2764     my $globalFlags = [];
2765     my $machineFlags = [];
2766     my $rtcFlags = [];
2767     my $cpuFlags = [];
2768     my $devices = [];
2769     my $pciaddr = '';
2770     my $bridges = {};
2771     my $kvmver = kvm_user_version();
2772     my $vernum = 0; # unknown
2773     my $ostype = $conf->{ostype};
2774     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2775         $vernum = $1*1000000+$2*1000;
2776     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2777         $vernum = $1*1000000+$2*1000+$3;
2778     }
2779
2780     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2781
2782     my $have_ovz = -f '/proc/vz/vestat';
2783
2784     my $q35 = machine_type_is_q35($conf);
2785     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2786     my $machine_type = $forcemachine || $conf->{machine};
2787     my $use_old_bios_files = undef;
2788     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2789
2790     my $cpuunits = defined($conf->{cpuunits}) ?
2791             $conf->{cpuunits} : $defaults->{cpuunits};
2792
2793     push @$cmd, '/usr/bin/systemd-run';
2794     push @$cmd, '--scope';
2795     push @$cmd, '--slice', "qemu";
2796     push @$cmd, '--unit', $vmid;
2797     # set KillMode=none, so that systemd don't kill those scopes
2798     # at shutdown (pve-manager service should stop the VMs instead)
2799     push @$cmd, '-p', "KillMode=none";
2800     push @$cmd, '-p', "CPUShares=$cpuunits";
2801     if ($conf->{cpulimit}) {
2802         my $cpulimit = int($conf->{cpulimit} * 100);
2803         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2804     }
2805
2806     push @$cmd, '/usr/bin/kvm';
2807
2808     push @$cmd, '-id', $vmid;
2809
2810     my $use_virtio = 0;
2811
2812     my $qmpsocket = qmp_socket($vmid);
2813     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2814     push @$cmd, '-mon', "chardev=qmp,mode=control";
2815
2816
2817     push @$cmd, '-pidfile' , pidfile_name($vmid);
2818
2819     push @$cmd, '-daemonize';
2820
2821     if ($conf->{smbios1}) {
2822         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2823     }
2824
2825     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2826         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2827         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2828         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2829         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2830         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2831         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2832     }
2833
2834     if ($q35) {
2835         # the q35 chipset support native usb2, so we enable usb controller
2836         # by default for this machine type
2837         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2838     } else {
2839         $pciaddr = print_pci_addr("piix3", $bridges);
2840         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2841
2842         my $use_usb2 = 0;
2843         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2844             next if !$conf->{"usb$i"};
2845             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2846             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2847             $use_usb2 = 1;
2848         }
2849         # include usb device config
2850         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2851     }
2852
2853     # add usb3 controller if needed
2854
2855     my $use_usb3 = 0;
2856     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2857         next if !$conf->{"usb$i"};
2858         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2859         next if !$d || !$d->{usb3};
2860         $use_usb3 = 1;
2861     }
2862
2863     $pciaddr = print_pci_addr("xhci", $bridges);
2864     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2865
2866     my $vga = $conf->{vga};
2867
2868     my $qxlnum = vga_conf_has_spice($vga);
2869     $vga = 'qxl' if $qxlnum;
2870
2871     if (!$vga) {
2872         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2873                                 $conf->{ostype} eq 'win7' ||
2874                                 $conf->{ostype} eq 'w2k8')) {
2875             $vga = 'std';
2876         } else {
2877             $vga = 'cirrus';
2878         }
2879     }
2880
2881     # enable absolute mouse coordinates (needed by vnc)
2882     my $tablet;
2883     if (defined($conf->{tablet})) {
2884         $tablet = $conf->{tablet};
2885     } else {
2886         $tablet = $defaults->{tablet};
2887         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2888         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2889     }
2890
2891     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2892
2893     my $kvm_off = 0;
2894     # host pci devices
2895     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2896         my $d = parse_hostpci($conf->{"hostpci$i"});
2897         next if !$d;
2898
2899         my $pcie = $d->{pcie};
2900         if($pcie){
2901             die "q35 machine model is not enabled" if !$q35;
2902             $pciaddr = print_pcie_addr("hostpci$i");
2903         }else{
2904             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2905         }
2906
2907         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2908         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2909         if ($xvga && $xvga ne '') {
2910             $kvm_off = 1;
2911             $vga = 'none';
2912             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2913                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2914             }
2915             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2916                 $xvga = "";
2917             }
2918         }
2919         my $pcidevices = $d->{pciid};
2920         my $multifunction = 1 if @$pcidevices > 1;
2921
2922         my $j=0;
2923         foreach my $pcidevice (@$pcidevices) {
2924
2925             my $id = "hostpci$i";
2926             $id .= ".$j" if $multifunction;
2927             my $addr = $pciaddr;
2928             $addr .= ".$j" if $multifunction;
2929             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2930
2931             if($j == 0){
2932                 $devicestr .= "$rombar$xvga";
2933                 $devicestr .= ",multifunction=on" if $multifunction;
2934             }
2935
2936             push @$devices, '-device', $devicestr;
2937             $j++;
2938         }
2939     }
2940
2941     # usb devices
2942     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2943         next if !$conf->{"usb$i"};
2944         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2945         next if !$d;
2946
2947         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2948         my $usbbus = '';
2949         if (defined($d->{usb3}) && $d->{usb3}) {
2950             $usbbus = ',bus=xhci.0';
2951         }
2952
2953         if (defined($d->{host})) {
2954             $d = parse_usb_device($d->{host});
2955             if (defined($d->{vendorid}) && defined($d->{productid})) {
2956                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2957             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2958                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2959             } elsif (defined($d->{spice}) && $d->{spice}) {
2960                 # usb redir support for spice, currently no usb3
2961                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2962                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2963             }
2964         }
2965     }
2966
2967     # serial devices
2968     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2969         if (my $path = $conf->{"serial$i"}) {
2970             if ($path eq 'socket') {
2971                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2972                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2973                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2974             } else {
2975                 die "no such serial device\n" if ! -c $path;
2976                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2977                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2978             }
2979         }
2980     }
2981
2982     # parallel devices
2983     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2984         if (my $path = $conf->{"parallel$i"}) {
2985             die "no such parallel device\n" if ! -c $path;
2986             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2987             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2988             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2989         }
2990     }
2991
2992     my $vmname = $conf->{name} || "vm$vmid";
2993
2994     push @$cmd, '-name', $vmname;
2995
2996     my $sockets = 1;
2997     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2998     $sockets = $conf->{sockets} if  $conf->{sockets};
2999
3000     my $cores = $conf->{cores} || 1;
3001
3002     my $maxcpus = $sockets * $cores;
3003
3004     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3005
3006     my $allowed_vcpus = $cpuinfo->{cpus};
3007
3008     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3009         if ($allowed_vcpus < $maxcpus);
3010
3011     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3012
3013     push @$cmd, '-nodefaults';
3014
3015     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3016
3017     my $bootindex_hash = {};
3018     my $i = 1;
3019     foreach my $o (split(//, $bootorder)) {
3020         $bootindex_hash->{$o} = $i*100;
3021         $i++;
3022     }
3023
3024     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3025
3026     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3027
3028     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3029
3030     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3031
3032     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3033         my $socket = vnc_socket($vmid);
3034         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3035     } else {
3036         push @$cmd, '-nographic';
3037     }
3038
3039     # time drift fix
3040     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3041
3042     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3043     my $useLocaltime = $conf->{localtime};
3044
3045     if ($ostype) {
3046         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3047
3048         if ($ostype =~ m/^w/) { # windows
3049             $useLocaltime = 1 if !defined($conf->{localtime});
3050
3051             # use time drift fix when acpi is enabled
3052             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3053                 $tdf = 1 if !defined($conf->{tdf});
3054             }
3055         }
3056
3057         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3058             $ostype eq 'wvista') {
3059             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3060             push @$cmd, '-no-hpet';
3061             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3062                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3063                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3064                 push @$cpuFlags , 'hv_time' if !$nokvm;
3065
3066             } else {
3067                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3068             }
3069         }
3070
3071         if ($ostype eq 'win7' || $ostype eq 'win8') {
3072             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3073         }
3074     }
3075
3076     push @$rtcFlags, 'driftfix=slew' if $tdf;
3077
3078     if ($nokvm) {
3079         push @$machineFlags, 'accel=tcg';
3080     } else {
3081         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3082     }
3083
3084     if ($machine_type) {
3085         push @$machineFlags, "type=${machine_type}";
3086     }
3087
3088     if ($conf->{startdate}) {
3089         push @$rtcFlags, "base=$conf->{startdate}";
3090     } elsif ($useLocaltime) {
3091         push @$rtcFlags, 'base=localtime';
3092     }
3093
3094     my $cpu = $nokvm ? "qemu64" : "kvm64";
3095     if (my $cputype = $conf->{cpu}) {
3096         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3097             or die "Cannot parse cpu description: $cputype\n";
3098         $cpu = $cpuconf->{cputype};
3099         $kvm_off = 1 if $cpuconf->{hidden};
3100     }
3101
3102     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3103
3104     push @$cpuFlags , '-x2apic'
3105         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3106
3107     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3108
3109     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3110
3111     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3112
3113         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3114         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3115     }
3116
3117     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3118
3119     push @$cpuFlags, 'kvm=off' if $kvm_off;
3120
3121     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3122
3123     push @$cmd, '-cpu', $cpu;
3124
3125     my $memory = $conf->{memory} || $defaults->{memory};
3126     my $static_memory = 0;
3127     my $dimm_memory = 0;
3128
3129     if ($hotplug_features->{memory}) {
3130         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3131         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3132         $static_memory = $STATICMEM;
3133         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3134         $dimm_memory = $memory - $static_memory;
3135         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3136
3137     } else {
3138
3139         $static_memory = $memory;
3140         push @$cmd, '-m', $static_memory;
3141     }
3142
3143     if ($conf->{numa}) {
3144
3145         my $numa_totalmemory = undef;
3146         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3147             next if !$conf->{"numa$i"};
3148             my $numa = parse_numa($conf->{"numa$i"});
3149             next if !$numa;
3150             # memory
3151             die "missing numa node$i memory value\n" if !$numa->{memory};
3152             my $numa_memory = $numa->{memory};
3153             $numa_totalmemory += $numa_memory;
3154             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3155
3156             # cpus
3157             my $cpus_start = $numa->{cpus}->{start};
3158             die "missing numa node$i cpus\n" if !defined($cpus_start);
3159             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3160             my $cpus = $cpus_start;
3161             if (defined($cpus_end)) {
3162                 $cpus .= "-$cpus_end";
3163                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3164             }
3165
3166             # hostnodes
3167             my $hostnodes_start = $numa->{hostnodes}->{start};
3168             if (defined($hostnodes_start)) {
3169                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3170                 my $hostnodes = $hostnodes_start;
3171                 if (defined($hostnodes_end)) {
3172                     $hostnodes .= "-$hostnodes_end";
3173                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3174                 }
3175
3176                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3177                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3178                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3179                 }
3180
3181                 # policy
3182                 my $policy = $numa->{policy};
3183                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3184                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3185             }
3186
3187             push @$cmd, '-object', $numa_object;
3188             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3189         }
3190
3191         die "total memory for NUMA nodes must be equal to vm static memory\n"
3192             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3193
3194         #if no custom tology, we split memory and cores across numa nodes
3195         if(!$numa_totalmemory) {
3196
3197             my $numa_memory = ($static_memory / $sockets) . "M";
3198
3199             for (my $i = 0; $i < $sockets; $i++)  {
3200
3201                 my $cpustart = ($cores * $i);
3202                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3203                 my $cpus = $cpustart;
3204                 $cpus .= "-$cpuend" if $cpuend;
3205
3206                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3207                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3208             }
3209         }
3210     }
3211
3212     if ($hotplug_features->{memory}) {
3213         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3214             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3215             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3216             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3217
3218             #if dimm_memory is not aligned to dimm map
3219             if($current_size > $memory) {
3220                  $conf->{memory} = $current_size;
3221                  write_config($vmid, $conf);
3222             }
3223         });
3224     }
3225
3226     push @$cmd, '-S' if $conf->{freeze};
3227
3228     # set keyboard layout
3229     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3230     push @$cmd, '-k', $kb if $kb;
3231
3232     # enable sound
3233     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3234     #push @$cmd, '-soundhw', 'es1370';
3235     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3236
3237     if($conf->{agent}) {
3238         my $qgasocket = qmp_socket($vmid, 1);
3239         my $pciaddr = print_pci_addr("qga0", $bridges);
3240         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3241         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3242         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3243     }
3244
3245     my $spice_port;
3246
3247     if ($qxlnum) {
3248         if ($qxlnum > 1) {
3249             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3250                 for(my $i = 1; $i < $qxlnum; $i++){
3251                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3252                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3253                 }
3254             } else {
3255                 # assume other OS works like Linux
3256                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3257                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3258             }
3259         }
3260
3261         my $pciaddr = print_pci_addr("spice", $bridges);
3262
3263         my $nodename = PVE::INotify::nodename();
3264         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3265         $spice_port = PVE::Tools::next_spice_port($pfamily);
3266
3267         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3268
3269         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3270         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3271         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3272     }
3273
3274     # enable balloon by default, unless explicitly disabled
3275     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3276         $pciaddr = print_pci_addr("balloon0", $bridges);
3277         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3278     }
3279
3280     if ($conf->{watchdog}) {
3281         my $wdopts = parse_watchdog($conf->{watchdog});
3282         $pciaddr = print_pci_addr("watchdog", $bridges);
3283         my $watchdog = $wdopts->{model} || 'i6300esb';
3284         push @$devices, '-device', "$watchdog$pciaddr";
3285         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3286     }
3287
3288     my $vollist = [];
3289     my $scsicontroller = {};
3290     my $ahcicontroller = {};
3291     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3292
3293     # Add iscsi initiator name if available
3294     if (my $initiator = get_initiator_name()) {
3295         push @$devices, '-iscsi', "initiator-name=$initiator";
3296     }
3297
3298     foreach_drive($conf, sub {
3299         my ($ds, $drive) = @_;
3300
3301         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3302             push @$vollist, $drive->{file};
3303         }
3304
3305         $use_virtio = 1 if $ds =~ m/^virtio/;
3306
3307         if (drive_is_cdrom ($drive)) {
3308             if ($bootindex_hash->{d}) {
3309                 $drive->{bootindex} = $bootindex_hash->{d};
3310                 $bootindex_hash->{d} += 1;
3311             }
3312         } else {
3313             if ($bootindex_hash->{c}) {
3314                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3315                 $bootindex_hash->{c} += 1;
3316             }
3317         }
3318
3319         if($drive->{interface} eq 'virtio'){
3320            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3321         }
3322
3323         if ($drive->{interface} eq 'scsi') {
3324
3325             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3326
3327             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3328             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3329
3330             my $iothread = '';
3331             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3332                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3333                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3334             }
3335
3336             my $queues = '';
3337             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3338                 $queues = ",num_queues=$drive->{queues}";
3339             }
3340
3341             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3342             $scsicontroller->{$controller}=1;
3343         }
3344
3345         if ($drive->{interface} eq 'sata') {
3346            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3347            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3348            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3349            $ahcicontroller->{$controller}=1;
3350         }
3351
3352         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3353         push @$devices, '-drive',$drive_cmd;
3354         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3355     });
3356
3357     for (my $i = 0; $i < $MAX_NETS; $i++) {
3358          next if !$conf->{"net$i"};
3359          my $d = parse_net($conf->{"net$i"});
3360          next if !$d;
3361
3362          $use_virtio = 1 if $d->{model} eq 'virtio';
3363
3364          if ($bootindex_hash->{n}) {
3365             $d->{bootindex} = $bootindex_hash->{n};
3366             $bootindex_hash->{n} += 1;
3367          }
3368
3369          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3370          push @$devices, '-netdev', $netdevfull;
3371
3372          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3373          push @$devices, '-device', $netdevicefull;
3374     }
3375
3376     if (!$q35) {
3377         # add pci bridges
3378         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3379            $bridges->{1} = 1;
3380            $bridges->{2} = 1;
3381         }
3382
3383         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3384
3385         while (my ($k, $v) = each %$bridges) {
3386             $pciaddr = print_pci_addr("pci.$k");
3387             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3388         }
3389     }
3390
3391     # add custom args
3392     if ($conf->{args}) {
3393         my $aa = PVE::Tools::split_args($conf->{args});
3394         push @$cmd, @$aa;
3395     }
3396
3397     push @$cmd, @$devices;
3398     push @$cmd, '-rtc', join(',', @$rtcFlags)
3399         if scalar(@$rtcFlags);
3400     push @$cmd, '-machine', join(',', @$machineFlags)
3401         if scalar(@$machineFlags);
3402     push @$cmd, '-global', join(',', @$globalFlags)
3403         if scalar(@$globalFlags);
3404
3405     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3406 }
3407
3408 sub vnc_socket {
3409     my ($vmid) = @_;
3410     return "${var_run_tmpdir}/$vmid.vnc";
3411 }
3412
3413 sub spice_port {
3414     my ($vmid) = @_;
3415
3416     my $res = vm_mon_cmd($vmid, 'query-spice');
3417
3418     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3419 }
3420
3421 sub qmp_socket {
3422     my ($vmid, $qga) = @_;
3423     my $sockettype = $qga ? 'qga' : 'qmp';
3424     return "${var_run_tmpdir}/$vmid.$sockettype";
3425 }
3426
3427 sub pidfile_name {
3428     my ($vmid) = @_;
3429     return "${var_run_tmpdir}/$vmid.pid";
3430 }
3431
3432 sub vm_devices_list {
3433     my ($vmid) = @_;
3434
3435     my $res = vm_mon_cmd($vmid, 'query-pci');
3436     my $devices = {};
3437     foreach my $pcibus (@$res) {
3438         foreach my $device (@{$pcibus->{devices}}) {
3439             next if !$device->{'qdev_id'};
3440             if ($device->{'pci_bridge'}) {
3441                 $devices->{$device->{'qdev_id'}} = 1;
3442                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3443                     next if !$bridge_device->{'qdev_id'};
3444                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3445                     $devices->{$device->{'qdev_id'}}++;
3446                 }
3447             } else {
3448                 $devices->{$device->{'qdev_id'}} = 1;
3449             }
3450         }
3451     }
3452
3453     my $resblock = vm_mon_cmd($vmid, 'query-block');
3454     foreach my $block (@$resblock) {
3455         if($block->{device} =~ m/^drive-(\S+)/){
3456                 $devices->{$1} = 1;
3457         }
3458     }
3459
3460     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3461     foreach my $mice (@$resmice) {
3462         if ($mice->{name} eq 'QEMU HID Tablet') {
3463             $devices->{tablet} = 1;
3464             last;
3465         }
3466     }
3467
3468     return $devices;
3469 }
3470
3471 sub vm_deviceplug {
3472     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3473
3474     my $q35 = machine_type_is_q35($conf);
3475
3476     my $devices_list = vm_devices_list($vmid);
3477     return 1 if defined($devices_list->{$deviceid});
3478
3479     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3480
3481     if ($deviceid eq 'tablet') {
3482
3483         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3484
3485     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3486
3487         qemu_iothread_add($vmid, $deviceid, $device);
3488
3489         qemu_driveadd($storecfg, $vmid, $device);
3490         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3491
3492         qemu_deviceadd($vmid, $devicefull);
3493         eval { qemu_deviceaddverify($vmid, $deviceid); };
3494         if (my $err = $@) {
3495             eval { qemu_drivedel($vmid, $deviceid); };
3496             warn $@ if $@;
3497             die $err;
3498         }
3499
3500     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3501
3502
3503         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3504         my $pciaddr = print_pci_addr($deviceid);
3505         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3506
3507         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3508
3509         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3510             qemu_iothread_add($vmid, $deviceid, $device);
3511             $devicefull .= ",iothread=iothread-$deviceid";
3512         }
3513
3514         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3515             $devicefull .= ",num_queues=$device->{queues}";
3516         }
3517
3518         qemu_deviceadd($vmid, $devicefull);
3519         qemu_deviceaddverify($vmid, $deviceid);
3520
3521     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3522
3523         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3524         qemu_driveadd($storecfg, $vmid, $device);
3525
3526         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3527         eval { qemu_deviceadd($vmid, $devicefull); };
3528         if (my $err = $@) {
3529             eval { qemu_drivedel($vmid, $deviceid); };
3530             warn $@ if $@;
3531             die $err;
3532         }
3533
3534     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3535
3536         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3537
3538         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3539         my $use_old_bios_files = undef;
3540         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3541
3542         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3543         qemu_deviceadd($vmid, $netdevicefull);
3544         eval { qemu_deviceaddverify($vmid, $deviceid); };
3545         if (my $err = $@) {
3546             eval { qemu_netdevdel($vmid, $deviceid); };
3547             warn $@ if $@;
3548             die $err;
3549         }
3550
3551     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3552
3553         my $bridgeid = $2;
3554         my $pciaddr = print_pci_addr($deviceid);
3555         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3556
3557         qemu_deviceadd($vmid, $devicefull);
3558         qemu_deviceaddverify($vmid, $deviceid);
3559
3560     } else {
3561         die "can't hotplug device '$deviceid'\n";
3562     }
3563
3564     return 1;
3565 }
3566
3567 # fixme: this should raise exceptions on error!
3568 sub vm_deviceunplug {
3569     my ($vmid, $conf, $deviceid) = @_;
3570
3571     my $devices_list = vm_devices_list($vmid);
3572     return 1 if !defined($devices_list->{$deviceid});
3573
3574     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3575
3576     if ($deviceid eq 'tablet') {
3577
3578         qemu_devicedel($vmid, $deviceid);
3579
3580     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3581
3582         qemu_devicedel($vmid, $deviceid);
3583         qemu_devicedelverify($vmid, $deviceid);
3584         qemu_drivedel($vmid, $deviceid);
3585         qemu_iothread_del($conf, $vmid, $deviceid);
3586
3587     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3588
3589         qemu_devicedel($vmid, $deviceid);
3590         qemu_devicedelverify($vmid, $deviceid);
3591         qemu_iothread_del($conf, $vmid, $deviceid);
3592
3593     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3594
3595         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3596         my $device = parse_drive($deviceid, $conf->{$deviceid});
3597         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3598
3599         qemu_devicedel($vmid, $deviceid);
3600         qemu_drivedel($vmid, $deviceid);
3601         qemu_deletescsihw($conf, $vmid, $deviceid);
3602
3603     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3604
3605         qemu_devicedel($vmid, $deviceid);
3606         qemu_devicedelverify($vmid, $deviceid);
3607         qemu_netdevdel($vmid, $deviceid);
3608
3609     } else {
3610         die "can't unplug device '$deviceid'\n";
3611     }
3612
3613     return 1;
3614 }
3615
3616 sub qemu_deviceadd {
3617     my ($vmid, $devicefull) = @_;
3618
3619     $devicefull = "driver=".$devicefull;
3620     my %options =  split(/[=,]/, $devicefull);
3621
3622     vm_mon_cmd($vmid, "device_add" , %options);
3623 }
3624
3625 sub qemu_devicedel {
3626     my ($vmid, $deviceid) = @_;
3627
3628     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3629 }
3630
3631 sub qemu_iothread_add {
3632     my($vmid, $deviceid, $device) = @_;
3633
3634     if ($device->{iothread}) {
3635         my $iothreads = vm_iothreads_list($vmid);
3636         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3637     }
3638 }
3639
3640 sub qemu_iothread_del {
3641     my($conf, $vmid, $deviceid) = @_;
3642
3643     my $device = parse_drive($deviceid, $conf->{$deviceid});
3644     if ($device->{iothread}) {
3645         my $iothreads = vm_iothreads_list($vmid);
3646         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3647     }
3648 }
3649
3650 sub qemu_objectadd {
3651     my($vmid, $objectid, $qomtype) = @_;
3652
3653     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3654
3655     return 1;
3656 }
3657
3658 sub qemu_objectdel {
3659     my($vmid, $objectid) = @_;
3660
3661     vm_mon_cmd($vmid, "object-del", id => $objectid);
3662
3663     return 1;
3664 }
3665
3666 sub qemu_driveadd {
3667     my ($storecfg, $vmid, $device) = @_;
3668
3669     my $drive = print_drive_full($storecfg, $vmid, $device);
3670     $drive =~ s/\\/\\\\/g;
3671     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3672
3673     # If the command succeeds qemu prints: "OK"
3674     return 1 if $ret =~ m/OK/s;
3675
3676     die "adding drive failed: $ret\n";
3677 }
3678
3679 sub qemu_drivedel {
3680     my($vmid, $deviceid) = @_;
3681
3682     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3683     $ret =~ s/^\s+//;
3684
3685     return 1 if $ret eq "";
3686
3687     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3688     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3689
3690     die "deleting drive $deviceid failed : $ret\n";
3691 }
3692
3693 sub qemu_deviceaddverify {
3694     my ($vmid, $deviceid) = @_;
3695
3696     for (my $i = 0; $i <= 5; $i++) {
3697          my $devices_list = vm_devices_list($vmid);
3698          return 1 if defined($devices_list->{$deviceid});
3699          sleep 1;
3700     }
3701
3702     die "error on hotplug device '$deviceid'\n";
3703 }
3704
3705
3706 sub qemu_devicedelverify {
3707     my ($vmid, $deviceid) = @_;
3708
3709     # need to verify that the device is correctly removed as device_del
3710     # is async and empty return is not reliable
3711
3712     for (my $i = 0; $i <= 5; $i++) {
3713          my $devices_list = vm_devices_list($vmid);
3714          return 1 if !defined($devices_list->{$deviceid});
3715          sleep 1;
3716     }
3717
3718     die "error on hot-unplugging device '$deviceid'\n";
3719 }
3720
3721 sub qemu_findorcreatescsihw {
3722     my ($storecfg, $conf, $vmid, $device) = @_;
3723
3724     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3725
3726     my $scsihwid="$controller_prefix$controller";
3727     my $devices_list = vm_devices_list($vmid);
3728
3729     if(!defined($devices_list->{$scsihwid})) {
3730         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3731     }
3732
3733     return 1;
3734 }
3735
3736 sub qemu_deletescsihw {
3737     my ($conf, $vmid, $opt) = @_;
3738
3739     my $device = parse_drive($opt, $conf->{$opt});
3740
3741     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3742         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3743         return 1;
3744     }
3745
3746     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3747
3748     my $devices_list = vm_devices_list($vmid);
3749     foreach my $opt (keys %{$devices_list}) {
3750         if (PVE::QemuServer::valid_drivename($opt)) {
3751             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3752             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3753                 return 1;
3754             }
3755         }
3756     }
3757
3758     my $scsihwid="scsihw$controller";
3759
3760     vm_deviceunplug($vmid, $conf, $scsihwid);
3761
3762     return 1;
3763 }
3764
3765 sub qemu_add_pci_bridge {
3766     my ($storecfg, $conf, $vmid, $device) = @_;
3767
3768     my $bridges = {};
3769
3770     my $bridgeid;
3771
3772     print_pci_addr($device, $bridges);
3773
3774     while (my ($k, $v) = each %$bridges) {
3775         $bridgeid = $k;
3776     }
3777     return 1 if !defined($bridgeid) || $bridgeid < 1;
3778
3779     my $bridge = "pci.$bridgeid";
3780     my $devices_list = vm_devices_list($vmid);
3781
3782     if (!defined($devices_list->{$bridge})) {
3783         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3784     }
3785
3786     return 1;
3787 }
3788
3789 sub qemu_set_link_status {
3790     my ($vmid, $device, $up) = @_;
3791
3792     vm_mon_cmd($vmid, "set_link", name => $device,
3793                up => $up ? JSON::true : JSON::false);
3794 }
3795
3796 sub qemu_netdevadd {
3797     my ($vmid, $conf, $device, $deviceid) = @_;
3798
3799     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3800     my %options =  split(/[=,]/, $netdev);
3801
3802     vm_mon_cmd($vmid, "netdev_add",  %options);
3803     return 1;
3804 }
3805
3806 sub qemu_netdevdel {
3807     my ($vmid, $deviceid) = @_;
3808
3809     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3810 }
3811
3812 sub qemu_cpu_hotplug {
3813     my ($vmid, $conf, $vcpus) = @_;
3814
3815     my $sockets = 1;
3816     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3817     $sockets = $conf->{sockets} if  $conf->{sockets};
3818     my $cores = $conf->{cores} || 1;
3819     my $maxcpus = $sockets * $cores;
3820
3821     $vcpus = $maxcpus if !$vcpus;
3822
3823     die "you can't add more vcpus than maxcpus\n"
3824         if $vcpus > $maxcpus;
3825
3826     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3827     die "online cpu unplug is not yet possible\n"
3828         if $vcpus < $currentvcpus;
3829
3830     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3831     die "vcpus in running vm is different than configuration\n"
3832         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3833
3834     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3835         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3836     }
3837 }
3838
3839 sub qemu_memory_hotplug {
3840     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3841
3842     return $value if !check_running($vmid);
3843
3844     my $memory = $conf->{memory} || $defaults->{memory};
3845     $value = $defaults->{memory} if !$value;
3846     return $value if $value == $memory;
3847
3848     my $static_memory = $STATICMEM;
3849     my $dimm_memory = $memory - $static_memory;
3850
3851     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3852     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3853
3854
3855     my $sockets = 1;
3856     $sockets = $conf->{sockets} if $conf->{sockets};
3857
3858     if($value > $memory) {
3859
3860         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3861             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3862
3863                 return if $current_size <= $conf->{memory};
3864
3865                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3866                 if (my $err = $@) {
3867                     eval { qemu_objectdel($vmid, "mem-$name"); };
3868                     die $err;
3869                 }
3870
3871                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3872                 if (my $err = $@) {
3873                     eval { qemu_objectdel($vmid, "mem-$name"); };
3874                     die $err;
3875                 }
3876                 #update conf after each succesful module hotplug
3877                 $conf->{memory} = $current_size;
3878                 write_config($vmid, $conf);
3879         });
3880
3881     } else {
3882
3883         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3884             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3885
3886                 return if $current_size >= $conf->{memory};
3887                 print "try to unplug memory dimm $name\n";
3888
3889                 my $retry = 0;
3890                 while (1) {
3891                     eval { qemu_devicedel($vmid, $name) };
3892                     sleep 3;
3893                     my $dimm_list = qemu_dimm_list($vmid);
3894                     last if !$dimm_list->{$name};
3895                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3896                     $retry++;
3897                 }
3898
3899                 #update conf after each succesful module unplug
3900                 $conf->{memory} = $current_size;
3901
3902                 eval { qemu_objectdel($vmid, "mem-$name"); };
3903                 write_config($vmid, $conf);
3904         });
3905     }
3906 }
3907
3908 sub qemu_dimm_list {
3909     my ($vmid) = @_;
3910
3911     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3912     my $dimms = {};
3913
3914     foreach my $dimm (@$dimmarray) {
3915
3916         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3917         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3918         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3919         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3920         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3921     }
3922     return $dimms;
3923 }
3924
3925 sub qemu_block_set_io_throttle {
3926     my ($vmid, $deviceid,
3927         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3928         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3929
3930     return if !check_running($vmid) ;
3931
3932     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3933         bps => int($bps),
3934         bps_rd => int($bps_rd),
3935         bps_wr => int($bps_wr),
3936         iops => int($iops),
3937         iops_rd => int($iops_rd),
3938         iops_wr => int($iops_wr),
3939         bps_max => int($bps_max),
3940         bps_rd_max => int($bps_rd_max),
3941         bps_wr_max => int($bps_wr_max),
3942         iops_max => int($iops_max),
3943         iops_rd_max => int($iops_rd_max),
3944         iops_wr_max => int($iops_wr_max)
3945     );
3946
3947 }
3948
3949 # old code, only used to shutdown old VM after update
3950 sub __read_avail {
3951     my ($fh, $timeout) = @_;
3952
3953     my $sel = new IO::Select;
3954     $sel->add($fh);
3955
3956     my $res = '';
3957     my $buf;
3958
3959     my @ready;
3960     while (scalar (@ready = $sel->can_read($timeout))) {
3961         my $count;
3962         if ($count = $fh->sysread($buf, 8192)) {
3963             if ($buf =~ /^(.*)\(qemu\) $/s) {
3964                 $res .= $1;
3965                 last;
3966             } else {
3967                 $res .= $buf;
3968             }
3969         } else {
3970             if (!defined($count)) {
3971                 die "$!\n";
3972             }
3973             last;
3974         }
3975     }
3976
3977     die "monitor read timeout\n" if !scalar(@ready);
3978
3979     return $res;
3980 }
3981
3982 # old code, only used to shutdown old VM after update
3983 sub vm_monitor_command {
3984     my ($vmid, $cmdstr, $nocheck) = @_;
3985
3986     my $res;
3987
3988     eval {
3989         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3990
3991         my $sname = "${var_run_tmpdir}/$vmid.mon";
3992
3993         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3994             die "unable to connect to VM $vmid socket - $!\n";
3995
3996         my $timeout = 3;
3997
3998         # hack: migrate sometime blocks the monitor (when migrate_downtime
3999         # is set)
4000         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4001             $timeout = 60*60; # 1 hour
4002         }
4003
4004         # read banner;
4005         my $data = __read_avail($sock, $timeout);
4006
4007         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4008             die "got unexpected qemu monitor banner\n";
4009         }
4010
4011         my $sel = new IO::Select;
4012         $sel->add($sock);
4013
4014         if (!scalar(my @ready = $sel->can_write($timeout))) {
4015             die "monitor write error - timeout";
4016         }
4017
4018         my $fullcmd = "$cmdstr\r";
4019
4020         # syslog('info', "VM $vmid monitor command: $cmdstr");
4021
4022         my $b;
4023         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4024             die "monitor write error - $!";
4025         }
4026
4027         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4028
4029         $timeout = 20;
4030
4031         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4032             $timeout = 60*60; # 1 hour
4033         } elsif ($cmdstr =~ m/^(eject|change)/) {
4034             $timeout = 60; # note: cdrom mount command is slow
4035         }
4036         if ($res = __read_avail($sock, $timeout)) {
4037
4038             my @lines = split("\r?\n", $res);
4039
4040             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4041
4042             $res = join("\n", @lines);
4043             $res .= "\n";
4044         }
4045     };
4046
4047     my $err = $@;
4048
4049     if ($err) {
4050         syslog("err", "VM $vmid monitor command failed - $err");
4051         die $err;
4052     }
4053
4054     return $res;
4055 }
4056
4057 sub qemu_block_resize {
4058     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4059
4060     my $running = check_running($vmid);
4061
4062     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4063
4064     return if !$running;
4065
4066     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4067
4068 }
4069
4070 sub qemu_volume_snapshot {
4071     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4072
4073     my $running = check_running($vmid);
4074
4075     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4076         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4077     } else {
4078         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4079     }
4080 }
4081
4082 sub qemu_volume_snapshot_delete {
4083     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4084
4085     my $running = check_running($vmid);
4086
4087     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4088
4089     return if !$running;
4090
4091     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4092 }
4093
4094 sub set_migration_caps {
4095     my ($vmid) = @_;
4096
4097     my $cap_ref = [];
4098
4099     my $enabled_cap = {
4100         "auto-converge" => 1,
4101         "xbzrle" => 1,
4102         "x-rdma-pin-all" => 0,
4103         "zero-blocks" => 0,
4104         "compress" => 0
4105     };
4106
4107     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4108
4109     for my $supported_capability (@$supported_capabilities) {
4110         push @$cap_ref, {
4111             capability => $supported_capability->{capability},
4112             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4113         };
4114     }
4115
4116     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4117 }
4118
4119 my $fast_plug_option = {
4120     'lock' => 1,
4121     'name' => 1,
4122     'onboot' => 1,
4123     'shares' => 1,
4124     'startup' => 1,
4125     'description' => 1,
4126 };
4127
4128 # hotplug changes in [PENDING]
4129 # $selection hash can be used to only apply specified options, for
4130 # example: { cores => 1 } (only apply changed 'cores')
4131 # $errors ref is used to return error messages
4132 sub vmconfig_hotplug_pending {
4133     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4134
4135     my $defaults = load_defaults();
4136
4137     # commit values which do not have any impact on running VM first
4138     # Note: those option cannot raise errors, we we do not care about
4139     # $selection and always apply them.
4140
4141     my $add_error = sub {
4142         my ($opt, $msg) = @_;
4143         $errors->{$opt} = "hotplug problem - $msg";
4144     };
4145
4146     my $changes = 0;
4147     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4148         if ($fast_plug_option->{$opt}) {
4149             $conf->{$opt} = $conf->{pending}->{$opt};
4150             delete $conf->{pending}->{$opt};
4151             $changes = 1;
4152         }
4153     }
4154
4155     if ($changes) {
4156         write_config($vmid, $conf);
4157         $conf = load_config($vmid); # update/reload
4158     }
4159
4160     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4161
4162     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4163     while (my ($opt, $force) = each %$pending_delete_hash) {
4164         next if $selection && !$selection->{$opt};
4165         eval {
4166             if ($opt eq 'hotplug') {
4167                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4168             } elsif ($opt eq 'tablet') {
4169                 die "skip\n" if !$hotplug_features->{usb};
4170                 if ($defaults->{tablet}) {
4171                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4172                 } else {
4173                     vm_deviceunplug($vmid, $conf, $opt);
4174                 }
4175             } elsif ($opt eq 'vcpus') {
4176                 die "skip\n" if !$hotplug_features->{cpu};
4177                 qemu_cpu_hotplug($vmid, $conf, undef);
4178             } elsif ($opt eq 'balloon') {
4179                 # enable balloon device is not hotpluggable
4180                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4181             } elsif ($fast_plug_option->{$opt}) {
4182                 # do nothing
4183             } elsif ($opt =~ m/^net(\d+)$/) {
4184                 die "skip\n" if !$hotplug_features->{network};
4185                 vm_deviceunplug($vmid, $conf, $opt);
4186             } elsif (valid_drivename($opt)) {
4187                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4188                 vm_deviceunplug($vmid, $conf, $opt);
4189                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4190             } elsif ($opt =~ m/^memory$/) {
4191                 die "skip\n" if !$hotplug_features->{memory};
4192                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4193             } elsif ($opt eq 'cpuunits') {
4194                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4195             } elsif ($opt eq 'cpulimit') {
4196                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4197             } else {
4198                 die "skip\n";
4199             }
4200         };
4201         if (my $err = $@) {
4202             &$add_error($opt, $err) if $err ne "skip\n";
4203         } else {
4204             # save new config if hotplug was successful
4205             delete $conf->{$opt};
4206             vmconfig_undelete_pending_option($conf, $opt);
4207             write_config($vmid, $conf);
4208             $conf = load_config($vmid); # update/reload
4209         }
4210     }
4211
4212     foreach my $opt (keys %{$conf->{pending}}) {
4213         next if $selection && !$selection->{$opt};
4214         my $value = $conf->{pending}->{$opt};
4215         eval {
4216             if ($opt eq 'hotplug') {
4217                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4218             } elsif ($opt eq 'tablet') {
4219                 die "skip\n" if !$hotplug_features->{usb};
4220                 if ($value == 1) {
4221                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4222                 } elsif ($value == 0) {
4223                     vm_deviceunplug($vmid, $conf, $opt);
4224                 }
4225             } elsif ($opt eq 'vcpus') {
4226                 die "skip\n" if !$hotplug_features->{cpu};
4227                 qemu_cpu_hotplug($vmid, $conf, $value);
4228             } elsif ($opt eq 'balloon') {
4229                 # enable/disable balloning device is not hotpluggable
4230                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4231                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4232                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4233
4234                 # allow manual ballooning if shares is set to zero
4235                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4236                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4237                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4238                 }
4239             } elsif ($opt =~ m/^net(\d+)$/) {
4240                 # some changes can be done without hotplug
4241                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4242                                     $vmid, $opt, $value);
4243             } elsif (valid_drivename($opt)) {
4244                 # some changes can be done without hotplug
4245                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4246                                      $vmid, $opt, $value, 1);
4247             } elsif ($opt =~ m/^memory$/) { #dimms
4248                 die "skip\n" if !$hotplug_features->{memory};
4249                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4250             } elsif ($opt eq 'cpuunits') {
4251                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4252             } elsif ($opt eq 'cpulimit') {
4253                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4254                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4255             } else {
4256                 die "skip\n";  # skip non-hot-pluggable options
4257             }
4258         };
4259         if (my $err = $@) {
4260             &$add_error($opt, $err) if $err ne "skip\n";
4261         } else {
4262             # save new config if hotplug was successful
4263             $conf->{$opt} = $value;
4264             delete $conf->{pending}->{$opt};
4265             write_config($vmid, $conf);
4266             $conf = load_config($vmid); # update/reload
4267         }
4268     }
4269 }
4270
4271 sub try_deallocate_drive {
4272     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4273
4274     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4275         my $volid = $drive->{file};
4276         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4277             my $sid = PVE::Storage::parse_volume_id($volid);
4278             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4279
4280             # check if the disk is really unused
4281             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4282                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4283             PVE::Storage::vdisk_free($storecfg, $volid);
4284             return 1;
4285         } else {
4286             # If vm is not owner of this disk remove from config
4287             return 1;
4288         }
4289     }
4290
4291     return undef;
4292 }
4293
4294 sub vmconfig_delete_or_detach_drive {
4295     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4296
4297     my $drive = parse_drive($opt, $conf->{$opt});
4298
4299     my $rpcenv = PVE::RPCEnvironment::get();
4300     my $authuser = $rpcenv->get_user();
4301
4302     if ($force) {
4303         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4304         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4305     } else {
4306         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4307     }
4308 }
4309
4310 sub vmconfig_apply_pending {
4311     my ($vmid, $conf, $storecfg) = @_;
4312
4313     # cold plug
4314
4315     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4316     while (my ($opt, $force) = each %$pending_delete_hash) {
4317         die "internal error" if $opt =~ m/^unused/;
4318         $conf = load_config($vmid); # update/reload
4319         if (!defined($conf->{$opt})) {
4320             vmconfig_undelete_pending_option($conf, $opt);
4321             write_config($vmid, $conf);
4322         } elsif (valid_drivename($opt)) {
4323             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4324             vmconfig_undelete_pending_option($conf, $opt);
4325             delete $conf->{$opt};
4326             write_config($vmid, $conf);
4327         } else {
4328             vmconfig_undelete_pending_option($conf, $opt);
4329             delete $conf->{$opt};
4330             write_config($vmid, $conf);
4331         }
4332     }
4333
4334     $conf = load_config($vmid); # update/reload
4335
4336     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4337         $conf = load_config($vmid); # update/reload
4338
4339         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4340             # skip if nothing changed
4341         } elsif (valid_drivename($opt)) {
4342             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4343                 if defined($conf->{$opt});
4344             $conf->{$opt} = $conf->{pending}->{$opt};
4345         } else {
4346             $conf->{$opt} = $conf->{pending}->{$opt};
4347         }
4348
4349         delete $conf->{pending}->{$opt};
4350         write_config($vmid, $conf);
4351     }
4352 }
4353
4354 my $safe_num_ne = sub {
4355     my ($a, $b) = @_;
4356
4357     return 0 if !defined($a) && !defined($b);
4358     return 1 if !defined($a);
4359     return 1 if !defined($b);
4360
4361     return $a != $b;
4362 };
4363
4364 my $safe_string_ne = sub {
4365     my ($a, $b) = @_;
4366
4367     return 0 if !defined($a) && !defined($b);
4368     return 1 if !defined($a);
4369     return 1 if !defined($b);
4370
4371     return $a ne $b;
4372 };
4373
4374 sub vmconfig_update_net {
4375     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4376
4377     my $newnet = parse_net($value);
4378
4379     if ($conf->{$opt}) {
4380         my $oldnet = parse_net($conf->{$opt});
4381
4382         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4383             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4384             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4385             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4386
4387             # for non online change, we try to hot-unplug
4388             die "skip\n" if !$hotplug;
4389             vm_deviceunplug($vmid, $conf, $opt);
4390         } else {
4391
4392             die "internal error" if $opt !~ m/net(\d+)/;
4393             my $iface = "tap${vmid}i$1";
4394
4395             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4396                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4397             }
4398
4399             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4400                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4401                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4402                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4403                 PVE::Network::tap_unplug($iface);
4404                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4405             }
4406
4407             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4408                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4409             }
4410
4411             return 1;
4412         }
4413     }
4414
4415     if ($hotplug) {
4416         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4417     } else {
4418         die "skip\n";
4419     }
4420 }
4421
4422 sub vmconfig_update_disk {
4423     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4424
4425     # fixme: do we need force?
4426
4427     my $drive = parse_drive($opt, $value);
4428
4429     if ($conf->{$opt}) {
4430
4431         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4432
4433             my $media = $drive->{media} || 'disk';
4434             my $oldmedia = $old_drive->{media} || 'disk';
4435             die "unable to change media type\n" if $media ne $oldmedia;
4436
4437             if (!drive_is_cdrom($old_drive)) {
4438
4439                 if ($drive->{file} ne $old_drive->{file}) {
4440
4441                     die "skip\n" if !$hotplug;
4442
4443                     # unplug and register as unused
4444                     vm_deviceunplug($vmid, $conf, $opt);
4445                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4446
4447                 } else {
4448                     # update existing disk
4449
4450                     # skip non hotpluggable value
4451                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4452                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4453                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4454                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4455                         die "skip\n";
4456                     }
4457
4458                     # apply throttle
4459                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4460                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4461                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4462                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4463                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4464                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4465                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4466                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4467                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4468                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4469                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4470                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4471
4472                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4473                                                    ($drive->{mbps} || 0)*1024*1024,
4474                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4475                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4476                                                    $drive->{iops} || 0,
4477                                                    $drive->{iops_rd} || 0,
4478                                                    $drive->{iops_wr} || 0,
4479                                                    ($drive->{mbps_max} || 0)*1024*1024,
4480                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4481                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4482                                                    $drive->{iops_max} || 0,
4483                                                    $drive->{iops_rd_max} || 0,
4484                                                    $drive->{iops_wr_max} || 0);
4485
4486                     }
4487
4488                     return 1;
4489                 }
4490
4491             } else { # cdrom
4492
4493                 if ($drive->{file} eq 'none') {
4494                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4495                 } else {
4496                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4497                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4498                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4499                 }
4500
4501                 return 1;
4502             }
4503         }
4504     }
4505
4506     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4507     # hotplug new disks
4508     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4509     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4510 }
4511
4512 sub vm_start {
4513     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4514         $forcemachine, $spice_ticket) = @_;
4515
4516     lock_config($vmid, sub {
4517         my $conf = load_config($vmid, $migratedfrom);
4518
4519         die "you can't start a vm if it's a template\n" if is_template($conf);
4520
4521         check_lock($conf) if !$skiplock;
4522
4523         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4524
4525         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4526             vmconfig_apply_pending($vmid, $conf, $storecfg);
4527             $conf = load_config($vmid); # update/reload
4528         }
4529
4530         my $defaults = load_defaults();
4531
4532         # set environment variable useful inside network script
4533         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4534
4535         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4536
4537         my $migrate_port = 0;
4538         my $migrate_uri;
4539         if ($statefile) {
4540             if ($statefile eq 'tcp') {
4541                 my $localip = "localhost";
4542                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4543                 my $nodename = PVE::INotify::nodename();
4544                 if ($datacenterconf->{migration_unsecure}) {
4545                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4546                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4547                 }
4548                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4549                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4550                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4551                 push @$cmd, '-incoming', $migrate_uri;
4552                 push @$cmd, '-S';
4553             } else {
4554                 push @$cmd, '-loadstate', $statefile;
4555             }
4556         } elsif ($paused) {
4557             push @$cmd, '-S';
4558         }
4559
4560         # host pci devices
4561         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4562           my $d = parse_hostpci($conf->{"hostpci$i"});
4563           next if !$d;
4564           my $pcidevices = $d->{pciid};
4565           foreach my $pcidevice (@$pcidevices) {
4566                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4567
4568                 my $info = pci_device_info("0000:$pciid");
4569                 die "IOMMU not present\n" if !check_iommu_support();
4570                 die "no pci device info for device '$pciid'\n" if !$info;
4571                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4572                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4573           }
4574         }
4575
4576         PVE::Storage::activate_volumes($storecfg, $vollist);
4577
4578         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4579                     umask => 0077); };
4580
4581         if (my $err = $@) {
4582             # deactivate volumes if start fails
4583             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4584             die "start failed: $err";
4585         }
4586
4587         print "migration listens on $migrate_uri\n" if $migrate_uri;
4588
4589         if ($statefile && $statefile ne 'tcp')  {
4590             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4591             warn $@ if $@;
4592         }
4593
4594         if ($migratedfrom) {
4595
4596             eval {
4597                 set_migration_caps($vmid);
4598             };
4599             warn $@ if $@;
4600
4601             if ($spice_port) {
4602                 print "spice listens on port $spice_port\n";
4603                 if ($spice_ticket) {
4604                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4605                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4606                 }
4607             }
4608
4609         } else {
4610
4611             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4612                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4613                     if $conf->{balloon};
4614             }
4615
4616             foreach my $opt (keys %$conf) {
4617                 next if $opt !~  m/^net\d+$/;
4618                 my $nicconf = parse_net($conf->{$opt});
4619                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4620             }
4621         }
4622
4623         vm_mon_cmd_nocheck($vmid, 'qom-set',
4624                     path => "machine/peripheral/balloon0",
4625                     property => "guest-stats-polling-interval",
4626                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4627
4628     });
4629 }
4630
4631 sub vm_mon_cmd {
4632     my ($vmid, $execute, %params) = @_;
4633
4634     my $cmd = { execute => $execute, arguments => \%params };
4635     vm_qmp_command($vmid, $cmd);
4636 }
4637
4638 sub vm_mon_cmd_nocheck {
4639     my ($vmid, $execute, %params) = @_;
4640
4641     my $cmd = { execute => $execute, arguments => \%params };
4642     vm_qmp_command($vmid, $cmd, 1);
4643 }
4644
4645 sub vm_qmp_command {
4646     my ($vmid, $cmd, $nocheck) = @_;
4647
4648     my $res;
4649
4650     my $timeout;
4651     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4652         $timeout = $cmd->{arguments}->{timeout};
4653         delete $cmd->{arguments}->{timeout};
4654     }
4655
4656     eval {
4657         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4658         my $sname = qmp_socket($vmid);
4659         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4660             my $qmpclient = PVE::QMPClient->new();
4661
4662             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4663         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4664             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4665                 if scalar(%{$cmd->{arguments}});
4666             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4667         } else {
4668             die "unable to open monitor socket\n";
4669         }
4670     };
4671     if (my $err = $@) {
4672         syslog("err", "VM $vmid qmp command failed - $err");
4673         die $err;
4674     }
4675
4676     return $res;
4677 }
4678
4679 sub vm_human_monitor_command {
4680     my ($vmid, $cmdline) = @_;
4681
4682     my $res;
4683
4684     my $cmd = {
4685         execute => 'human-monitor-command',
4686         arguments => { 'command-line' => $cmdline},
4687     };
4688
4689     return vm_qmp_command($vmid, $cmd);
4690 }
4691
4692 sub vm_commandline {
4693     my ($storecfg, $vmid) = @_;
4694
4695     my $conf = load_config($vmid);
4696
4697     my $defaults = load_defaults();
4698
4699     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4700
4701     return join(' ', @$cmd);
4702 }
4703
4704 sub vm_reset {
4705     my ($vmid, $skiplock) = @_;
4706
4707     lock_config($vmid, sub {
4708
4709         my $conf = load_config($vmid);
4710
4711         check_lock($conf) if !$skiplock;
4712
4713         vm_mon_cmd($vmid, "system_reset");
4714     });
4715 }
4716
4717 sub get_vm_volumes {
4718     my ($conf) = @_;
4719
4720     my $vollist = [];
4721     foreach_volid($conf, sub {
4722         my ($volid, $is_cdrom) = @_;
4723
4724         return if $volid =~ m|^/|;
4725
4726         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4727         return if !$sid;
4728
4729         push @$vollist, $volid;
4730     });
4731
4732     return $vollist;
4733 }
4734
4735 sub vm_stop_cleanup {
4736     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4737
4738     eval {
4739
4740         if (!$keepActive) {
4741             my $vollist = get_vm_volumes($conf);
4742             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4743         }
4744
4745         foreach my $ext (qw(mon qmp pid vnc qga)) {
4746             unlink "/var/run/qemu-server/${vmid}.$ext";
4747         }
4748
4749         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4750     };
4751     warn $@ if $@; # avoid errors - just warn
4752 }
4753
4754 # Note: use $nockeck to skip tests if VM configuration file exists.
4755 # We need that when migration VMs to other nodes (files already moved)
4756 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4757 sub vm_stop {
4758     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4759
4760     $force = 1 if !defined($force) && !$shutdown;
4761
4762     if ($migratedfrom){
4763         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4764         kill 15, $pid if $pid;
4765         my $conf = load_config($vmid, $migratedfrom);
4766         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4767         return;
4768     }
4769
4770     lock_config($vmid, sub {
4771
4772         my $pid = check_running($vmid, $nocheck);
4773         return if !$pid;
4774
4775         my $conf;
4776         if (!$nocheck) {
4777             $conf = load_config($vmid);
4778             check_lock($conf) if !$skiplock;
4779             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4780                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4781                 $timeout = $opts->{down} if $opts->{down};
4782             }
4783         }
4784
4785         $timeout = 60 if !defined($timeout);
4786
4787         eval {
4788             if ($shutdown) {
4789                 if (defined($conf) && $conf->{agent}) {
4790                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4791                 } else {
4792                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4793                 }
4794             } else {
4795                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4796             }
4797         };
4798         my $err = $@;
4799
4800         if (!$err) {
4801             my $count = 0;
4802             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4803                 $count++;
4804                 sleep 1;
4805             }
4806
4807             if ($count >= $timeout) {
4808                 if ($force) {
4809                     warn "VM still running - terminating now with SIGTERM\n";
4810                     kill 15, $pid;
4811                 } else {
4812                     die "VM quit/powerdown failed - got timeout\n";
4813                 }
4814             } else {
4815                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4816                 return;
4817             }
4818         } else {
4819             if ($force) {
4820                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4821                 kill 15, $pid;
4822             } else {
4823                 die "VM quit/powerdown failed\n";
4824             }
4825         }
4826
4827         # wait again
4828         $timeout = 10;
4829
4830         my $count = 0;
4831         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4832             $count++;
4833             sleep 1;
4834         }
4835
4836         if ($count >= $timeout) {
4837             warn "VM still running - terminating now with SIGKILL\n";
4838             kill 9, $pid;
4839             sleep 1;
4840         }
4841
4842         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4843    });
4844 }
4845
4846 sub vm_suspend {
4847     my ($vmid, $skiplock) = @_;
4848
4849     lock_config($vmid, sub {
4850
4851         my $conf = load_config($vmid);
4852
4853         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4854
4855         vm_mon_cmd($vmid, "stop");
4856     });
4857 }
4858
4859 sub vm_resume {
4860     my ($vmid, $skiplock, $nocheck) = @_;
4861
4862     lock_config($vmid, sub {
4863
4864         if (!$nocheck) {
4865
4866             my $conf = load_config($vmid);
4867
4868             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4869
4870             vm_mon_cmd($vmid, "cont");
4871
4872         } else {
4873             vm_mon_cmd_nocheck($vmid, "cont");
4874         }
4875     });
4876 }
4877
4878 sub vm_sendkey {
4879     my ($vmid, $skiplock, $key) = @_;
4880
4881     lock_config($vmid, sub {
4882
4883         my $conf = load_config($vmid);
4884
4885         # there is no qmp command, so we use the human monitor command
4886         vm_human_monitor_command($vmid, "sendkey $key");
4887     });
4888 }
4889
4890 sub vm_destroy {
4891     my ($storecfg, $vmid, $skiplock) = @_;
4892
4893     lock_config($vmid, sub {
4894
4895         my $conf = load_config($vmid);
4896
4897         if (!check_running($vmid)) {
4898             destroy_vm($storecfg, $vmid, undef, $skiplock);
4899         } else {
4900             die "VM $vmid is running - destroy failed\n";
4901         }
4902     });
4903 }
4904
4905 # pci helpers
4906
4907 sub file_write {
4908     my ($filename, $buf) = @_;
4909
4910     my $fh = IO::File->new($filename, "w");
4911     return undef if !$fh;
4912
4913     my $res = print $fh $buf;
4914
4915     $fh->close();
4916
4917     return $res;
4918 }
4919
4920 sub pci_device_info {
4921     my ($name) = @_;
4922
4923     my $res;
4924
4925     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4926     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4927
4928     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4929     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4930
4931     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4932     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4933
4934     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4935     return undef if !defined($product) || $product !~ s/^0x//;
4936
4937     $res = {
4938         name => $name,
4939         vendor => $vendor,
4940         product => $product,
4941         domain => $domain,
4942         bus => $bus,
4943         slot => $slot,
4944         func => $func,
4945         irq => $irq,
4946         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4947     };
4948
4949     return $res;
4950 }
4951
4952 sub pci_dev_reset {
4953     my ($dev) = @_;
4954
4955     my $name = $dev->{name};
4956
4957     my $fn = "$pcisysfs/devices/$name/reset";
4958
4959     return file_write($fn, "1");
4960 }
4961
4962 sub pci_dev_bind_to_vfio {
4963     my ($dev) = @_;
4964
4965     my $name = $dev->{name};
4966
4967     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4968
4969     if (!-d $vfio_basedir) {
4970         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4971     }
4972     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4973
4974     my $testdir = "$vfio_basedir/$name";
4975     return 1 if -d $testdir;
4976
4977     my $data = "$dev->{vendor} $dev->{product}";
4978     return undef if !file_write("$vfio_basedir/new_id", $data);
4979
4980     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4981     if (!file_write($fn, $name)) {
4982         return undef if -f $fn;
4983     }
4984
4985     $fn = "$vfio_basedir/bind";
4986     if (! -d $testdir) {
4987         return undef if !file_write($fn, $name);
4988     }
4989
4990     return -d $testdir;
4991 }
4992
4993 sub pci_dev_group_bind_to_vfio {
4994     my ($pciid) = @_;
4995
4996     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4997
4998     if (!-d $vfio_basedir) {
4999         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5000     }
5001     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5002
5003     # get IOMMU group devices
5004     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5005       my @devs = grep /^0000:/, readdir($D);
5006     closedir($D);
5007
5008     foreach my $pciid (@devs) {
5009         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5010
5011         # pci bridges, switches or root ports are not supported
5012         # they have a pci_bus subdirectory so skip them
5013         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5014
5015         my $info = pci_device_info($1);
5016         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5017     }
5018
5019     return 1;
5020 }
5021
5022 sub print_pci_addr {
5023     my ($id, $bridges) = @_;
5024
5025     my $res = '';
5026     my $devices = {
5027         piix3 => { bus => 0, addr => 1 },
5028         #addr2 : first videocard
5029         balloon0 => { bus => 0, addr => 3 },
5030         watchdog => { bus => 0, addr => 4 },
5031         scsihw0 => { bus => 0, addr => 5 },
5032         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5033         scsihw1 => { bus => 0, addr => 6 },
5034         ahci0 => { bus => 0, addr => 7 },
5035         qga0 => { bus => 0, addr => 8 },
5036         spice => { bus => 0, addr => 9 },
5037         virtio0 => { bus => 0, addr => 10 },
5038         virtio1 => { bus => 0, addr => 11 },
5039         virtio2 => { bus => 0, addr => 12 },
5040         virtio3 => { bus => 0, addr => 13 },
5041         virtio4 => { bus => 0, addr => 14 },
5042         virtio5 => { bus => 0, addr => 15 },
5043         hostpci0 => { bus => 0, addr => 16 },
5044         hostpci1 => { bus => 0, addr => 17 },
5045         net0 => { bus => 0, addr => 18 },
5046         net1 => { bus => 0, addr => 19 },
5047         net2 => { bus => 0, addr => 20 },
5048         net3 => { bus => 0, addr => 21 },
5049         net4 => { bus => 0, addr => 22 },
5050         net5 => { bus => 0, addr => 23 },
5051         vga1 => { bus => 0, addr => 24 },
5052         vga2 => { bus => 0, addr => 25 },
5053         vga3 => { bus => 0, addr => 26 },
5054         hostpci2 => { bus => 0, addr => 27 },
5055         hostpci3 => { bus => 0, addr => 28 },
5056         #addr29 : usb-host (pve-usb.cfg)
5057         'pci.1' => { bus => 0, addr => 30 },
5058         'pci.2' => { bus => 0, addr => 31 },
5059         'net6' => { bus => 1, addr => 1 },
5060         'net7' => { bus => 1, addr => 2 },
5061         'net8' => { bus => 1, addr => 3 },
5062         'net9' => { bus => 1, addr => 4 },
5063         'net10' => { bus => 1, addr => 5 },
5064         'net11' => { bus => 1, addr => 6 },
5065         'net12' => { bus => 1, addr => 7 },
5066         'net13' => { bus => 1, addr => 8 },
5067         'net14' => { bus => 1, addr => 9 },
5068         'net15' => { bus => 1, addr => 10 },
5069         'net16' => { bus => 1, addr => 11 },
5070         'net17' => { bus => 1, addr => 12 },
5071         'net18' => { bus => 1, addr => 13 },
5072         'net19' => { bus => 1, addr => 14 },
5073         'net20' => { bus => 1, addr => 15 },
5074         'net21' => { bus => 1, addr => 16 },
5075         'net22' => { bus => 1, addr => 17 },
5076         'net23' => { bus => 1, addr => 18 },
5077         'net24' => { bus => 1, addr => 19 },
5078         'net25' => { bus => 1, addr => 20 },
5079         'net26' => { bus => 1, addr => 21 },
5080         'net27' => { bus => 1, addr => 22 },
5081         'net28' => { bus => 1, addr => 23 },
5082         'net29' => { bus => 1, addr => 24 },
5083         'net30' => { bus => 1, addr => 25 },
5084         'net31' => { bus => 1, addr => 26 },
5085         'xhci' => { bus => 1, addr => 27 },
5086         'virtio6' => { bus => 2, addr => 1 },
5087         'virtio7' => { bus => 2, addr => 2 },
5088         'virtio8' => { bus => 2, addr => 3 },
5089         'virtio9' => { bus => 2, addr => 4 },
5090         'virtio10' => { bus => 2, addr => 5 },
5091         'virtio11' => { bus => 2, addr => 6 },
5092         'virtio12' => { bus => 2, addr => 7 },
5093         'virtio13' => { bus => 2, addr => 8 },
5094         'virtio14' => { bus => 2, addr => 9 },
5095         'virtio15' => { bus => 2, addr => 10 },
5096         'virtioscsi0' => { bus => 3, addr => 1 },
5097         'virtioscsi1' => { bus => 3, addr => 2 },
5098         'virtioscsi2' => { bus => 3, addr => 3 },
5099         'virtioscsi3' => { bus => 3, addr => 4 },
5100         'virtioscsi4' => { bus => 3, addr => 5 },
5101         'virtioscsi5' => { bus => 3, addr => 6 },
5102         'virtioscsi6' => { bus => 3, addr => 7 },
5103         'virtioscsi7' => { bus => 3, addr => 8 },
5104         'virtioscsi8' => { bus => 3, addr => 9 },
5105         'virtioscsi9' => { bus => 3, addr => 10 },
5106         'virtioscsi10' => { bus => 3, addr => 11 },
5107         'virtioscsi11' => { bus => 3, addr => 12 },
5108         'virtioscsi12' => { bus => 3, addr => 13 },
5109         'virtioscsi13' => { bus => 3, addr => 14 },
5110         'virtioscsi14' => { bus => 3, addr => 15 },
5111         'virtioscsi15' => { bus => 3, addr => 16 },
5112         'virtioscsi16' => { bus => 3, addr => 17 },
5113         'virtioscsi17' => { bus => 3, addr => 18 },
5114         'virtioscsi18' => { bus => 3, addr => 19 },
5115         'virtioscsi19' => { bus => 3, addr => 20 },
5116         'virtioscsi20' => { bus => 3, addr => 21 },
5117         'virtioscsi21' => { bus => 3, addr => 22 },
5118         'virtioscsi22' => { bus => 3, addr => 23 },
5119         'virtioscsi23' => { bus => 3, addr => 24 },
5120         'virtioscsi24' => { bus => 3, addr => 25 },
5121         'virtioscsi25' => { bus => 3, addr => 26 },
5122         'virtioscsi26' => { bus => 3, addr => 27 },
5123         'virtioscsi27' => { bus => 3, addr => 28 },
5124         'virtioscsi28' => { bus => 3, addr => 29 },
5125         'virtioscsi29' => { bus => 3, addr => 30 },
5126         'virtioscsi30' => { bus => 3, addr => 31 },
5127
5128     };
5129
5130     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5131            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5132            my $bus = $devices->{$id}->{bus};
5133            $res = ",bus=pci.$bus,addr=$addr";
5134            $bridges->{$bus} = 1 if $bridges;
5135     }
5136     return $res;
5137
5138 }
5139
5140 sub print_pcie_addr {
5141     my ($id) = @_;
5142
5143     my $res = '';
5144     my $devices = {
5145         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5146         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5147         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5148         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5149     };
5150
5151     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5152            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5153            my $bus = $devices->{$id}->{bus};
5154            $res = ",bus=$bus,addr=$addr";
5155     }
5156     return $res;
5157
5158 }
5159
5160 # vzdump restore implementaion
5161
5162 sub tar_archive_read_firstfile {
5163     my $archive = shift;
5164
5165     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5166
5167     # try to detect archive type first
5168     my $pid = open (TMP, "tar tf '$archive'|") ||
5169         die "unable to open file '$archive'\n";
5170     my $firstfile = <TMP>;
5171     kill 15, $pid;
5172     close TMP;
5173
5174     die "ERROR: archive contaions no data\n" if !$firstfile;
5175     chomp $firstfile;
5176
5177     return $firstfile;
5178 }
5179
5180 sub tar_restore_cleanup {
5181     my ($storecfg, $statfile) = @_;
5182
5183     print STDERR "starting cleanup\n";
5184
5185     if (my $fd = IO::File->new($statfile, "r")) {
5186         while (defined(my $line = <$fd>)) {
5187             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5188                 my $volid = $2;
5189                 eval {
5190                     if ($volid =~ m|^/|) {
5191                         unlink $volid || die 'unlink failed\n';
5192                     } else {
5193                         PVE::Storage::vdisk_free($storecfg, $volid);
5194                     }
5195                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5196                 };
5197                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5198             } else {
5199                 print STDERR "unable to parse line in statfile - $line";
5200             }
5201         }
5202         $fd->close();
5203     }
5204 }
5205
5206 sub restore_archive {
5207     my ($archive, $vmid, $user, $opts) = @_;
5208
5209     my $format = $opts->{format};
5210     my $comp;
5211
5212     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5213         $format = 'tar' if !$format;
5214         $comp = 'gzip';
5215     } elsif ($archive =~ m/\.tar$/) {
5216         $format = 'tar' if !$format;
5217     } elsif ($archive =~ m/.tar.lzo$/) {
5218         $format = 'tar' if !$format;
5219         $comp = 'lzop';
5220     } elsif ($archive =~ m/\.vma$/) {
5221         $format = 'vma' if !$format;
5222     } elsif ($archive =~ m/\.vma\.gz$/) {
5223         $format = 'vma' if !$format;
5224         $comp = 'gzip';
5225     } elsif ($archive =~ m/\.vma\.lzo$/) {
5226         $format = 'vma' if !$format;
5227         $comp = 'lzop';
5228     } else {
5229         $format = 'vma' if !$format; # default
5230     }
5231
5232     # try to detect archive format
5233     if ($format eq 'tar') {
5234         return restore_tar_archive($archive, $vmid, $user, $opts);
5235     } else {
5236         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5237     }
5238 }
5239
5240 sub restore_update_config_line {
5241     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5242
5243     return if $line =~ m/^\#qmdump\#/;
5244     return if $line =~ m/^\#vzdump\#/;
5245     return if $line =~ m/^lock:/;
5246     return if $line =~ m/^unused\d+:/;
5247     return if $line =~ m/^parent:/;
5248     return if $line =~ m/^template:/; # restored VM is never a template
5249
5250     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5251         # try to convert old 1.X settings
5252         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5253         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5254             my ($model, $macaddr) = split(/\=/, $devconfig);
5255             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5256             my $net = {
5257                 model => $model,
5258                 bridge => "vmbr$ind",
5259                 macaddr => $macaddr,
5260             };
5261             my $netstr = print_net($net);
5262
5263             print $outfd "net$cookie->{netcount}: $netstr\n";
5264             $cookie->{netcount}++;
5265         }
5266     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5267         my ($id, $netstr) = ($1, $2);
5268         my $net = parse_net($netstr);
5269         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5270         $netstr = print_net($net);
5271         print $outfd "$id: $netstr\n";
5272     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5273         my $virtdev = $1;
5274         my $value = $3;
5275         my $di = parse_drive($virtdev, $value);
5276         if (defined($di->{backup}) && !$di->{backup}) {
5277             print $outfd "#$line";
5278         } elsif ($map->{$virtdev}) {
5279             delete $di->{format}; # format can change on restore
5280             $di->{file} = $map->{$virtdev};
5281             $value = print_drive($vmid, $di);
5282             print $outfd "$virtdev: $value\n";
5283         } else {
5284             print $outfd $line;
5285         }
5286     } else {
5287         print $outfd $line;
5288     }
5289 }
5290
5291 sub scan_volids {
5292     my ($cfg, $vmid) = @_;
5293
5294     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5295
5296     my $volid_hash = {};
5297     foreach my $storeid (keys %$info) {
5298         foreach my $item (@{$info->{$storeid}}) {
5299             next if !($item->{volid} && $item->{size});
5300             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5301             $volid_hash->{$item->{volid}} = $item;
5302         }
5303     }
5304
5305     return $volid_hash;
5306 }
5307
5308 sub is_volume_in_use {
5309     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5310
5311     my $path = PVE::Storage::path($storecfg, $volid);
5312
5313     my $scan_config = sub {
5314         my ($cref, $snapname) = @_;
5315
5316         foreach my $key (keys %$cref) {
5317             my $value = $cref->{$key};
5318             if (valid_drivename($key)) {
5319                 next if $skip_drive && $key eq $skip_drive;
5320                 my $drive = parse_drive($key, $value);
5321                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5322                 return 1 if $volid eq $drive->{file};
5323                 if ($drive->{file} =~ m!^/!) {
5324                     return 1 if $drive->{file} eq $path;
5325                 } else {
5326                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5327                     next if !$storeid;
5328                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5329                     next if !$scfg;
5330                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5331                 }
5332             }
5333         }
5334
5335         return 0;
5336     };
5337
5338     return 1 if &$scan_config($conf);
5339
5340     undef $skip_drive;
5341
5342     foreach my $snapname (keys %{$conf->{snapshots}}) {
5343         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5344     }
5345
5346     return 0;
5347 }
5348
5349 sub update_disksize {
5350     my ($vmid, $conf, $volid_hash) = @_;
5351
5352     my $changes;
5353
5354     my $used = {};
5355
5356     # Note: it is allowed to define multiple storages with same path (alias), so
5357     # we need to check both 'volid' and real 'path' (two different volid can point
5358     # to the same path).
5359
5360     my $usedpath = {};
5361
5362     # update size info
5363     foreach my $opt (keys %$conf) {
5364         if (valid_drivename($opt)) {
5365             my $drive = parse_drive($opt, $conf->{$opt});
5366             my $volid = $drive->{file};
5367             next if !$volid;
5368
5369             $used->{$volid} = 1;
5370             if ($volid_hash->{$volid} &&
5371                 (my $path = $volid_hash->{$volid}->{path})) {
5372                 $usedpath->{$path} = 1;
5373             }
5374
5375             next if drive_is_cdrom($drive);
5376             next if !$volid_hash->{$volid};
5377
5378             $drive->{size} = $volid_hash->{$volid}->{size};
5379             my $new = print_drive($vmid, $drive);
5380             if ($new ne $conf->{$opt}) {
5381                 $changes = 1;
5382                 $conf->{$opt} = $new;
5383             }
5384         }
5385     }
5386
5387     # remove 'unusedX' entry if volume is used
5388     foreach my $opt (keys %$conf) {
5389         next if $opt !~ m/^unused\d+$/;
5390         my $volid = $conf->{$opt};
5391         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5392         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5393             $changes = 1;
5394             delete $conf->{$opt};
5395         }
5396     }
5397
5398     foreach my $volid (sort keys %$volid_hash) {
5399         next if $volid =~ m/vm-$vmid-state-/;
5400         next if $used->{$volid};
5401         my $path = $volid_hash->{$volid}->{path};
5402         next if !$path; # just to be sure
5403         next if $usedpath->{$path};
5404         $changes = 1;
5405         add_unused_volume($conf, $volid);
5406         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5407     }
5408
5409     return $changes;
5410 }
5411
5412 sub rescan {
5413     my ($vmid, $nolock) = @_;
5414
5415     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5416
5417     my $volid_hash = scan_volids($cfg, $vmid);
5418
5419     my $updatefn =  sub {
5420         my ($vmid) = @_;
5421
5422         my $conf = load_config($vmid);
5423
5424         check_lock($conf);
5425
5426         my $vm_volids = {};
5427         foreach my $volid (keys %$volid_hash) {
5428             my $info = $volid_hash->{$volid};
5429             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5430         }
5431
5432         my $changes = update_disksize($vmid, $conf, $vm_volids);
5433
5434         write_config($vmid, $conf) if $changes;
5435     };
5436
5437     if (defined($vmid)) {
5438         if ($nolock) {
5439             &$updatefn($vmid);
5440         } else {
5441             lock_config($vmid, $updatefn, $vmid);
5442         }
5443     } else {
5444         my $vmlist = config_list();
5445         foreach my $vmid (keys %$vmlist) {
5446             if ($nolock) {
5447                 &$updatefn($vmid);
5448             } else {
5449                 lock_config($vmid, $updatefn, $vmid);
5450             }
5451         }
5452     }
5453 }
5454
5455 sub restore_vma_archive {
5456     my ($archive, $vmid, $user, $opts, $comp) = @_;
5457
5458     my $input = $archive eq '-' ? "<&STDIN" : undef;
5459     my $readfrom = $archive;
5460
5461     my $uncomp = '';
5462     if ($comp) {
5463         $readfrom = '-';
5464         my $qarchive = PVE::Tools::shellquote($archive);
5465         if ($comp eq 'gzip') {
5466             $uncomp = "zcat $qarchive|";
5467         } elsif ($comp eq 'lzop') {
5468             $uncomp = "lzop -d -c $qarchive|";
5469         } else {
5470             die "unknown compression method '$comp'\n";
5471         }
5472
5473     }
5474
5475     my $tmpdir = "/var/tmp/vzdumptmp$$";
5476     rmtree $tmpdir;
5477
5478     # disable interrupts (always do cleanups)
5479     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5480         warn "got interrupt - ignored\n";
5481     };
5482
5483     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5484     POSIX::mkfifo($mapfifo, 0600);
5485     my $fifofh;
5486
5487     my $openfifo = sub {
5488         open($fifofh, '>', $mapfifo) || die $!;
5489     };
5490
5491     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5492
5493     my $oldtimeout;
5494     my $timeout = 5;
5495
5496     my $devinfo = {};
5497
5498     my $rpcenv = PVE::RPCEnvironment::get();
5499
5500     my $conffile = config_file($vmid);
5501     my $tmpfn = "$conffile.$$.tmp";
5502
5503     # Note: $oldconf is undef if VM does not exists
5504     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5505
5506     my $print_devmap = sub {
5507         my $virtdev_hash = {};
5508
5509         my $cfgfn = "$tmpdir/qemu-server.conf";
5510
5511         # we can read the config - that is already extracted
5512         my $fh = IO::File->new($cfgfn, "r") ||
5513             "unable to read qemu-server.conf - $!\n";
5514
5515         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5516         if (-f $fwcfgfn) {
5517             my $pve_firewall_dir = '/etc/pve/firewall';
5518             mkdir $pve_firewall_dir; # make sure the dir exists
5519             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5520         }
5521
5522         while (defined(my $line = <$fh>)) {
5523             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5524                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5525                 die "archive does not contain data for drive '$virtdev'\n"
5526                     if !$devinfo->{$devname};
5527                 if (defined($opts->{storage})) {
5528                     $storeid = $opts->{storage} || 'local';
5529                 } elsif (!$storeid) {
5530                     $storeid = 'local';
5531                 }
5532                 $format = 'raw' if !$format;
5533                 $devinfo->{$devname}->{devname} = $devname;
5534                 $devinfo->{$devname}->{virtdev} = $virtdev;
5535                 $devinfo->{$devname}->{format} = $format;
5536                 $devinfo->{$devname}->{storeid} = $storeid;
5537
5538                 # check permission on storage
5539                 my $pool = $opts->{pool}; # todo: do we need that?
5540                 if ($user ne 'root@pam') {
5541                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5542                 }
5543
5544                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5545             }
5546         }
5547
5548         foreach my $devname (keys %$devinfo) {
5549             die "found no device mapping information for device '$devname'\n"
5550                 if !$devinfo->{$devname}->{virtdev};
5551         }
5552
5553         my $cfg = cfs_read_file('storage.cfg');
5554
5555         # create empty/temp config
5556         if ($oldconf) {
5557             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5558             foreach_drive($oldconf, sub {
5559                 my ($ds, $drive) = @_;
5560
5561                 return if drive_is_cdrom($drive);
5562
5563                 my $volid = $drive->{file};
5564
5565                 return if !$volid || $volid =~ m|^/|;
5566
5567                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5568                 return if !$path || !$owner || ($owner != $vmid);
5569
5570                 # Note: only delete disk we want to restore
5571                 # other volumes will become unused
5572                 if ($virtdev_hash->{$ds}) {
5573                     PVE::Storage::vdisk_free($cfg, $volid);
5574                 }
5575             });
5576
5577             # delete vmstate files
5578             # since after the restore we have no snapshots anymore
5579             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5580                 my $snap = $oldconf->{snapshots}->{$snapname};
5581                 if ($snap->{vmstate}) {
5582                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5583                     if (my $err = $@) {
5584                         warn $err;
5585                     }
5586                 }
5587             }
5588         }
5589
5590         my $map = {};
5591         foreach my $virtdev (sort keys %$virtdev_hash) {
5592             my $d = $virtdev_hash->{$virtdev};
5593             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5594             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5595
5596             # test if requested format is supported
5597             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5598             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5599             $d->{format} = $defFormat if !$supported;
5600
5601             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5602                                                   $d->{format}, undef, $alloc_size);
5603             print STDERR "new volume ID is '$volid'\n";
5604             $d->{volid} = $volid;
5605             my $path = PVE::Storage::path($cfg, $volid);
5606
5607             PVE::Storage::activate_volumes($cfg,[$volid]);
5608
5609             my $write_zeros = 1;
5610             # fixme: what other storages types initialize volumes with zero?
5611             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5612                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5613                 $write_zeros = 0;
5614             }
5615
5616             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5617
5618             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5619             $map->{$virtdev} = $volid;
5620         }
5621
5622         $fh->seek(0, 0) || die "seek failed - $!\n";
5623
5624         my $outfd = new IO::File ($tmpfn, "w") ||
5625             die "unable to write config for VM $vmid\n";
5626
5627         my $cookie = { netcount => 0 };
5628         while (defined(my $line = <$fh>)) {
5629             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5630         }
5631
5632         $fh->close();
5633         $outfd->close();
5634     };
5635
5636     eval {
5637         # enable interrupts
5638         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5639             die "interrupted by signal\n";
5640         };
5641         local $SIG{ALRM} = sub { die "got timeout\n"; };
5642
5643         $oldtimeout = alarm($timeout);
5644
5645         my $parser = sub {
5646             my $line = shift;
5647
5648             print "$line\n";
5649
5650             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5651                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5652                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5653             } elsif ($line =~ m/^CTIME: /) {
5654                 # we correctly received the vma config, so we can disable
5655                 # the timeout now for disk allocation (set to 10 minutes, so
5656                 # that we always timeout if something goes wrong)
5657                 alarm(600);
5658                 &$print_devmap();
5659                 print $fifofh "done\n";
5660                 my $tmp = $oldtimeout || 0;
5661                 $oldtimeout = undef;
5662                 alarm($tmp);
5663                 close($fifofh);
5664             }
5665         };
5666
5667         print "restore vma archive: $cmd\n";
5668         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5669     };
5670     my $err = $@;
5671
5672     alarm($oldtimeout) if $oldtimeout;
5673
5674     my $vollist = [];
5675     foreach my $devname (keys %$devinfo) {
5676         my $volid = $devinfo->{$devname}->{volid};
5677         push @$vollist, $volid if $volid;
5678     }
5679
5680     my $cfg = cfs_read_file('storage.cfg');
5681     PVE::Storage::deactivate_volumes($cfg, $vollist);
5682
5683     unlink $mapfifo;
5684
5685     if ($err) {
5686         rmtree $tmpdir;
5687         unlink $tmpfn;
5688
5689         foreach my $devname (keys %$devinfo) {
5690             my $volid = $devinfo->{$devname}->{volid};
5691             next if !$volid;
5692             eval {
5693                 if ($volid =~ m|^/|) {
5694                     unlink $volid || die 'unlink failed\n';
5695                 } else {
5696                     PVE::Storage::vdisk_free($cfg, $volid);
5697                 }
5698                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5699             };
5700             print STDERR "unable to cleanup '$volid' - $@" if $@;
5701         }
5702         die $err;
5703     }
5704
5705     rmtree $tmpdir;
5706
5707     rename($tmpfn, $conffile) ||
5708         die "unable to commit configuration file '$conffile'\n";
5709
5710     PVE::Cluster::cfs_update(); # make sure we read new file
5711
5712     eval { rescan($vmid, 1); };
5713     warn $@ if $@;
5714 }
5715
5716 sub restore_tar_archive {
5717     my ($archive, $vmid, $user, $opts) = @_;
5718
5719     if ($archive ne '-') {
5720         my $firstfile = tar_archive_read_firstfile($archive);
5721         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5722             if $firstfile ne 'qemu-server.conf';
5723     }
5724
5725     my $storecfg = cfs_read_file('storage.cfg');
5726
5727     # destroy existing data - keep empty config
5728     my $vmcfgfn = config_file($vmid);
5729     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5730
5731     my $tocmd = "/usr/lib/qemu-server/qmextract";
5732
5733     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5734     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5735     $tocmd .= ' --prealloc' if $opts->{prealloc};
5736     $tocmd .= ' --info' if $opts->{info};
5737
5738     # tar option "xf" does not autodetect compression when read from STDIN,
5739     # so we pipe to zcat
5740     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5741         PVE::Tools::shellquote("--to-command=$tocmd");
5742
5743     my $tmpdir = "/var/tmp/vzdumptmp$$";
5744     mkpath $tmpdir;
5745
5746     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5747     local $ENV{VZDUMP_VMID} = $vmid;
5748     local $ENV{VZDUMP_USER} = $user;
5749
5750     my $conffile = config_file($vmid);
5751     my $tmpfn = "$conffile.$$.tmp";
5752
5753     # disable interrupts (always do cleanups)
5754     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5755         print STDERR "got interrupt - ignored\n";
5756     };
5757
5758     eval {
5759         # enable interrupts
5760         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5761             die "interrupted by signal\n";
5762         };
5763
5764         if ($archive eq '-') {
5765             print "extracting archive from STDIN\n";
5766             run_command($cmd, input => "<&STDIN");
5767         } else {
5768             print "extracting archive '$archive'\n";
5769             run_command($cmd);
5770         }
5771
5772         return if $opts->{info};
5773
5774         # read new mapping
5775         my $map = {};
5776         my $statfile = "$tmpdir/qmrestore.stat";
5777         if (my $fd = IO::File->new($statfile, "r")) {
5778             while (defined (my $line = <$fd>)) {
5779                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5780                     $map->{$1} = $2 if $1;
5781                 } else {
5782                     print STDERR "unable to parse line in statfile - $line\n";
5783                 }
5784             }
5785             $fd->close();
5786         }
5787
5788         my $confsrc = "$tmpdir/qemu-server.conf";
5789
5790         my $srcfd = new IO::File($confsrc, "r") ||
5791             die "unable to open file '$confsrc'\n";
5792
5793         my $outfd = new IO::File ($tmpfn, "w") ||
5794             die "unable to write config for VM $vmid\n";
5795
5796         my $cookie = { netcount => 0 };
5797         while (defined (my $line = <$srcfd>)) {
5798             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5799         }
5800
5801         $srcfd->close();
5802         $outfd->close();
5803     };
5804     my $err = $@;
5805
5806     if ($err) {
5807
5808         unlink $tmpfn;
5809
5810         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5811
5812         die $err;
5813     }
5814
5815     rmtree $tmpdir;
5816
5817     rename $tmpfn, $conffile ||
5818         die "unable to commit configuration file '$conffile'\n";
5819
5820     PVE::Cluster::cfs_update(); # make sure we read new file
5821
5822     eval { rescan($vmid, 1); };
5823     warn $@ if $@;
5824 };
5825
5826
5827 # Internal snapshots
5828
5829 # NOTE: Snapshot create/delete involves several non-atomic
5830 # action, and can take a long time.
5831 # So we try to avoid locking the file and use 'lock' variable
5832 # inside the config file instead.
5833
5834 my $snapshot_copy_config = sub {
5835     my ($source, $dest) = @_;
5836
5837     foreach my $k (keys %$source) {
5838         next if $k eq 'snapshots';
5839         next if $k eq 'snapstate';
5840         next if $k eq 'snaptime';
5841         next if $k eq 'vmstate';
5842         next if $k eq 'lock';
5843         next if $k eq 'digest';
5844         next if $k eq 'description';
5845         next if $k =~ m/^unused\d+$/;
5846
5847         $dest->{$k} = $source->{$k};
5848     }
5849 };
5850
5851 my $snapshot_apply_config = sub {
5852     my ($conf, $snap) = @_;
5853
5854     # copy snapshot list
5855     my $newconf = {
5856         snapshots => $conf->{snapshots},
5857     };
5858
5859     # keep description and list of unused disks
5860     foreach my $k (keys %$conf) {
5861         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5862         $newconf->{$k} = $conf->{$k};
5863     }
5864
5865     &$snapshot_copy_config($snap, $newconf);
5866
5867     return $newconf;
5868 };
5869
5870 sub foreach_writable_storage {
5871     my ($conf, $func) = @_;
5872
5873     my $sidhash = {};
5874
5875     foreach my $ds (keys %$conf) {
5876         next if !valid_drivename($ds);
5877
5878         my $drive = parse_drive($ds, $conf->{$ds});
5879         next if !$drive;
5880         next if drive_is_cdrom($drive);
5881
5882         my $volid = $drive->{file};
5883
5884         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5885         $sidhash->{$sid} = $sid if $sid;
5886     }
5887
5888     foreach my $sid (sort keys %$sidhash) {
5889         &$func($sid);
5890     }
5891 }
5892
5893 my $alloc_vmstate_volid = sub {
5894     my ($storecfg, $vmid, $conf, $snapname) = @_;
5895
5896     # Note: we try to be smart when selecting a $target storage
5897
5898     my $target;
5899
5900     # search shared storage first
5901     foreach_writable_storage($conf, sub {
5902         my ($sid) = @_;
5903         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5904         return if !$scfg->{shared};
5905
5906         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5907     });
5908
5909     if (!$target) {
5910         # now search local storage
5911         foreach_writable_storage($conf, sub {
5912             my ($sid) = @_;
5913             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5914             return if $scfg->{shared};
5915
5916             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5917         });
5918     }
5919
5920     $target = 'local' if !$target;
5921
5922     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5923     # we abort live save after $conf->{memory}, so we need at max twice that space
5924     my $size = $conf->{memory}*2 + $driver_state_size;
5925
5926     my $name = "vm-$vmid-state-$snapname";
5927     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5928     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5929     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5930
5931     return $volid;
5932 };
5933
5934 my $snapshot_save_vmstate = sub {
5935     my ($vmid, $conf, $snapname, $storecfg) = @_;
5936
5937     my $snap = $conf->{snapshots}->{$snapname};
5938
5939     $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5940     # always overwrite machine if we save vmstate. This makes sure we
5941     # can restore it later using correct machine type
5942     $snap->{machine} = get_current_qemu_machine($vmid);
5943 };
5944
5945 my $snapshot_prepare = sub {
5946     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5947
5948     my $snap;
5949
5950     my $updatefn =  sub {
5951
5952         my $conf = load_config($vmid);
5953
5954         die "you can't take a snapshot if it's a template\n"
5955             if is_template($conf);
5956
5957         check_lock($conf);
5958
5959         $conf->{lock} = 'snapshot';
5960
5961         die "snapshot name '$snapname' already used\n"
5962             if defined($conf->{snapshots}->{$snapname});
5963
5964         my $storecfg = PVE::Storage::config();
5965         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
5966
5967         $snap = $conf->{snapshots}->{$snapname} = {};
5968
5969         if ($save_vmstate && check_running($vmid)) {
5970             &$snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
5971         }
5972
5973         &$snapshot_copy_config($conf, $snap);
5974
5975         $snap->{snapstate} = "prepare";
5976         $snap->{snaptime} = time();
5977         $snap->{description} = $comment if $comment;
5978
5979         write_config($vmid, $conf);
5980     };
5981
5982     lock_config($vmid, $updatefn);
5983
5984     return $snap;
5985 };
5986
5987 my $snapshot_commit = sub {
5988     my ($vmid, $snapname) = @_;
5989
5990     my $updatefn = sub {
5991
5992         my $conf = load_config($vmid);
5993
5994         die "missing snapshot lock\n"
5995             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5996
5997         my $has_machine_config = defined($conf->{machine});
5998
5999         my $snap = $conf->{snapshots}->{$snapname};
6000
6001         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6002
6003         die "wrong snapshot state\n"
6004             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
6005
6006         delete $snap->{snapstate};
6007         delete $conf->{lock};
6008
6009         my $newconf = &$snapshot_apply_config($conf, $snap);
6010
6011         delete $newconf->{machine} if !$has_machine_config;
6012
6013         $newconf->{parent} = $snapname;
6014
6015         write_config($vmid, $newconf);
6016     };
6017
6018     lock_config($vmid, $updatefn);
6019 };
6020
6021 sub snapshot_rollback {
6022     my ($vmid, $snapname) = @_;
6023
6024     my $prepare = 1;
6025
6026     my $storecfg = PVE::Storage::config();
6027
6028     my $conf = load_config($vmid);
6029
6030     my $get_snapshot_config = sub {
6031
6032         die "you can't rollback if vm is a template\n" if is_template($conf);
6033
6034         my $res = $conf->{snapshots}->{$snapname};
6035
6036         die "snapshot '$snapname' does not exist\n" if !defined($res);
6037
6038         return $res;
6039     };
6040
6041     my $snap = &$get_snapshot_config();
6042
6043     foreach_drive($snap, sub {
6044         my ($ds, $drive) = @_;
6045
6046         return if drive_is_cdrom($drive);
6047
6048         my $volid = $drive->{file};
6049
6050         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
6051     });
6052
6053     my $updatefn = sub {
6054
6055         $conf = load_config($vmid);
6056
6057         $snap = &$get_snapshot_config();
6058
6059         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6060             if $snap->{snapstate};
6061
6062         if ($prepare) {
6063             check_lock($conf);
6064             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6065         }
6066
6067         die "unable to rollback vm $vmid: vm is running\n"
6068             if check_running($vmid);
6069
6070         if ($prepare) {
6071             $conf->{lock} = 'rollback';
6072         } else {
6073             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6074             delete $conf->{lock};
6075         }
6076
6077         my $forcemachine;
6078
6079         if (!$prepare) {
6080             my $has_machine_config = defined($conf->{machine});
6081
6082             # copy snapshot config to current config
6083             $conf = &$snapshot_apply_config($conf, $snap);
6084             $conf->{parent} = $snapname;
6085
6086             # Note: old code did not store 'machine', so we try to be smart
6087             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6088             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6089             # we remove the 'machine' configuration if not explicitly specified
6090             # in the original config.
6091             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6092         }
6093
6094         write_config($vmid, $conf);
6095
6096         if (!$prepare && $snap->{vmstate}) {
6097             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6098             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6099         }
6100     };
6101
6102     lock_config($vmid, $updatefn);
6103
6104     foreach_drive($snap, sub {
6105         my ($ds, $drive) = @_;
6106
6107         return if drive_is_cdrom($drive);
6108
6109         my $volid = $drive->{file};
6110         my $device = "drive-$ds";
6111
6112         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6113     });
6114
6115     $prepare = 0;
6116     lock_config($vmid, $updatefn);
6117 }
6118
6119 my $savevm_wait = sub {
6120     my ($vmid) = @_;
6121
6122     for(;;) {
6123         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6124         if (!$stat->{status}) {
6125             die "savevm not active\n";
6126         } elsif ($stat->{status} eq 'active') {
6127             sleep(1);
6128             next;
6129         } elsif ($stat->{status} eq 'completed') {
6130             last;
6131         } else {
6132             die "query-savevm returned status '$stat->{status}'\n";
6133         }
6134     }
6135 };
6136
6137 sub do_snapshots_with_qemu {
6138     my ($storecfg, $volid) = @_;
6139
6140     my $storage_name = PVE::Storage::parse_volume_id($volid);
6141
6142     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6143         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6144         return 1;
6145     }
6146
6147     if ($volid =~ m/\.(qcow2|qed)$/){
6148         return 1;
6149     }
6150
6151     return undef;
6152 }
6153
6154 sub qga_check_running {
6155     my ($vmid) = @_;
6156
6157     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6158     if ($@) {
6159         warn "Qemu Guest Agent are not running - $@";
6160         return 0;
6161     }
6162     return 1;
6163 }
6164
6165 sub snapshot_create {
6166     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6167
6168     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6169
6170     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6171
6172     my $config = load_config($vmid);
6173
6174     my $running = check_running($vmid);
6175
6176     my $freezefs = $running && $config->{agent} && qga_check_running($vmid);
6177     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6178
6179     my $drivehash = {};
6180
6181     if ($freezefs) {
6182         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6183         warn "guest-fsfreeze-freeze problems - $@" if $@;
6184     }
6185
6186     eval {
6187         # create internal snapshots of all drives
6188
6189         my $storecfg = PVE::Storage::config();
6190
6191         if ($running) {
6192             if ($snap->{vmstate}) {
6193                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6194                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6195                 &$savevm_wait($vmid);
6196             } else {
6197                 vm_mon_cmd($vmid, "savevm-start");
6198             }
6199         };
6200
6201         foreach_drive($snap, sub {
6202             my ($ds, $drive) = @_;
6203
6204             return if drive_is_cdrom($drive);
6205
6206             my $volid = $drive->{file};
6207             my $device = "drive-$ds";
6208
6209             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6210             $drivehash->{$ds} = 1;
6211        });
6212     };
6213     my $err = $@;
6214
6215     if ($running) {
6216         eval { vm_mon_cmd($vmid, "savevm-end")  };
6217         warn $@ if $@;
6218
6219         if ($freezefs) {
6220             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6221             warn "guest-fsfreeze-thaw problems - $@" if $@;
6222         }
6223
6224         # savevm-end is async, we need to wait
6225         for (;;) {
6226             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6227             if (!$stat->{bytes}) {
6228                 last;
6229             } else {
6230                 print "savevm not yet finished\n";
6231                 sleep(1);
6232                 next;
6233             }
6234         }
6235     }
6236
6237     if ($err) {
6238         warn "snapshot create failed: starting cleanup\n";
6239         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6240         warn $@ if $@;
6241         die $err;
6242     }
6243
6244     &$snapshot_commit($vmid, $snapname);
6245 }
6246
6247 # Note: $drivehash is only set when called from snapshot_create.
6248 sub snapshot_delete {
6249     my ($vmid, $snapname, $force, $drivehash) = @_;
6250
6251     my $prepare = 1;
6252
6253     my $snap;
6254     my $unused = [];
6255
6256     my $unlink_parent = sub {
6257         my ($confref, $new_parent) = @_;
6258
6259         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6260             if ($new_parent) {
6261                 $confref->{parent} = $new_parent;
6262             } else {
6263                 delete $confref->{parent};
6264             }
6265         }
6266     };
6267
6268     my $updatefn =  sub {
6269         my ($remove_drive) = @_;
6270
6271         my $conf = load_config($vmid);
6272
6273         if (!$drivehash) {
6274             check_lock($conf);
6275             die "you can't delete a snapshot if vm is a template\n"
6276                 if is_template($conf);
6277         }
6278
6279         $snap = $conf->{snapshots}->{$snapname};
6280
6281         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6282
6283         # remove parent refs
6284         if (!$prepare) {
6285             &$unlink_parent($conf, $snap->{parent});
6286             foreach my $sn (keys %{$conf->{snapshots}}) {
6287                 next if $sn eq $snapname;
6288                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6289             }
6290         }
6291
6292         if ($remove_drive) {
6293             if ($remove_drive eq 'vmstate') {
6294                 delete $snap->{$remove_drive};
6295             } else {
6296                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6297                 my $volid = $drive->{file};
6298                 delete $snap->{$remove_drive};
6299                 add_unused_volume($conf, $volid);
6300             }
6301         }
6302
6303         if ($prepare) {
6304             $snap->{snapstate} = 'delete';
6305         } else {
6306             delete $conf->{snapshots}->{$snapname};
6307             delete $conf->{lock} if $drivehash;
6308             foreach my $volid (@$unused) {
6309                 add_unused_volume($conf, $volid);
6310             }
6311         }
6312
6313         write_config($vmid, $conf);
6314     };
6315
6316     lock_config($vmid, $updatefn);
6317
6318     # now remove vmstate file
6319
6320     my $storecfg = PVE::Storage::config();
6321
6322     if ($snap->{vmstate}) {
6323         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6324         if (my $err = $@) {
6325             die $err if !$force;
6326             warn $err;
6327         }
6328         # save changes (remove vmstate from snapshot)
6329         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6330     };
6331
6332     # now remove all internal snapshots
6333     foreach_drive($snap, sub {
6334         my ($ds, $drive) = @_;
6335
6336         return if drive_is_cdrom($drive);
6337
6338         my $volid = $drive->{file};
6339         my $device = "drive-$ds";
6340
6341         if (!$drivehash || $drivehash->{$ds}) {
6342             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6343             if (my $err = $@) {
6344                 die $err if !$force;
6345                 warn $err;
6346             }
6347         }
6348
6349         # save changes (remove drive fron snapshot)
6350         lock_config($vmid, $updatefn, $ds) if !$force;
6351         push @$unused, $volid;
6352     });
6353
6354     # now cleanup config
6355     $prepare = 0;
6356     lock_config($vmid, $updatefn);
6357 }
6358
6359 sub has_feature {
6360     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6361
6362     my $err;
6363     foreach_drive($conf, sub {
6364         my ($ds, $drive) = @_;
6365
6366         return if drive_is_cdrom($drive);
6367         my $volid = $drive->{file};
6368         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6369     });
6370
6371     return $err ? 0 : 1;
6372 }
6373
6374 sub template_create {
6375     my ($vmid, $conf, $disk) = @_;
6376
6377     my $storecfg = PVE::Storage::config();
6378
6379     foreach_drive($conf, sub {
6380         my ($ds, $drive) = @_;
6381
6382         return if drive_is_cdrom($drive);
6383         return if $disk && $ds ne $disk;
6384
6385         my $volid = $drive->{file};
6386         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6387
6388         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6389         $drive->{file} = $voliddst;
6390         $conf->{$ds} = print_drive($vmid, $drive);
6391         write_config($vmid, $conf);
6392     });
6393 }
6394
6395 sub is_template {
6396     my ($conf) = @_;
6397
6398     return 1 if defined $conf->{template} && $conf->{template} == 1;
6399 }
6400
6401 sub qemu_img_convert {
6402     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6403
6404     my $storecfg = PVE::Storage::config();
6405     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6406     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6407
6408     if ($src_storeid && $dst_storeid) {
6409
6410         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6411
6412         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6413         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6414
6415         my $src_format = qemu_img_format($src_scfg, $src_volname);
6416         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6417
6418         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6419         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6420
6421         my $cmd = [];
6422         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6423         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6424         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6425
6426         my $parser = sub {
6427             my $line = shift;
6428             if($line =~ m/\((\S+)\/100\%\)/){
6429                 my $percent = $1;
6430                 my $transferred = int($size * $percent / 100);
6431                 my $remaining = $size - $transferred;
6432
6433                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6434             }
6435
6436         };
6437
6438         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6439         my $err = $@;
6440         die "copy failed: $err" if $err;
6441     }
6442 }
6443
6444 sub qemu_img_format {
6445     my ($scfg, $volname) = @_;
6446
6447     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6448         return $1;
6449     } else {
6450         return "raw";
6451     }
6452 }
6453
6454 sub qemu_drive_mirror {
6455     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6456
6457     my $storecfg = PVE::Storage::config();
6458     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6459
6460     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6461
6462     my $format = qemu_img_format($dst_scfg, $dst_volname);
6463
6464     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6465
6466     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6467     $opts->{format} = $format if $format;
6468
6469     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6470
6471     eval {
6472     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6473         while (1) {
6474             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6475             my $stat = @$stats[0];
6476             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6477             die "error job is not mirroring" if $stat->{type} ne "mirror";
6478
6479             my $busy = $stat->{busy};
6480             my $ready = $stat->{ready};
6481
6482             if (my $total = $stat->{len}) {
6483                 my $transferred = $stat->{offset} || 0;
6484                 my $remaining = $total - $transferred;
6485                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6486
6487                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6488             }
6489
6490
6491             if ($stat->{ready} eq 'true') {
6492
6493                 last if $vmiddst != $vmid;
6494
6495                 # try to switch the disk if source and destination are on the same guest
6496                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6497                 last if !$@;
6498                 die $@ if $@ !~ m/cannot be completed/;
6499             }
6500             sleep 1;
6501         }
6502
6503
6504     };
6505     my $err = $@;
6506
6507     my $cancel_job = sub {
6508         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6509         while (1) {
6510             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6511             my $stat = @$stats[0];
6512             last if !$stat;
6513             sleep 1;
6514         }
6515     };
6516
6517     if ($err) {
6518         eval { &$cancel_job(); };
6519         die "mirroring error: $err";
6520     }
6521
6522     if ($vmiddst != $vmid) {
6523         # if we clone a disk for a new target vm, we don't switch the disk
6524         &$cancel_job(); # so we call block-job-cancel
6525     }
6526 }
6527
6528 sub clone_disk {
6529     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6530         $newvmid, $storage, $format, $full, $newvollist) = @_;
6531
6532     my $newvolid;
6533
6534     if (!$full) {
6535         print "create linked clone of drive $drivename ($drive->{file})\n";
6536         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6537         push @$newvollist, $newvolid;
6538     } else {
6539         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6540         $storeid = $storage if $storage;
6541
6542         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6543         if (!$format) {
6544             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6545             $format = qemu_img_format($scfg, $volname);
6546         }
6547
6548         # test if requested format is supported - else use default
6549         my $supported = grep { $_ eq $format } @$validFormats;
6550         $format = $defFormat if !$supported;
6551
6552         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6553
6554         print "create full clone of drive $drivename ($drive->{file})\n";
6555         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6556         push @$newvollist, $newvolid;
6557
6558         PVE::Storage::activate_volumes($storecfg, $newvollist);
6559
6560         if (!$running || $snapname) {
6561             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6562         } else {
6563             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6564         }
6565     }
6566
6567     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6568
6569     my $disk = $drive;
6570     $disk->{format} = undef;
6571     $disk->{file} = $newvolid;
6572     $disk->{size} = $size;
6573
6574     return $disk;
6575 }
6576
6577 # this only works if VM is running
6578 sub get_current_qemu_machine {
6579     my ($vmid) = @_;
6580
6581     my $cmd = { execute => 'query-machines', arguments => {} };
6582     my $res = vm_qmp_command($vmid, $cmd);
6583
6584     my ($current, $default);
6585     foreach my $e (@$res) {
6586         $default = $e->{name} if $e->{'is-default'};
6587         $current = $e->{name} if $e->{'is-current'};
6588     }
6589
6590     # fallback to the default machine if current is not supported by qemu
6591     return $current || $default || 'pc';
6592 }
6593
6594 sub qemu_machine_feature_enabled {
6595     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6596
6597     my $current_major;
6598     my $current_minor;
6599
6600     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6601
6602         $current_major = $3;
6603         $current_minor = $4;
6604
6605     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6606
6607         $current_major = $1;
6608         $current_minor = $2;
6609     }
6610
6611     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6612
6613
6614 }
6615
6616 sub qemu_machine_pxe {
6617     my ($vmid, $conf, $machine) = @_;
6618
6619     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6620
6621     foreach my $opt (keys %$conf) {
6622         next if $opt !~ m/^net(\d+)$/;
6623         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6624         next if !$net;
6625         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6626         return $machine.".pxe" if $romfile =~ m/pxe/;
6627         last;
6628     }
6629
6630 }
6631
6632 sub qemu_use_old_bios_files {
6633     my ($machine_type) = @_;
6634
6635     return if !$machine_type;
6636
6637     my $use_old_bios_files = undef;
6638
6639     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6640         $machine_type = $1;
6641         $use_old_bios_files = 1;
6642     } else {
6643         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6644         # load new efi bios files on migration. So this hack is required to allow
6645         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6646         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6647         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6648     }
6649
6650     return ($use_old_bios_files, $machine_type);
6651 }
6652
6653 sub lspci {
6654
6655     my $devices = {};
6656
6657     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6658             my (undef, $id, $function) = @_;
6659             my $res = { id => $id, function => $function};
6660             push @{$devices->{$id}}, $res;
6661     });
6662
6663     return $devices;
6664 }
6665
6666 sub vm_iothreads_list {
6667     my ($vmid) = @_;
6668
6669     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6670
6671     my $iothreads = {};
6672     foreach my $iothread (@$res) {
6673         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6674     }
6675
6676     return $iothreads;
6677 }
6678
6679 sub scsihw_infos {
6680     my ($conf, $drive) = @_;
6681
6682     my $maxdev = 0;
6683
6684     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6685         $maxdev = 7;
6686     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6687         $maxdev = 1;
6688     } else {
6689         $maxdev = 256;
6690     }
6691
6692     my $controller = int($drive->{index} / $maxdev);
6693     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6694
6695     return ($maxdev, $controller, $controller_prefix);
6696 }
6697
6698 # bash completion helper
6699
6700 sub complete_backup_archives {
6701     my ($cmdname, $pname, $cvalue) = @_;
6702
6703     my $cfg = PVE::Storage::config();
6704
6705     my $storeid;
6706
6707     if ($cvalue =~ m/^([^:]+):/) {
6708         $storeid = $1;
6709     }
6710
6711     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6712
6713     my $res = [];
6714     foreach my $id (keys %$data) {
6715         foreach my $item (@{$data->{$id}}) {
6716             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6717             push @$res, $item->{volid} if defined($item->{volid});
6718         }
6719     }
6720
6721     return $res;
6722 }
6723
6724 my $complete_vmid_full = sub {
6725     my ($running) = @_;
6726
6727     my $idlist = vmstatus();
6728
6729     my $res = [];
6730
6731     foreach my $id (keys %$idlist) {
6732         my $d = $idlist->{$id};
6733         if (defined($running)) {
6734             next if $d->{template};
6735             next if $running && $d->{status} ne 'running';
6736             next if !$running && $d->{status} eq 'running';
6737         }
6738         push @$res, $id;
6739
6740     }
6741     return $res;
6742 };
6743
6744 sub complete_vmid {
6745     return &$complete_vmid_full();
6746 }
6747
6748 sub complete_vmid_stopped {
6749     return &$complete_vmid_full(0);
6750 }
6751
6752 sub complete_vmid_running {
6753     return &$complete_vmid_full(1);
6754 }
6755
6756 sub complete_storage {
6757
6758     my $cfg = PVE::Storage::config();
6759     my $ids = $cfg->{ids};
6760
6761     my $res = [];
6762     foreach my $sid (keys %$ids) {
6763         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6764         next if !$ids->{$sid}->{content}->{images};
6765         push @$res, $sid;
6766     }
6767
6768     return $res;
6769 }
6770
6771 1;