PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbformat = {
 731     host => {
 732         default_key => 1,
 733         type => 'string', format => 'pve-qm-usb-device',
 734         format_description => 'HOSTUSBDEVICE|spice',
 735         description => 'The Host USB device or port or the value spice',
 736     },
 737     usb3 => {
 738         optional => 1,
 739         type => 'boolean',
 740         format_description => 'yes|no',
 741         description => 'Specifies whether if given host option is a USB3 device or port',
 742     },
 743 };
 744
 745 my $usbdesc = {
 746     optional => 1,
 747     type => 'string', format => $usbformat,
 748     description => <<EODESCR,
 749 Configure an USB device (n is 0 to 4). This can be used to
 750 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 751
 752 'bus-port(.port)*' (decimal numbers) or
 753 'vendor_id:product_id' (hexadeciaml numbers) or
 754 'spice'
 755
 756 You can use the 'lsusb -t' command to list existing usb devices.
 757
 758 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 759
 760 The value 'spice' can be used to add a usb redirection devices for spice.
 761
 762 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 763
 764 EODESCR
 765 };
 766 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 767
 768 my $hostpcidesc = {
 769         optional => 1,
 770         type => 'string', format => 'pve-qm-hostpci',
 771         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 772         description => <<EODESCR,
 773 Map host pci devices. HOSTPCIDEVICE syntax is:
 774
 775 'bus:dev.func' (hexadecimal numbers)
 776
 777 You can us the 'lspci' command to list existing pci devices.
 778
 779 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 780
 781 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 782
 783 Experimental: user reported problems with this option.
 784 EODESCR
 785 };
 786 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 787
 788 my $serialdesc = {
 789         optional => 1,
 790         type => 'string',
 791         pattern => '(/dev/.+|socket)',
 792         description =>  <<EODESCR,
 793 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 794
 795 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 796
 797 Experimental: user reported problems with this option.
 798 EODESCR
 799 };
 800
 801 my $paralleldesc= {
 802         optional => 1,
 803         type => 'string',
 804         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 805         description =>  <<EODESCR,
 806 Map host parallel devices (n is 0 to 2).
 807
 808 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 809
 810 Experimental: user reported problems with this option.
 811 EODESCR
 812 };
 813
 814 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 815     $confdesc->{"parallel$i"} = $paralleldesc;
 816 }
 817
 818 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 819     $confdesc->{"serial$i"} = $serialdesc;
 820 }
 821
 822 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 823     $confdesc->{"hostpci$i"} = $hostpcidesc;
 824 }
 825
 826 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 827     $drivename_hash->{"ide$i"} = 1;
 828     $confdesc->{"ide$i"} = $idedesc;
 829 }
 830
 831 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 832     $drivename_hash->{"sata$i"} = 1;
 833     $confdesc->{"sata$i"} = $satadesc;
 834 }
 835
 836 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 837     $drivename_hash->{"scsi$i"} = 1;
 838     $confdesc->{"scsi$i"} = $scsidesc ;
 839 }
 840
 841 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 842     $drivename_hash->{"virtio$i"} = 1;
 843     $confdesc->{"virtio$i"} = $virtiodesc;
 844 }
 845
 846 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 847     $confdesc->{"usb$i"} = $usbdesc;
 848 }
 849
 850 my $unuseddesc = {
 851     optional => 1,
 852     type => 'string', format => 'pve-volume-id',
 853     description => "Reference to unused volumes.",
 854 };
 855
 856 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 857     $confdesc->{"unused$i"} = $unuseddesc;
 858 }
 859
 860 my $kvm_api_version = 0;
 861
 862 sub kvm_version {
 863
 864     return $kvm_api_version if $kvm_api_version;
 865
 866     my $fh = IO::File->new("</dev/kvm") ||
 867         return 0;
 868
 869     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 870         $kvm_api_version = $v;
 871     }
 872
 873     $fh->close();
 874
 875     return  $kvm_api_version;
 876 }
 877
 878 my $kvm_user_version;
 879
 880 sub kvm_user_version {
 881
 882     return $kvm_user_version if $kvm_user_version;
 883
 884     $kvm_user_version = 'unknown';
 885
 886     my $code = sub {
 887         my $line = shift;
 888         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 889             $kvm_user_version = $2;
 890         }
 891     };
 892
 893     eval { run_command("kvm -version", outfunc => $code); };
 894     warn $@ if $@;
 895
 896     return $kvm_user_version;
 897
 898 }
 899
 900 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 901
 902 sub disknames {
 903     # order is important - used to autoselect boot disk
 904     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 905             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 906             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 907             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 908 }
 909
 910 sub valid_drivename {
 911     my $dev = shift;
 912
 913     return defined($drivename_hash->{$dev});
 914 }
 915
 916 sub option_exists {
 917     my $key = shift;
 918     return defined($confdesc->{$key});
 919 }
 920
 921 sub nic_models {
 922     return $nic_model_list;
 923 }
 924
 925 sub os_list_description {
 926
 927     return {
 928         other => 'Other',
 929         wxp => 'Windows XP',
 930         w2k => 'Windows 2000',
 931         w2k3 =>, 'Windows 2003',
 932         w2k8 => 'Windows 2008',
 933         wvista => 'Windows Vista',
 934         win7 => 'Windows 7',
 935         win8 => 'Windows 8/2012',
 936         l24 => 'Linux 2.4',
 937         l26 => 'Linux 2.6',
 938     };
 939 }
 940
 941 my $cdrom_path;
 942
 943 sub get_cdrom_path {
 944
 945     return  $cdrom_path if $cdrom_path;
 946
 947     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 948     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 949     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 950 }
 951
 952 sub get_iso_path {
 953     my ($storecfg, $vmid, $cdrom) = @_;
 954
 955     if ($cdrom eq 'cdrom') {
 956         return get_cdrom_path();
 957     } elsif ($cdrom eq 'none') {
 958         return '';
 959     } elsif ($cdrom =~ m|^/|) {
 960         return $cdrom;
 961     } else {
 962         return PVE::Storage::path($storecfg, $cdrom);
 963     }
 964 }
 965
 966 # try to convert old style file names to volume IDs
 967 sub filename_to_volume_id {
 968     my ($vmid, $file, $media) = @_;
 969
 970     if (!($file eq 'none' || $file eq 'cdrom' ||
 971           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 972
 973         return undef if $file =~ m|/|;
 974
 975         if ($media && $media eq 'cdrom') {
 976             $file = "local:iso/$file";
 977         } else {
 978             $file = "local:$vmid/$file";
 979         }
 980     }
 981
 982     return $file;
 983 }
 984
 985 sub verify_media_type {
 986     my ($opt, $vtype, $media) = @_;
 987
 988     return if !$media;
 989
 990     my $etype;
 991     if ($media eq 'disk') {
 992         $etype = 'images';
 993     } elsif ($media eq 'cdrom') {
 994         $etype = 'iso';
 995     } else {
 996         die "internal error";
 997     }
 998
 999     return if ($vtype eq $etype);
1000
1001     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1002 }
1003
1004 sub cleanup_drive_path {
1005     my ($opt, $storecfg, $drive) = @_;
1006
1007     # try to convert filesystem paths to volume IDs
1008
1009     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1010         ($drive->{file} !~ m|^/dev/.+|) &&
1011         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1012         ($drive->{file} !~ m/^\d+$/)) {
1013         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1014         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1015         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1016         verify_media_type($opt, $vtype, $drive->{media});
1017         $drive->{file} = $volid;
1018     }
1019
1020     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1021 }
1022
1023 sub create_conf_nolock {
1024     my ($vmid, $settings) = @_;
1025
1026     my $filename = config_file($vmid);
1027
1028     die "configuration file '$filename' already exists\n" if -f $filename;
1029
1030     my $defaults = load_defaults();
1031
1032     $settings->{name} = "vm$vmid" if !$settings->{name};
1033     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
1034
1035     my $data = '';
1036     foreach my $opt (keys %$settings) {
1037         next if !$confdesc->{$opt};
1038
1039         my $value = $settings->{$opt};
1040         next if !$value;
1041
1042         $data .= "$opt: $value\n";
1043     }
1044
1045     PVE::Tools::file_set_contents($filename, $data);
1046 }
1047
1048 sub parse_hotplug_features {
1049     my ($data) = @_;
1050
1051     my $res = {};
1052
1053     return $res if $data eq '0';
1054
1055     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1056
1057     foreach my $feature (PVE::Tools::split_list($data)) {
1058         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1059             $res->{$1} = 1;
1060         } else {
1061             warn "ignoring unknown hotplug feature '$feature'\n";
1062         }
1063     }
1064     return $res;
1065 }
1066
1067 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1068 sub pve_verify_hotplug_features {
1069     my ($value, $noerr) = @_;
1070
1071     return $value if parse_hotplug_features($value);
1072
1073     return undef if $noerr;
1074
1075     die "unable to parse hotplug option\n";
1076 }
1077
1078 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1079 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1080 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1081 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1082 #        [,iothread=on][,serial=serial][,model=model]
1083
1084 sub parse_drive {
1085     my ($key, $data) = @_;
1086
1087     my ($interface, $index);
1088
1089     if ($key =~ m/^([^\d]+)(\d+)$/) {
1090         $interface = $1;
1091         $index = $2;
1092     } else {
1093         return undef;
1094     }
1095
1096     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1097                                      : $confdesc->{$key}->{format};
1098     if (!$desc) {
1099         warn "invalid drive key: $key\n";
1100         return undef;
1101     }
1102     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1103     return undef if !$res;
1104     $res->{interface} = $interface;
1105     $res->{index} = $index;
1106
1107     my $error = 0;
1108     foreach my $opt (qw(bps bps_rd bps_wr)) {
1109         if (my $bps = defined(delete $res->{$opt})) {
1110             if (defined($res->{"m$opt"})) {
1111                 warn "both $opt and m$opt specified\n";
1112                 ++$error;
1113                 next;
1114             }
1115             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1116         }
1117     }
1118     return undef if $error;
1119
1120     return undef if $res->{mbps_rd} && $res->{mbps};
1121     return undef if $res->{mbps_wr} && $res->{mbps};
1122     return undef if $res->{iops_rd} && $res->{iops};
1123     return undef if $res->{iops_wr} && $res->{iops};
1124
1125     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1126         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1127         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1128         return undef if $res->{interface} eq 'virtio';
1129     }
1130
1131     if (my $size = $res->{size}) {
1132         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1133     }
1134
1135     return $res;
1136 }
1137
1138 sub print_drive {
1139     my ($vmid, $drive) = @_;
1140     my $data = { %$drive };
1141     delete $data->{$_} for qw(index interface);
1142     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1143 }
1144
1145 sub scsi_inquiry {
1146     my($fh, $noerr) = @_;
1147
1148     my $SG_IO = 0x2285;
1149     my $SG_GET_VERSION_NUM = 0x2282;
1150
1151     my $versionbuf = "\x00" x 8;
1152     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1153     if (!$ret) {
1154         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1155         return undef;
1156     }
1157     my $version = unpack("I", $versionbuf);
1158     if ($version < 30000) {
1159         die "scsi generic interface too old\n"  if !$noerr;
1160         return undef;
1161     }
1162
1163     my $buf = "\x00" x 36;
1164     my $sensebuf = "\x00" x 8;
1165     my $cmd = pack("C x3 C x1", 0x12, 36);
1166
1167     # see /usr/include/scsi/sg.h
1168     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1169
1170     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1171                       length($sensebuf), 0, length($buf), $buf,
1172                       $cmd, $sensebuf, 6000);
1173
1174     $ret = ioctl($fh, $SG_IO, $packet);
1175     if (!$ret) {
1176         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1177         return undef;
1178     }
1179
1180     my @res = unpack($sg_io_hdr_t, $packet);
1181     if ($res[17] || $res[18]) {
1182         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1183         return undef;
1184     }
1185
1186     my $res = {};
1187     (my $byte0, my $byte1, $res->{vendor},
1188      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1189
1190     $res->{removable} = $byte1 & 128 ? 1 : 0;
1191     $res->{type} = $byte0 & 31;
1192
1193     return $res;
1194 }
1195
1196 sub path_is_scsi {
1197     my ($path) = @_;
1198
1199     my $fh = IO::File->new("+<$path") || return undef;
1200     my $res = scsi_inquiry($fh, 1);
1201     close($fh);
1202
1203     return $res;
1204 }
1205
1206 sub machine_type_is_q35 {
1207     my ($conf) = @_;
1208
1209     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1210 }
1211
1212 sub print_tabletdevice_full {
1213     my ($conf) = @_;
1214
1215     my $q35 = machine_type_is_q35($conf);
1216
1217     # we use uhci for old VMs because tablet driver was buggy in older qemu
1218     my $usbbus = $q35 ? "ehci" : "uhci";
1219
1220     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1221 }
1222
1223 sub print_drivedevice_full {
1224     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1225
1226     my $device = '';
1227     my $maxdev = 0;
1228
1229     if ($drive->{interface} eq 'virtio') {
1230         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1231         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1232         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1233     } elsif ($drive->{interface} eq 'scsi') {
1234
1235         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1236         my $unit = $drive->{index} % $maxdev;
1237         my $devicetype = 'hd';
1238         my $path = '';
1239         if (drive_is_cdrom($drive)) {
1240               $devicetype = 'cd';
1241         } else {
1242               if ($drive->{file} =~ m|^/|) {
1243                   $path = $drive->{file};
1244               } else {
1245                   $path = PVE::Storage::path($storecfg, $drive->{file});
1246               }
1247
1248               if($path =~ m/^iscsi\:\/\//){
1249                   $devicetype = 'generic';
1250               } else {
1251                   if (my $info = path_is_scsi($path)) {
1252                       if ($info->{type} == 0) {
1253                           $devicetype = 'block';
1254                       } elsif ($info->{type} == 1) { # tape
1255                           $devicetype = 'generic';
1256                       }
1257                   }
1258               }
1259          }
1260
1261         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1262             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1263         } else {
1264             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1265         }
1266
1267     } elsif ($drive->{interface} eq 'ide'){
1268         $maxdev = 2;
1269         my $controller = int($drive->{index} / $maxdev);
1270         my $unit = $drive->{index} % $maxdev;
1271         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1272
1273         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1274         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1275             $device .= ",model=$model";
1276         }
1277     } elsif ($drive->{interface} eq 'sata'){
1278         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1279         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1280         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1281     } elsif ($drive->{interface} eq 'usb') {
1282         die "implement me";
1283         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1284     } else {
1285         die "unsupported interface type";
1286     }
1287
1288     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1289
1290     return $device;
1291 }
1292
1293 sub get_initiator_name {
1294     my $initiator;
1295
1296     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1297     while (defined(my $line = <$fh>)) {
1298         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1299         $initiator = $1;
1300         last;
1301     }
1302     $fh->close();
1303
1304     return $initiator;
1305 }
1306
1307 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1308 sub print_drive_full {
1309     my ($storecfg, $vmid, $drive) = @_;
1310
1311     my $path;
1312     my $volid = $drive->{file};
1313     my $format;
1314
1315     if (drive_is_cdrom($drive)) {
1316         $path = get_iso_path($storecfg, $vmid, $volid);
1317     } else {
1318         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1319         if ($storeid) {
1320             $path = PVE::Storage::path($storecfg, $volid);
1321             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1322             $format = qemu_img_format($scfg, $volname);
1323         } else {
1324             $path = $volid;
1325             $format = "raw";
1326         }
1327    }
1328
1329     my $opts = '';
1330     foreach my $o (@qemu_drive_options) {
1331         next if $o eq 'bootindex';
1332         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1333     }
1334
1335     $opts .= ",format=$format" if $format && !$drive->{format};
1336
1337     foreach my $o (qw(bps bps_rd bps_wr)) {
1338         my $v = $drive->{"m$o"};
1339         $opts .= ",$o=" . int($v*1024*1024) if $v;
1340     }
1341
1342     my $cache_direct = 0;
1343
1344     if (my $cache = $drive->{cache}) {
1345         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1346     } elsif (!drive_is_cdrom($drive)) {
1347         $opts .= ",cache=none";
1348         $cache_direct = 1;
1349     }
1350
1351     # aio native works only with O_DIRECT
1352     if (!$drive->{aio}) {
1353         if($cache_direct) {
1354             $opts .= ",aio=native";
1355         } else {
1356             $opts .= ",aio=threads";
1357         }
1358     }
1359
1360     if (!drive_is_cdrom($drive)) {
1361         my $detectzeroes;
1362         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1363             $detectzeroes = 'off';
1364         } elsif ($drive->{discard}) {
1365             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1366         } else {
1367             # This used to be our default with discard not being specified:
1368             $detectzeroes = 'on';
1369         }
1370         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1371     }
1372
1373     my $pathinfo = $path ? "file=$path," : '';
1374
1375     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1376 }
1377
1378 sub print_netdevice_full {
1379     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1380
1381     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1382
1383     my $device = $net->{model};
1384     if ($net->{model} eq 'virtio') {
1385          $device = 'virtio-net-pci';
1386      };
1387
1388     my $pciaddr = print_pci_addr("$netid", $bridges);
1389     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1390     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1391         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1392         my $vectors = $net->{queues} * 2 + 2;
1393         $tmpstr .= ",vectors=$vectors,mq=on";
1394     }
1395     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1396
1397     if ($use_old_bios_files) {
1398         my $romfile;
1399         if ($device eq 'virtio-net-pci') {
1400             $romfile = 'pxe-virtio.rom';
1401         } elsif ($device eq 'e1000') {
1402             $romfile = 'pxe-e1000.rom';
1403         } elsif ($device eq 'ne2k') {
1404             $romfile = 'pxe-ne2k_pci.rom';
1405         } elsif ($device eq 'pcnet') {
1406             $romfile = 'pxe-pcnet.rom';
1407         } elsif ($device eq 'rtl8139') {
1408             $romfile = 'pxe-rtl8139.rom';
1409         }
1410         $tmpstr .= ",romfile=$romfile" if $romfile;
1411     }
1412
1413     return $tmpstr;
1414 }
1415
1416 sub print_netdev_full {
1417     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1418
1419     my $i = '';
1420     if ($netid =~ m/^net(\d+)$/) {
1421         $i = int($1);
1422     }
1423
1424     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1425
1426     my $ifname = "tap${vmid}i$i";
1427
1428     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1429     die "interface name '$ifname' is too long (max 15 character)\n"
1430         if length($ifname) >= 16;
1431
1432     my $vhostparam = '';
1433     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1434
1435     my $vmname = $conf->{name} || "vm$vmid";
1436
1437     my $netdev = "";
1438     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1439
1440     if ($net->{bridge}) {
1441         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1442     } else {
1443         $netdev = "type=user,id=$netid,hostname=$vmname";
1444     }
1445
1446     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1447
1448     return $netdev;
1449 }
1450
1451 sub drive_is_cdrom {
1452     my ($drive) = @_;
1453
1454     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1455
1456 }
1457
1458 sub parse_numa {
1459     my ($data) = @_;
1460
1461     my $res = {};
1462
1463     foreach my $kvp (split(/,/, $data)) {
1464
1465         if ($kvp =~ m/^memory=(\S+)$/) {
1466             $res->{memory} = $1;
1467         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1468             $res->{policy} = $1;
1469         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1470             $res->{cpus}->{start} = $1;
1471             $res->{cpus}->{end} = $3;
1472         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1473             $res->{hostnodes}->{start} = $1;
1474             $res->{hostnodes}->{end} = $3;
1475         } else {
1476             return undef;
1477         }
1478     }
1479
1480     return $res;
1481 }
1482
1483 sub parse_hostpci {
1484     my ($value) = @_;
1485
1486     return undef if !$value;
1487
1488
1489     my @list = split(/,/, $value);
1490     my $found;
1491
1492     my $res = {};
1493     foreach my $kv (@list) {
1494
1495         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1496             $found = 1;
1497             if(defined($4)){
1498                 push @{$res->{pciid}}, { id => $2 , function => $4};
1499
1500             }else{
1501                 my $pcidevices = lspci($2);
1502                 $res->{pciid} = $pcidevices->{$2};
1503             }
1504         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1505             $res->{rombar} = $1;
1506         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1507             $res->{'x-vga'} = $1;
1508         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1509             $res->{pcie} = 1 if $1 == 1;
1510         } else {
1511             warn "unknown hostpci setting '$kv'\n";
1512         }
1513     }
1514
1515     return undef if !$found;
1516
1517     return $res;
1518 }
1519
1520 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1521 sub parse_net {
1522     my ($data) = @_;
1523
1524     my $res = {};
1525
1526     foreach my $kvp (split(/,/, $data)) {
1527
1528         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1529             my $model = lc($1);
1530             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1531             $res->{model} = $model;
1532             $res->{macaddr} = $mac;
1533         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1534             $res->{bridge} = $1;
1535         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1536             $res->{queues} = $1;
1537         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1538             $res->{rate} = $1;
1539         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1540             $res->{tag} = $1;
1541         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1542             $res->{trunks} = $1;
1543         } elsif ($kvp =~ m/^firewall=([01])$/) {
1544             $res->{firewall} = $1;
1545         } elsif ($kvp =~ m/^link_down=([01])$/) {
1546             $res->{link_down} = $1;
1547         } else {
1548             return undef;
1549         }
1550
1551     }
1552
1553     return undef if !$res->{model};
1554
1555     return $res;
1556 }
1557
1558 sub print_net {
1559     my $net = shift;
1560
1561     my $res = "$net->{model}";
1562     $res .= "=$net->{macaddr}" if $net->{macaddr};
1563     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1564     $res .= ",rate=$net->{rate}" if $net->{rate};
1565     $res .= ",tag=$net->{tag}" if $net->{tag};
1566     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1567     $res .= ",firewall=1" if $net->{firewall};
1568     $res .= ",link_down=1" if $net->{link_down};
1569     $res .= ",queues=$net->{queues}" if $net->{queues};
1570
1571     return $res;
1572 }
1573
1574 sub add_random_macs {
1575     my ($settings) = @_;
1576
1577     foreach my $opt (keys %$settings) {
1578         next if $opt !~ m/^net(\d+)$/;
1579         my $net = parse_net($settings->{$opt});
1580         next if !$net;
1581         $settings->{$opt} = print_net($net);
1582     }
1583 }
1584
1585 sub add_unused_volume {
1586     my ($config, $volid) = @_;
1587
1588     my $key;
1589     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1590         my $test = "unused$ind";
1591         if (my $vid = $config->{$test}) {
1592             return if $vid eq $volid; # do not add duplicates
1593         } else {
1594             $key = $test;
1595         }
1596     }
1597
1598     die "To many unused volume - please delete them first.\n" if !$key;
1599
1600     $config->{$key} = $volid;
1601
1602     return $key;
1603 }
1604
1605 sub vm_is_volid_owner {
1606     my ($storecfg, $vmid, $volid) = @_;
1607
1608     if ($volid !~  m|^/|) {
1609         my ($path, $owner);
1610         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1611         if ($owner && ($owner == $vmid)) {
1612             return 1;
1613         }
1614     }
1615
1616     return undef;
1617 }
1618
1619 sub split_flagged_list {
1620     my $text = shift || '';
1621     $text =~ s/[,;]/ /g;
1622     $text =~ s/^\s+//;
1623     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1624 }
1625
1626 sub join_flagged_list {
1627     my ($how, $lst) = @_;
1628     join $how, map { $lst->{$_} . $_ } keys %$lst;
1629 }
1630
1631 sub vmconfig_delete_pending_option {
1632     my ($conf, $key, $force) = @_;
1633
1634     delete $conf->{pending}->{$key};
1635     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1636     $pending_delete_hash->{$key} = $force ? '!' : '';
1637     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1638 }
1639
1640 sub vmconfig_undelete_pending_option {
1641     my ($conf, $key) = @_;
1642
1643     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1644     delete $pending_delete_hash->{$key};
1645
1646     if (%$pending_delete_hash) {
1647         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1648     } else {
1649         delete $conf->{pending}->{delete};
1650     }
1651 }
1652
1653 sub vmconfig_register_unused_drive {
1654     my ($storecfg, $vmid, $conf, $drive) = @_;
1655
1656     if (!drive_is_cdrom($drive)) {
1657         my $volid = $drive->{file};
1658         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1659             add_unused_volume($conf, $volid, $vmid);
1660         }
1661     }
1662 }
1663
1664 sub vmconfig_cleanup_pending {
1665     my ($conf) = @_;
1666
1667     # remove pending changes when nothing changed
1668     my $changes;
1669     foreach my $opt (keys %{$conf->{pending}}) {
1670         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1671             $changes = 1;
1672             delete $conf->{pending}->{$opt};
1673         }
1674     }
1675
1676     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1677     my $pending_delete_hash = {};
1678     while (my ($opt, $force) = each %$current_delete_hash) {
1679         if (defined($conf->{$opt})) {
1680             $pending_delete_hash->{$opt} = $force;
1681         } else {
1682             $changes = 1;
1683         }
1684     }
1685
1686     if (%$pending_delete_hash) {
1687         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1688     } else {
1689         delete $conf->{pending}->{delete};
1690     }
1691
1692     return $changes;
1693 }
1694
1695 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1696 my $smbios1_desc = {
1697     uuid => {
1698         type => 'string',
1699         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1700         format_description => 'UUID',
1701         optional => 1,
1702     },
1703     version => {
1704         type => 'string',
1705         pattern => '\S+',
1706         format_description => 'str',
1707         optional => 1,
1708     },
1709     serial => {
1710         type => 'string',
1711         pattern => '\S+',
1712         format_description => 'str',
1713         optional => 1,
1714     },
1715     manufacturer => {
1716         type => 'string',
1717         pattern => '\S+',
1718         format_description => 'name',
1719         optional => 1,
1720     },
1721     product => {
1722         type => 'string',
1723         pattern => '\S+',
1724         format_description => 'name',
1725         optional => 1,
1726     },
1727     sku => {
1728         type => 'string',
1729         pattern => '\S+',
1730         format_description => 'str',
1731         optional => 1,
1732     },
1733     family => {
1734         type => 'string',
1735         pattern => '\S+',
1736         format_description => 'str',
1737         optional => 1,
1738     },
1739 };
1740
1741 sub parse_smbios1 {
1742     my ($data) = @_;
1743
1744     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1745     warn $@ if $@;
1746     return $res;
1747 }
1748
1749 sub print_smbios1 {
1750     my ($smbios1) = @_;
1751     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1752 }
1753
1754 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1755
1756 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1757 sub verify_bootdisk {
1758     my ($value, $noerr) = @_;
1759
1760     return $value if valid_drivename($value);
1761
1762     return undef if $noerr;
1763
1764     die "invalid boot disk '$value'\n";
1765 }
1766
1767 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1768 sub verify_numa {
1769     my ($value, $noerr) = @_;
1770
1771     return $value if parse_numa($value);
1772
1773     return undef if $noerr;
1774
1775     die "unable to parse numa options\n";
1776 }
1777
1778 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1779 sub verify_net {
1780     my ($value, $noerr) = @_;
1781
1782     return $value if parse_net($value);
1783
1784     return undef if $noerr;
1785
1786     die "unable to parse network options\n";
1787 }
1788
1789 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1790 sub verify_hostpci {
1791     my ($value, $noerr) = @_;
1792
1793     return $value if parse_hostpci($value);
1794
1795     return undef if $noerr;
1796
1797     die "unable to parse pci id\n";
1798 }
1799
1800 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1801 sub verify_watchdog {
1802     my ($value, $noerr) = @_;
1803
1804     return $value if parse_watchdog($value);
1805
1806     return undef if $noerr;
1807
1808     die "unable to parse watchdog options\n";
1809 }
1810
1811 sub parse_watchdog {
1812     my ($value) = @_;
1813
1814     return undef if !$value;
1815
1816     my $res = {};
1817
1818     foreach my $p (split(/,/, $value)) {
1819         next if $p =~ m/^\s*$/;
1820
1821         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1822             $res->{model} = $2;
1823         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1824             $res->{action} = $2;
1825         } else {
1826             return undef;
1827         }
1828     }
1829
1830     return $res;
1831 }
1832
1833 sub parse_usb_device {
1834     my ($value) = @_;
1835
1836     return undef if !$value;
1837
1838     my $res = {};
1839     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1840         $res->{vendorid} = $2;
1841         $res->{productid} = $4;
1842     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1843         $res->{hostbus} = $1;
1844         $res->{hostport} = $2;
1845     } elsif ($value =~ m/^spice$/i) {
1846         $res->{spice} = 1;
1847     } else {
1848         return undef;
1849     }
1850
1851     return $res;
1852 }
1853
1854 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1855 sub verify_usb_device {
1856     my ($value, $noerr) = @_;
1857
1858     return $value if parse_usb_device($value);
1859
1860     return undef if $noerr;
1861
1862     die "unable to parse usb device\n";
1863 }
1864
1865 # add JSON properties for create and set function
1866 sub json_config_properties {
1867     my $prop = shift;
1868
1869     foreach my $opt (keys %$confdesc) {
1870         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1871         $prop->{$opt} = $confdesc->{$opt};
1872     }
1873
1874     return $prop;
1875 }
1876
1877 sub check_type {
1878     my ($key, $value) = @_;
1879
1880     die "unknown setting '$key'\n" if !$confdesc->{$key};
1881
1882     my $type = $confdesc->{$key}->{type};
1883
1884     if (!defined($value)) {
1885         die "got undefined value\n";
1886     }
1887
1888     if ($value =~ m/[\n\r]/) {
1889         die "property contains a line feed\n";
1890     }
1891
1892     if ($type eq 'boolean') {
1893         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1894         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1895         die "type check ('boolean') failed - got '$value'\n";
1896     } elsif ($type eq 'integer') {
1897         return int($1) if $value =~ m/^(\d+)$/;
1898         die "type check ('integer') failed - got '$value'\n";
1899     } elsif ($type eq 'number') {
1900         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1901         die "type check ('number') failed - got '$value'\n";
1902     } elsif ($type eq 'string') {
1903         if (my $fmt = $confdesc->{$key}->{format}) {
1904             if ($fmt eq 'pve-qm-drive') {
1905                 # special case - we need to pass $key to parse_drive()
1906                 my $drive = parse_drive($key, $value);
1907                 return $value if $drive;
1908                 die "unable to parse drive options\n";
1909             }
1910             PVE::JSONSchema::check_format($fmt, $value);
1911             return $value;
1912         }
1913         $value =~ s/^\"(.*)\"$/$1/;
1914         return $value;
1915     } else {
1916         die "internal error"
1917     }
1918 }
1919
1920 sub lock_config_full {
1921     my ($vmid, $timeout, $code, @param) = @_;
1922
1923     my $filename = config_file_lock($vmid);
1924
1925     my $res = lock_file($filename, $timeout, $code, @param);
1926
1927     die $@ if $@;
1928
1929     return $res;
1930 }
1931
1932 sub lock_config_mode {
1933     my ($vmid, $timeout, $shared, $code, @param) = @_;
1934
1935     my $filename = config_file_lock($vmid);
1936
1937     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1938
1939     die $@ if $@;
1940
1941     return $res;
1942 }
1943
1944 sub lock_config {
1945     my ($vmid, $code, @param) = @_;
1946
1947     return lock_config_full($vmid, 10, $code, @param);
1948 }
1949
1950 sub cfs_config_path {
1951     my ($vmid, $node) = @_;
1952
1953     $node = $nodename if !$node;
1954     return "nodes/$node/qemu-server/$vmid.conf";
1955 }
1956
1957 sub check_iommu_support{
1958     #fixme : need to check IOMMU support
1959     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1960
1961     my $iommu=1;
1962     return $iommu;
1963
1964 }
1965
1966 sub config_file {
1967     my ($vmid, $node) = @_;
1968
1969     my $cfspath = cfs_config_path($vmid, $node);
1970     return "/etc/pve/$cfspath";
1971 }
1972
1973 sub config_file_lock {
1974     my ($vmid) = @_;
1975
1976     return "$lock_dir/lock-$vmid.conf";
1977 }
1978
1979 sub touch_config {
1980     my ($vmid) = @_;
1981
1982     my $conf = config_file($vmid);
1983     utime undef, undef, $conf;
1984 }
1985
1986 sub destroy_vm {
1987     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1988
1989     my $conffile = config_file($vmid);
1990
1991     my $conf = load_config($vmid);
1992
1993     check_lock($conf) if !$skiplock;
1994
1995     # only remove disks owned by this VM
1996     foreach_drive($conf, sub {
1997         my ($ds, $drive) = @_;
1998
1999         return if drive_is_cdrom($drive);
2000
2001         my $volid = $drive->{file};
2002
2003         return if !$volid || $volid =~ m|^/|;
2004
2005         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2006         return if !$path || !$owner || ($owner != $vmid);
2007
2008         PVE::Storage::vdisk_free($storecfg, $volid);
2009     });
2010
2011     if ($keep_empty_config) {
2012         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2013     } else {
2014         unlink $conffile;
2015     }
2016
2017     # also remove unused disk
2018     eval {
2019         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2020
2021         eval {
2022             PVE::Storage::foreach_volid($dl, sub {
2023                 my ($volid, $sid, $volname, $d) = @_;
2024                 PVE::Storage::vdisk_free($storecfg, $volid);
2025             });
2026         };
2027         warn $@ if $@;
2028
2029     };
2030     warn $@ if $@;
2031 }
2032
2033 sub load_config {
2034     my ($vmid, $node) = @_;
2035
2036     my $cfspath = cfs_config_path($vmid, $node);
2037
2038     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2039
2040     die "no such VM ('$vmid')\n" if !defined($conf);
2041
2042     return $conf;
2043 }
2044
2045 sub parse_vm_config {
2046     my ($filename, $raw) = @_;
2047
2048     return undef if !defined($raw);
2049
2050     my $res = {
2051         digest => Digest::SHA::sha1_hex($raw),
2052         snapshots => {},
2053         pending => {},
2054     };
2055
2056     $filename =~ m|/qemu-server/(\d+)\.conf$|
2057         || die "got strange filename '$filename'";
2058
2059     my $vmid = $1;
2060
2061     my $conf = $res;
2062     my $descr;
2063     my $section = '';
2064
2065     my @lines = split(/\n/, $raw);
2066     foreach my $line (@lines) {
2067         next if $line =~ m/^\s*$/;
2068
2069         if ($line =~ m/^\[PENDING\]\s*$/i) {
2070             $section = 'pending';
2071             if (defined($descr)) {
2072                 $descr =~ s/\s+$//;
2073                 $conf->{description} = $descr;
2074             }
2075             $descr = undef;
2076             $conf = $res->{$section} = {};
2077             next;
2078
2079         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2080             $section = $1;
2081             if (defined($descr)) {
2082                 $descr =~ s/\s+$//;
2083                 $conf->{description} = $descr;
2084             }
2085             $descr = undef;
2086             $conf = $res->{snapshots}->{$section} = {};
2087             next;
2088         }
2089
2090         if ($line =~ m/^\#(.*)\s*$/) {
2091             $descr = '' if !defined($descr);
2092             $descr .= PVE::Tools::decode_text($1) . "\n";
2093             next;
2094         }
2095
2096         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2097             $descr = '' if !defined($descr);
2098             $descr .= PVE::Tools::decode_text($2);
2099         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2100             $conf->{snapstate} = $1;
2101         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2102             my $key = $1;
2103             my $value = $2;
2104             $conf->{$key} = $value;
2105         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2106             my $value = $1;
2107             if ($section eq 'pending') {
2108                 $conf->{delete} = $value; # we parse this later
2109             } else {
2110                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2111             }
2112         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2113             my $key = $1;
2114             my $value = $2;
2115             eval { $value = check_type($key, $value); };
2116             if ($@) {
2117                 warn "vm $vmid - unable to parse value of '$key' - $@";
2118             } else {
2119                 my $fmt = $confdesc->{$key}->{format};
2120                 if ($fmt && $fmt eq 'pve-qm-drive') {
2121                     my $v = parse_drive($key, $value);
2122                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2123                         $v->{file} = $volid;
2124                         $value = print_drive($vmid, $v);
2125                     } else {
2126                         warn "vm $vmid - unable to parse value of '$key'\n";
2127                         next;
2128                     }
2129                 }
2130
2131                 if ($key eq 'cdrom') {
2132                     $conf->{ide2} = $value;
2133                 } else {
2134                     $conf->{$key} = $value;
2135                 }
2136             }
2137         }
2138     }
2139
2140     if (defined($descr)) {
2141         $descr =~ s/\s+$//;
2142         $conf->{description} = $descr;
2143     }
2144     delete $res->{snapstate}; # just to be sure
2145
2146     return $res;
2147 }
2148
2149 sub write_vm_config {
2150     my ($filename, $conf) = @_;
2151
2152     delete $conf->{snapstate}; # just to be sure
2153
2154     if ($conf->{cdrom}) {
2155         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2156         $conf->{ide2} = $conf->{cdrom};
2157         delete $conf->{cdrom};
2158     }
2159
2160     # we do not use 'smp' any longer
2161     if ($conf->{sockets}) {
2162         delete $conf->{smp};
2163     } elsif ($conf->{smp}) {
2164         $conf->{sockets} = $conf->{smp};
2165         delete $conf->{cores};
2166         delete $conf->{smp};
2167     }
2168
2169     my $used_volids = {};
2170
2171     my $cleanup_config = sub {
2172         my ($cref, $pending, $snapname) = @_;
2173
2174         foreach my $key (keys %$cref) {
2175             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2176                 $key eq 'snapstate' || $key eq 'pending';
2177             my $value = $cref->{$key};
2178             if ($key eq 'delete') {
2179                 die "propertry 'delete' is only allowed in [PENDING]\n"
2180                     if !$pending;
2181                 # fixme: check syntax?
2182                 next;
2183             }
2184             eval { $value = check_type($key, $value); };
2185             die "unable to parse value of '$key' - $@" if $@;
2186
2187             $cref->{$key} = $value;
2188
2189             if (!$snapname && valid_drivename($key)) {
2190                 my $drive = parse_drive($key, $value);
2191                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2192             }
2193         }
2194     };
2195
2196     &$cleanup_config($conf);
2197
2198     &$cleanup_config($conf->{pending}, 1);
2199
2200     foreach my $snapname (keys %{$conf->{snapshots}}) {
2201         die "internal error" if $snapname eq 'pending';
2202         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2203     }
2204
2205     # remove 'unusedX' settings if we re-add a volume
2206     foreach my $key (keys %$conf) {
2207         my $value = $conf->{$key};
2208         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2209             delete $conf->{$key};
2210         }
2211     }
2212
2213     my $generate_raw_config = sub {
2214         my ($conf, $pending) = @_;
2215
2216         my $raw = '';
2217
2218         # add description as comment to top of file
2219         if (defined(my $descr = $conf->{description})) {
2220             if ($descr) {
2221                 foreach my $cl (split(/\n/, $descr)) {
2222                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2223                 }
2224             } else {
2225                 $raw .= "#\n" if $pending;
2226             }
2227         }
2228
2229         foreach my $key (sort keys %$conf) {
2230             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2231             $raw .= "$key: $conf->{$key}\n";
2232         }
2233         return $raw;
2234     };
2235
2236     my $raw = &$generate_raw_config($conf);
2237
2238     if (scalar(keys %{$conf->{pending}})){
2239         $raw .= "\n[PENDING]\n";
2240         $raw .= &$generate_raw_config($conf->{pending}, 1);
2241     }
2242
2243     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2244         $raw .= "\n[$snapname]\n";
2245         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2246     }
2247
2248     return $raw;
2249 }
2250
2251 sub write_config {
2252     my ($vmid, $conf) = @_;
2253
2254     my $cfspath = cfs_config_path($vmid);
2255
2256     PVE::Cluster::cfs_write_file($cfspath, $conf);
2257 }
2258
2259 sub load_defaults {
2260
2261     my $res = {};
2262
2263     # we use static defaults from our JSON schema configuration
2264     foreach my $key (keys %$confdesc) {
2265         if (defined(my $default = $confdesc->{$key}->{default})) {
2266             $res->{$key} = $default;
2267         }
2268     }
2269
2270     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2272
2273     return $res;
2274 }
2275
2276 sub config_list {
2277     my $vmlist = PVE::Cluster::get_vmlist();
2278     my $res = {};
2279     return $res if !$vmlist || !$vmlist->{ids};
2280     my $ids = $vmlist->{ids};
2281
2282     foreach my $vmid (keys %$ids) {
2283         my $d = $ids->{$vmid};
2284         next if !$d->{node} || $d->{node} ne $nodename;
2285         next if !$d->{type} || $d->{type} ne 'qemu';
2286         $res->{$vmid}->{exists} = 1;
2287     }
2288     return $res;
2289 }
2290
2291 # test if VM uses local resources (to prevent migration)
2292 sub check_local_resources {
2293     my ($conf, $noerr) = @_;
2294
2295     my $loc_res = 0;
2296
2297     $loc_res = 1 if $conf->{hostusb}; # old syntax
2298     $loc_res = 1 if $conf->{hostpci}; # old syntax
2299
2300     foreach my $k (keys %$conf) {
2301         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2302         # sockets are safe: they will recreated be on the target side post-migrate
2303         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2304         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2305     }
2306
2307     die "VM uses local resources\n" if $loc_res && !$noerr;
2308
2309     return $loc_res;
2310 }
2311
2312 # check if used storages are available on all nodes (use by migrate)
2313 sub check_storage_availability {
2314     my ($storecfg, $conf, $node) = @_;
2315
2316     foreach_drive($conf, sub {
2317         my ($ds, $drive) = @_;
2318
2319         my $volid = $drive->{file};
2320         return if !$volid;
2321
2322         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2323         return if !$sid;
2324
2325         # check if storage is available on both nodes
2326         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2327         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2328    });
2329 }
2330
2331 # list nodes where all VM images are available (used by has_feature API)
2332 sub shared_nodes {
2333     my ($conf, $storecfg) = @_;
2334
2335     my $nodelist = PVE::Cluster::get_nodelist();
2336     my $nodehash = { map { $_ => 1 } @$nodelist };
2337     my $nodename = PVE::INotify::nodename();
2338
2339     foreach_drive($conf, sub {
2340         my ($ds, $drive) = @_;
2341
2342         my $volid = $drive->{file};
2343         return if !$volid;
2344
2345         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2346         if ($storeid) {
2347             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2348             if ($scfg->{disable}) {
2349                 $nodehash = {};
2350             } elsif (my $avail = $scfg->{nodes}) {
2351                 foreach my $node (keys %$nodehash) {
2352                     delete $nodehash->{$node} if !$avail->{$node};
2353                 }
2354             } elsif (!$scfg->{shared}) {
2355                 foreach my $node (keys %$nodehash) {
2356                     delete $nodehash->{$node} if $node ne $nodename
2357                 }
2358             }
2359         }
2360     });
2361
2362     return $nodehash
2363 }
2364
2365 sub check_lock {
2366     my ($conf) = @_;
2367
2368     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2369 }
2370
2371 sub check_cmdline {
2372     my ($pidfile, $pid) = @_;
2373
2374     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2375     if (defined($fh)) {
2376         my $line = <$fh>;
2377         $fh->close;
2378         return undef if !$line;
2379         my @param = split(/\0/, $line);
2380
2381         my $cmd = $param[0];
2382         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2383
2384         for (my $i = 0; $i < scalar (@param); $i++) {
2385             my $p = $param[$i];
2386             next if !$p;
2387             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2388                 my $p = $param[$i+1];
2389                 return 1 if $p && ($p eq $pidfile);
2390                 return undef;
2391             }
2392         }
2393     }
2394     return undef;
2395 }
2396
2397 sub check_running {
2398     my ($vmid, $nocheck, $node) = @_;
2399
2400     my $filename = config_file($vmid, $node);
2401
2402     die "unable to find configuration file for VM $vmid - no such machine\n"
2403         if !$nocheck && ! -f $filename;
2404
2405     my $pidfile = pidfile_name($vmid);
2406
2407     if (my $fd = IO::File->new("<$pidfile")) {
2408         my $st = stat($fd);
2409         my $line = <$fd>;
2410         close($fd);
2411
2412         my $mtime = $st->mtime;
2413         if ($mtime > time()) {
2414             warn "file '$filename' modified in future\n";
2415         }
2416
2417         if ($line =~ m/^(\d+)$/) {
2418             my $pid = $1;
2419             if (check_cmdline($pidfile, $pid)) {
2420                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2421                     return $pid;
2422                 }
2423             }
2424         }
2425     }
2426
2427     return undef;
2428 }
2429
2430 sub vzlist {
2431
2432     my $vzlist = config_list();
2433
2434     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2435
2436     while (defined(my $de = $fd->read)) {
2437         next if $de !~ m/^(\d+)\.pid$/;
2438         my $vmid = $1;
2439         next if !defined($vzlist->{$vmid});
2440         if (my $pid = check_running($vmid)) {
2441             $vzlist->{$vmid}->{pid} = $pid;
2442         }
2443     }
2444
2445     return $vzlist;
2446 }
2447
2448 sub disksize {
2449     my ($storecfg, $conf) = @_;
2450
2451     my $bootdisk = $conf->{bootdisk};
2452     return undef if !$bootdisk;
2453     return undef if !valid_drivename($bootdisk);
2454
2455     return undef if !$conf->{$bootdisk};
2456
2457     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2458     return undef if !defined($drive);
2459
2460     return undef if drive_is_cdrom($drive);
2461
2462     my $volid = $drive->{file};
2463     return undef if !$volid;
2464
2465     return $drive->{size};
2466 }
2467
2468 my $last_proc_pid_stat;
2469
2470 # get VM status information
2471 # This must be fast and should not block ($full == false)
2472 # We only query KVM using QMP if $full == true (this can be slow)
2473 sub vmstatus {
2474     my ($opt_vmid, $full) = @_;
2475
2476     my $res = {};
2477
2478     my $storecfg = PVE::Storage::config();
2479
2480     my $list = vzlist();
2481     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2482
2483     my $cpucount = $cpuinfo->{cpus} || 1;
2484
2485     foreach my $vmid (keys %$list) {
2486         next if $opt_vmid && ($vmid ne $opt_vmid);
2487
2488         my $cfspath = cfs_config_path($vmid);
2489         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2490
2491         my $d = {};
2492         $d->{pid} = $list->{$vmid}->{pid};
2493
2494         # fixme: better status?
2495         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2496
2497         my $size = disksize($storecfg, $conf);
2498         if (defined($size)) {
2499             $d->{disk} = 0; # no info available
2500             $d->{maxdisk} = $size;
2501         } else {
2502             $d->{disk} = 0;
2503             $d->{maxdisk} = 0;
2504         }
2505
2506         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2507         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2508         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2509
2510         $d->{name} = $conf->{name} || "VM $vmid";
2511         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2512
2513         if ($conf->{balloon}) {
2514             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2515             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2516         }
2517
2518         $d->{uptime} = 0;
2519         $d->{cpu} = 0;
2520         $d->{mem} = 0;
2521
2522         $d->{netout} = 0;
2523         $d->{netin} = 0;
2524
2525         $d->{diskread} = 0;
2526         $d->{diskwrite} = 0;
2527
2528         $d->{template} = is_template($conf);
2529
2530         $res->{$vmid} = $d;
2531     }
2532
2533     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2534     foreach my $dev (keys %$netdev) {
2535         next if $dev !~ m/^tap([1-9]\d*)i/;
2536         my $vmid = $1;
2537         my $d = $res->{$vmid};
2538         next if !$d;
2539
2540         $d->{netout} += $netdev->{$dev}->{receive};
2541         $d->{netin} += $netdev->{$dev}->{transmit};
2542
2543         if ($full) {
2544             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2545             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2546         }
2547
2548     }
2549
2550     my $ctime = gettimeofday;
2551
2552     foreach my $vmid (keys %$list) {
2553
2554         my $d = $res->{$vmid};
2555         my $pid = $d->{pid};
2556         next if !$pid;
2557
2558         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2559         next if !$pstat; # not running
2560
2561         my $used = $pstat->{utime} + $pstat->{stime};
2562
2563         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2564
2565         if ($pstat->{vsize}) {
2566             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2567         }
2568
2569         my $old = $last_proc_pid_stat->{$pid};
2570         if (!$old) {
2571             $last_proc_pid_stat->{$pid} = {
2572                 time => $ctime,
2573                 used => $used,
2574                 cpu => 0,
2575             };
2576             next;
2577         }
2578
2579         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2580
2581         if ($dtime > 1000) {
2582             my $dutime = $used -  $old->{used};
2583
2584             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2585             $last_proc_pid_stat->{$pid} = {
2586                 time => $ctime,
2587                 used => $used,
2588                 cpu => $d->{cpu},
2589             };
2590         } else {
2591             $d->{cpu} = $old->{cpu};
2592         }
2593     }
2594
2595     return $res if !$full;
2596
2597     my $qmpclient = PVE::QMPClient->new();
2598
2599     my $ballooncb = sub {
2600         my ($vmid, $resp) = @_;
2601
2602         my $info = $resp->{'return'};
2603         return if !$info->{max_mem};
2604
2605         my $d = $res->{$vmid};
2606
2607         # use memory assigned to VM
2608         $d->{maxmem} = $info->{max_mem};
2609         $d->{balloon} = $info->{actual};
2610
2611         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2612             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2613             $d->{freemem} = $info->{free_mem};
2614         }
2615
2616         $d->{ballooninfo} = $info;
2617     };
2618
2619     my $blockstatscb = sub {
2620         my ($vmid, $resp) = @_;
2621         my $data = $resp->{'return'} || [];
2622         my $totalrdbytes = 0;
2623         my $totalwrbytes = 0;
2624
2625         for my $blockstat (@$data) {
2626             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2627             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2628
2629             $blockstat->{device} =~ s/drive-//;
2630             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2631         }
2632         $res->{$vmid}->{diskread} = $totalrdbytes;
2633         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2634     };
2635
2636     my $statuscb = sub {
2637         my ($vmid, $resp) = @_;
2638
2639         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2640         # this fails if ballon driver is not loaded, so this must be
2641         # the last commnand (following command are aborted if this fails).
2642         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2643
2644         my $status = 'unknown';
2645         if (!defined($status = $resp->{'return'}->{status})) {
2646             warn "unable to get VM status\n";
2647             return;
2648         }
2649
2650         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2651     };
2652
2653     foreach my $vmid (keys %$list) {
2654         next if $opt_vmid && ($vmid ne $opt_vmid);
2655         next if !$res->{$vmid}->{pid}; # not running
2656         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2657     }
2658
2659     $qmpclient->queue_execute(undef, 1);
2660
2661     foreach my $vmid (keys %$list) {
2662         next if $opt_vmid && ($vmid ne $opt_vmid);
2663         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2664     }
2665
2666     return $res;
2667 }
2668
2669 sub foreach_dimm {
2670     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2671
2672     my $dimm_id = 0;
2673     my $current_size = 1024;
2674     my $dimm_size = 512;
2675     return if $current_size == $memory;
2676
2677     for (my $j = 0; $j < 8; $j++) {
2678         for (my $i = 0; $i < 32; $i++) {
2679             my $name = "dimm${dimm_id}";
2680             $dimm_id++;
2681             my $numanode = $i % $sockets;
2682             $current_size += $dimm_size;
2683             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2684             return  $current_size if $current_size >= $memory;
2685         }
2686         $dimm_size *= 2;
2687     }
2688 }
2689
2690 sub foreach_reverse_dimm {
2691     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2692
2693     my $dimm_id = 253;
2694     my $current_size = 4177920;
2695     my $dimm_size = 65536;
2696     return if $current_size == $memory;
2697
2698     for (my $j = 0; $j < 8; $j++) {
2699         for (my $i = 0; $i < 32; $i++) {
2700             my $name = "dimm${dimm_id}";
2701             $dimm_id--;
2702             my $numanode = $i % $sockets;
2703             $current_size -= $dimm_size;
2704             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2705             return  $current_size if $current_size <= $memory;
2706         }
2707         $dimm_size /= 2;
2708     }
2709 }
2710
2711 sub foreach_drive {
2712     my ($conf, $func) = @_;
2713
2714     foreach my $ds (keys %$conf) {
2715         next if !valid_drivename($ds);
2716
2717         my $drive = parse_drive($ds, $conf->{$ds});
2718         next if !$drive;
2719
2720         &$func($ds, $drive);
2721     }
2722 }
2723
2724 sub foreach_volid {
2725     my ($conf, $func) = @_;
2726
2727     my $volhash = {};
2728
2729     my $test_volid = sub {
2730         my ($volid, $is_cdrom) = @_;
2731
2732         return if !$volid;
2733
2734         $volhash->{$volid} = $is_cdrom || 0;
2735     };
2736
2737     foreach_drive($conf, sub {
2738         my ($ds, $drive) = @_;
2739         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2740     });
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         my $snap = $conf->{snapshots}->{$snapname};
2744         &$test_volid($snap->{vmstate}, 0);
2745         foreach_drive($snap, sub {
2746             my ($ds, $drive) = @_;
2747             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2748         });
2749     }
2750
2751     foreach my $volid (keys %$volhash) {
2752         &$func($volid, $volhash->{$volid});
2753     }
2754 }
2755
2756 sub vga_conf_has_spice {
2757     my ($vga) = @_;
2758
2759     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2760
2761     return $1 || 1;
2762 }
2763
2764 sub config_to_command {
2765     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2766
2767     my $cmd = [];
2768     my $globalFlags = [];
2769     my $machineFlags = [];
2770     my $rtcFlags = [];
2771     my $cpuFlags = [];
2772     my $devices = [];
2773     my $pciaddr = '';
2774     my $bridges = {};
2775     my $kvmver = kvm_user_version();
2776     my $vernum = 0; # unknown
2777     my $ostype = $conf->{ostype};
2778     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2779         $vernum = $1*1000000+$2*1000;
2780     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2781         $vernum = $1*1000000+$2*1000+$3;
2782     }
2783
2784     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2785
2786     my $have_ovz = -f '/proc/vz/vestat';
2787
2788     my $q35 = machine_type_is_q35($conf);
2789     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2790     my $machine_type = $forcemachine || $conf->{machine};
2791     my $use_old_bios_files = undef;
2792     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2793
2794     my $cpuunits = defined($conf->{cpuunits}) ?
2795             $conf->{cpuunits} : $defaults->{cpuunits};
2796
2797     push @$cmd, '/usr/bin/systemd-run';
2798     push @$cmd, '--scope';
2799     push @$cmd, '--slice', "qemu";
2800     push @$cmd, '--unit', $vmid;
2801     # set KillMode=none, so that systemd don't kill those scopes
2802     # at shutdown (pve-manager service should stop the VMs instead)
2803     push @$cmd, '-p', "KillMode=none";
2804     push @$cmd, '-p', "CPUShares=$cpuunits";
2805     if ($conf->{cpulimit}) {
2806         my $cpulimit = int($conf->{cpulimit} * 100);
2807         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2808     }
2809
2810     push @$cmd, '/usr/bin/kvm';
2811
2812     push @$cmd, '-id', $vmid;
2813
2814     my $use_virtio = 0;
2815
2816     my $qmpsocket = qmp_socket($vmid);
2817     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2818     push @$cmd, '-mon', "chardev=qmp,mode=control";
2819
2820
2821     push @$cmd, '-pidfile' , pidfile_name($vmid);
2822
2823     push @$cmd, '-daemonize';
2824
2825     if ($conf->{smbios1}) {
2826         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2827     }
2828
2829     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2830         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2831         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2832         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2833         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2834         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2835         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2836     }
2837
2838     if ($q35) {
2839         # the q35 chipset support native usb2, so we enable usb controller
2840         # by default for this machine type
2841         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2842     } else {
2843         $pciaddr = print_pci_addr("piix3", $bridges);
2844         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2845
2846         my $use_usb2 = 0;
2847         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2848             next if !$conf->{"usb$i"};
2849             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2850             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2851             $use_usb2 = 1;
2852         }
2853         # include usb device config
2854         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2855     }
2856
2857     # add usb3 controller if needed
2858
2859     my $use_usb3 = 0;
2860     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2861         next if !$conf->{"usb$i"};
2862         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2863         next if !$d || !$d->{usb3};
2864         $use_usb3 = 1;
2865     }
2866
2867     $pciaddr = print_pci_addr("xhci", $bridges);
2868     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2869
2870     my $vga = $conf->{vga};
2871
2872     my $qxlnum = vga_conf_has_spice($vga);
2873     $vga = 'qxl' if $qxlnum;
2874
2875     if (!$vga) {
2876         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2877                                 $conf->{ostype} eq 'win7' ||
2878                                 $conf->{ostype} eq 'w2k8')) {
2879             $vga = 'std';
2880         } else {
2881             $vga = 'cirrus';
2882         }
2883     }
2884
2885     # enable absolute mouse coordinates (needed by vnc)
2886     my $tablet;
2887     if (defined($conf->{tablet})) {
2888         $tablet = $conf->{tablet};
2889     } else {
2890         $tablet = $defaults->{tablet};
2891         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2892         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2893     }
2894
2895     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2896
2897     my $kvm_off = 0;
2898     # host pci devices
2899     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2900         my $d = parse_hostpci($conf->{"hostpci$i"});
2901         next if !$d;
2902
2903         my $pcie = $d->{pcie};
2904         if($pcie){
2905             die "q35 machine model is not enabled" if !$q35;
2906             $pciaddr = print_pcie_addr("hostpci$i");
2907         }else{
2908             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2909         }
2910
2911         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2912         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2913         if ($xvga && $xvga ne '') {
2914             $kvm_off = 1;
2915             $vga = 'none';
2916             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2917                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2918             }
2919             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2920                 $xvga = "";
2921             }
2922         }
2923         my $pcidevices = $d->{pciid};
2924         my $multifunction = 1 if @$pcidevices > 1;
2925
2926         my $j=0;
2927         foreach my $pcidevice (@$pcidevices) {
2928
2929             my $id = "hostpci$i";
2930             $id .= ".$j" if $multifunction;
2931             my $addr = $pciaddr;
2932             $addr .= ".$j" if $multifunction;
2933             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2934
2935             if($j == 0){
2936                 $devicestr .= "$rombar$xvga";
2937                 $devicestr .= ",multifunction=on" if $multifunction;
2938             }
2939
2940             push @$devices, '-device', $devicestr;
2941             $j++;
2942         }
2943     }
2944
2945     # usb devices
2946     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2947         next if !$conf->{"usb$i"};
2948         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2949         next if !$d;
2950
2951         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2952         my $usbbus = '';
2953         if (defined($d->{usb3}) && $d->{usb3}) {
2954             $usbbus = ',bus=xhci.0';
2955         }
2956
2957         if (defined($d->{host})) {
2958             $d = parse_usb_device($d->{host});
2959             if (defined($d->{vendorid}) && defined($d->{productid})) {
2960                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2961             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2962                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2963             } elsif (defined($d->{spice}) && $d->{spice}) {
2964                 # usb redir support for spice, currently no usb3
2965                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2966                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2967             }
2968         }
2969     }
2970
2971     # serial devices
2972     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2973         if (my $path = $conf->{"serial$i"}) {
2974             if ($path eq 'socket') {
2975                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2976                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2977                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2978             } else {
2979                 die "no such serial device\n" if ! -c $path;
2980                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2981                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2982             }
2983         }
2984     }
2985
2986     # parallel devices
2987     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2988         if (my $path = $conf->{"parallel$i"}) {
2989             die "no such parallel device\n" if ! -c $path;
2990             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2991             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2992             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2993         }
2994     }
2995
2996     my $vmname = $conf->{name} || "vm$vmid";
2997
2998     push @$cmd, '-name', $vmname;
2999
3000     my $sockets = 1;
3001     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3002     $sockets = $conf->{sockets} if  $conf->{sockets};
3003
3004     my $cores = $conf->{cores} || 1;
3005
3006     my $maxcpus = $sockets * $cores;
3007
3008     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3009
3010     my $allowed_vcpus = $cpuinfo->{cpus};
3011
3012     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3013         if ($allowed_vcpus < $maxcpus);
3014
3015     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3016
3017     push @$cmd, '-nodefaults';
3018
3019     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3020
3021     my $bootindex_hash = {};
3022     my $i = 1;
3023     foreach my $o (split(//, $bootorder)) {
3024         $bootindex_hash->{$o} = $i*100;
3025         $i++;
3026     }
3027
3028     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3029
3030     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3031
3032     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3033
3034     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3035
3036     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3037         my $socket = vnc_socket($vmid);
3038         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3039     } else {
3040         push @$cmd, '-nographic';
3041     }
3042
3043     # time drift fix
3044     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3045
3046     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3047     my $useLocaltime = $conf->{localtime};
3048
3049     if ($ostype) {
3050         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3051
3052         if ($ostype =~ m/^w/) { # windows
3053             $useLocaltime = 1 if !defined($conf->{localtime});
3054
3055             # use time drift fix when acpi is enabled
3056             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3057                 $tdf = 1 if !defined($conf->{tdf});
3058             }
3059         }
3060
3061         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3062             $ostype eq 'wvista') {
3063             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3064             push @$cmd, '-no-hpet';
3065             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3066                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3067                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3068                 push @$cpuFlags , 'hv_time' if !$nokvm;
3069
3070             } else {
3071                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3072             }
3073         }
3074
3075         if ($ostype eq 'win7' || $ostype eq 'win8') {
3076             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3077         }
3078     }
3079
3080     push @$rtcFlags, 'driftfix=slew' if $tdf;
3081
3082     if ($nokvm) {
3083         push @$machineFlags, 'accel=tcg';
3084     } else {
3085         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3086     }
3087
3088     if ($machine_type) {
3089         push @$machineFlags, "type=${machine_type}";
3090     }
3091
3092     if ($conf->{startdate}) {
3093         push @$rtcFlags, "base=$conf->{startdate}";
3094     } elsif ($useLocaltime) {
3095         push @$rtcFlags, 'base=localtime';
3096     }
3097
3098     my $cpu = $nokvm ? "qemu64" : "kvm64";
3099     if (my $cputype = $conf->{cpu}) {
3100         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3101             or die "Cannot parse cpu description: $cputype\n";
3102         $cpu = $cpuconf->{cputype};
3103         $kvm_off = 1 if $cpuconf->{hidden};
3104     }
3105
3106     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3107
3108     push @$cpuFlags , '-x2apic'
3109         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3110
3111     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3112
3113     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3114
3115     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3116
3117         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3118         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3119     }
3120
3121     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3122
3123     push @$cpuFlags, 'kvm=off' if $kvm_off;
3124
3125     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3126
3127     push @$cmd, '-cpu', $cpu;
3128
3129     my $memory = $conf->{memory} || $defaults->{memory};
3130     my $static_memory = 0;
3131     my $dimm_memory = 0;
3132
3133     if ($hotplug_features->{memory}) {
3134         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3135         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3136         $static_memory = $STATICMEM;
3137         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3138         $dimm_memory = $memory - $static_memory;
3139         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3140
3141     } else {
3142
3143         $static_memory = $memory;
3144         push @$cmd, '-m', $static_memory;
3145     }
3146
3147     if ($conf->{numa}) {
3148
3149         my $numa_totalmemory = undef;
3150         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3151             next if !$conf->{"numa$i"};
3152             my $numa = parse_numa($conf->{"numa$i"});
3153             next if !$numa;
3154             # memory
3155             die "missing numa node$i memory value\n" if !$numa->{memory};
3156             my $numa_memory = $numa->{memory};
3157             $numa_totalmemory += $numa_memory;
3158             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3159
3160             # cpus
3161             my $cpus_start = $numa->{cpus}->{start};
3162             die "missing numa node$i cpus\n" if !defined($cpus_start);
3163             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3164             my $cpus = $cpus_start;
3165             if (defined($cpus_end)) {
3166                 $cpus .= "-$cpus_end";
3167                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3168             }
3169
3170             # hostnodes
3171             my $hostnodes_start = $numa->{hostnodes}->{start};
3172             if (defined($hostnodes_start)) {
3173                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3174                 my $hostnodes = $hostnodes_start;
3175                 if (defined($hostnodes_end)) {
3176                     $hostnodes .= "-$hostnodes_end";
3177                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3178                 }
3179
3180                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3181                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3182                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3183                 }
3184
3185                 # policy
3186                 my $policy = $numa->{policy};
3187                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3188                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3189             }
3190
3191             push @$cmd, '-object', $numa_object;
3192             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3193         }
3194
3195         die "total memory for NUMA nodes must be equal to vm static memory\n"
3196             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3197
3198         #if no custom tology, we split memory and cores across numa nodes
3199         if(!$numa_totalmemory) {
3200
3201             my $numa_memory = ($static_memory / $sockets) . "M";
3202
3203             for (my $i = 0; $i < $sockets; $i++)  {
3204
3205                 my $cpustart = ($cores * $i);
3206                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3207                 my $cpus = $cpustart;
3208                 $cpus .= "-$cpuend" if $cpuend;
3209
3210                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3211                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3212             }
3213         }
3214     }
3215
3216     if ($hotplug_features->{memory}) {
3217         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3218             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3219             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3220             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3221
3222             #if dimm_memory is not aligned to dimm map
3223             if($current_size > $memory) {
3224                  $conf->{memory} = $current_size;
3225                  write_config($vmid, $conf);
3226             }
3227         });
3228     }
3229
3230     push @$cmd, '-S' if $conf->{freeze};
3231
3232     # set keyboard layout
3233     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3234     push @$cmd, '-k', $kb if $kb;
3235
3236     # enable sound
3237     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3238     #push @$cmd, '-soundhw', 'es1370';
3239     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3240
3241     if($conf->{agent}) {
3242         my $qgasocket = qmp_socket($vmid, 1);
3243         my $pciaddr = print_pci_addr("qga0", $bridges);
3244         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3245         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3246         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3247     }
3248
3249     my $spice_port;
3250
3251     if ($qxlnum) {
3252         if ($qxlnum > 1) {
3253             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3254                 for(my $i = 1; $i < $qxlnum; $i++){
3255                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3256                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3257                 }
3258             } else {
3259                 # assume other OS works like Linux
3260                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3261                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3262             }
3263         }
3264
3265         my $pciaddr = print_pci_addr("spice", $bridges);
3266
3267         my $nodename = PVE::INotify::nodename();
3268         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3269         $spice_port = PVE::Tools::next_spice_port($pfamily);
3270
3271         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3272
3273         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3274         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3275         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3276     }
3277
3278     # enable balloon by default, unless explicitly disabled
3279     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3280         $pciaddr = print_pci_addr("balloon0", $bridges);
3281         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3282     }
3283
3284     if ($conf->{watchdog}) {
3285         my $wdopts = parse_watchdog($conf->{watchdog});
3286         $pciaddr = print_pci_addr("watchdog", $bridges);
3287         my $watchdog = $wdopts->{model} || 'i6300esb';
3288         push @$devices, '-device', "$watchdog$pciaddr";
3289         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3290     }
3291
3292     my $vollist = [];
3293     my $scsicontroller = {};
3294     my $ahcicontroller = {};
3295     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3296
3297     # Add iscsi initiator name if available
3298     if (my $initiator = get_initiator_name()) {
3299         push @$devices, '-iscsi', "initiator-name=$initiator";
3300     }
3301
3302     foreach_drive($conf, sub {
3303         my ($ds, $drive) = @_;
3304
3305         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3306             push @$vollist, $drive->{file};
3307         }
3308
3309         $use_virtio = 1 if $ds =~ m/^virtio/;
3310
3311         if (drive_is_cdrom ($drive)) {
3312             if ($bootindex_hash->{d}) {
3313                 $drive->{bootindex} = $bootindex_hash->{d};
3314                 $bootindex_hash->{d} += 1;
3315             }
3316         } else {
3317             if ($bootindex_hash->{c}) {
3318                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3319                 $bootindex_hash->{c} += 1;
3320             }
3321         }
3322
3323         if($drive->{interface} eq 'virtio'){
3324            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3325         }
3326
3327         if ($drive->{interface} eq 'scsi') {
3328
3329             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3330
3331             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3332             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3333
3334             my $iothread = '';
3335             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3336                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3337                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3338             }
3339
3340             my $queues = '';
3341             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3342                 $queues = ",num_queues=$drive->{queues}";
3343             }
3344
3345             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3346             $scsicontroller->{$controller}=1;
3347         }
3348
3349         if ($drive->{interface} eq 'sata') {
3350            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3351            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3352            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3353            $ahcicontroller->{$controller}=1;
3354         }
3355
3356         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3357         push @$devices, '-drive',$drive_cmd;
3358         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3359     });
3360
3361     for (my $i = 0; $i < $MAX_NETS; $i++) {
3362          next if !$conf->{"net$i"};
3363          my $d = parse_net($conf->{"net$i"});
3364          next if !$d;
3365
3366          $use_virtio = 1 if $d->{model} eq 'virtio';
3367
3368          if ($bootindex_hash->{n}) {
3369             $d->{bootindex} = $bootindex_hash->{n};
3370             $bootindex_hash->{n} += 1;
3371          }
3372
3373          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3374          push @$devices, '-netdev', $netdevfull;
3375
3376          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3377          push @$devices, '-device', $netdevicefull;
3378     }
3379
3380     if (!$q35) {
3381         # add pci bridges
3382         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3383            $bridges->{1} = 1;
3384            $bridges->{2} = 1;
3385         }
3386
3387         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3388
3389         while (my ($k, $v) = each %$bridges) {
3390             $pciaddr = print_pci_addr("pci.$k");
3391             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3392         }
3393     }
3394
3395     # add custom args
3396     if ($conf->{args}) {
3397         my $aa = PVE::Tools::split_args($conf->{args});
3398         push @$cmd, @$aa;
3399     }
3400
3401     push @$cmd, @$devices;
3402     push @$cmd, '-rtc', join(',', @$rtcFlags)
3403         if scalar(@$rtcFlags);
3404     push @$cmd, '-machine', join(',', @$machineFlags)
3405         if scalar(@$machineFlags);
3406     push @$cmd, '-global', join(',', @$globalFlags)
3407         if scalar(@$globalFlags);
3408
3409     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3410 }
3411
3412 sub vnc_socket {
3413     my ($vmid) = @_;
3414     return "${var_run_tmpdir}/$vmid.vnc";
3415 }
3416
3417 sub spice_port {
3418     my ($vmid) = @_;
3419
3420     my $res = vm_mon_cmd($vmid, 'query-spice');
3421
3422     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3423 }
3424
3425 sub qmp_socket {
3426     my ($vmid, $qga) = @_;
3427     my $sockettype = $qga ? 'qga' : 'qmp';
3428     return "${var_run_tmpdir}/$vmid.$sockettype";
3429 }
3430
3431 sub pidfile_name {
3432     my ($vmid) = @_;
3433     return "${var_run_tmpdir}/$vmid.pid";
3434 }
3435
3436 sub vm_devices_list {
3437     my ($vmid) = @_;
3438
3439     my $res = vm_mon_cmd($vmid, 'query-pci');
3440     my $devices = {};
3441     foreach my $pcibus (@$res) {
3442         foreach my $device (@{$pcibus->{devices}}) {
3443             next if !$device->{'qdev_id'};
3444             if ($device->{'pci_bridge'}) {
3445                 $devices->{$device->{'qdev_id'}} = 1;
3446                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3447                     next if !$bridge_device->{'qdev_id'};
3448                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3449                     $devices->{$device->{'qdev_id'}}++;
3450                 }
3451             } else {
3452                 $devices->{$device->{'qdev_id'}} = 1;
3453             }
3454         }
3455     }
3456
3457     my $resblock = vm_mon_cmd($vmid, 'query-block');
3458     foreach my $block (@$resblock) {
3459         if($block->{device} =~ m/^drive-(\S+)/){
3460                 $devices->{$1} = 1;
3461         }
3462     }
3463
3464     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3465     foreach my $mice (@$resmice) {
3466         if ($mice->{name} eq 'QEMU HID Tablet') {
3467             $devices->{tablet} = 1;
3468             last;
3469         }
3470     }
3471
3472     return $devices;
3473 }
3474
3475 sub vm_deviceplug {
3476     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3477
3478     my $q35 = machine_type_is_q35($conf);
3479
3480     my $devices_list = vm_devices_list($vmid);
3481     return 1 if defined($devices_list->{$deviceid});
3482
3483     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3484
3485     if ($deviceid eq 'tablet') {
3486
3487         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3488
3489     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3490
3491         qemu_iothread_add($vmid, $deviceid, $device);
3492
3493         qemu_driveadd($storecfg, $vmid, $device);
3494         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3495
3496         qemu_deviceadd($vmid, $devicefull);
3497         eval { qemu_deviceaddverify($vmid, $deviceid); };
3498         if (my $err = $@) {
3499             eval { qemu_drivedel($vmid, $deviceid); };
3500             warn $@ if $@;
3501             die $err;
3502         }
3503
3504     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3505
3506
3507         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3508         my $pciaddr = print_pci_addr($deviceid);
3509         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3510
3511         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3512
3513         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3514             qemu_iothread_add($vmid, $deviceid, $device);
3515             $devicefull .= ",iothread=iothread-$deviceid";
3516         }
3517
3518         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3519             $devicefull .= ",num_queues=$device->{queues}";
3520         }
3521
3522         qemu_deviceadd($vmid, $devicefull);
3523         qemu_deviceaddverify($vmid, $deviceid);
3524
3525     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3526
3527         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3528         qemu_driveadd($storecfg, $vmid, $device);
3529
3530         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3531         eval { qemu_deviceadd($vmid, $devicefull); };
3532         if (my $err = $@) {
3533             eval { qemu_drivedel($vmid, $deviceid); };
3534             warn $@ if $@;
3535             die $err;
3536         }
3537
3538     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3539
3540         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3541
3542         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3543         my $use_old_bios_files = undef;
3544         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3545
3546         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3547         qemu_deviceadd($vmid, $netdevicefull);
3548         eval { qemu_deviceaddverify($vmid, $deviceid); };
3549         if (my $err = $@) {
3550             eval { qemu_netdevdel($vmid, $deviceid); };
3551             warn $@ if $@;
3552             die $err;
3553         }
3554
3555     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3556
3557         my $bridgeid = $2;
3558         my $pciaddr = print_pci_addr($deviceid);
3559         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3560
3561         qemu_deviceadd($vmid, $devicefull);
3562         qemu_deviceaddverify($vmid, $deviceid);
3563
3564     } else {
3565         die "can't hotplug device '$deviceid'\n";
3566     }
3567
3568     return 1;
3569 }
3570
3571 # fixme: this should raise exceptions on error!
3572 sub vm_deviceunplug {
3573     my ($vmid, $conf, $deviceid) = @_;
3574
3575     my $devices_list = vm_devices_list($vmid);
3576     return 1 if !defined($devices_list->{$deviceid});
3577
3578     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3579
3580     if ($deviceid eq 'tablet') {
3581
3582         qemu_devicedel($vmid, $deviceid);
3583
3584     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3585
3586         qemu_devicedel($vmid, $deviceid);
3587         qemu_devicedelverify($vmid, $deviceid);
3588         qemu_drivedel($vmid, $deviceid);
3589         qemu_iothread_del($conf, $vmid, $deviceid);
3590
3591     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3592
3593         qemu_devicedel($vmid, $deviceid);
3594         qemu_devicedelverify($vmid, $deviceid);
3595         qemu_iothread_del($conf, $vmid, $deviceid);
3596
3597     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3598
3599         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3600         my $device = parse_drive($deviceid, $conf->{$deviceid});
3601         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3602
3603         qemu_devicedel($vmid, $deviceid);
3604         qemu_drivedel($vmid, $deviceid);
3605         qemu_deletescsihw($conf, $vmid, $deviceid);
3606
3607     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3608
3609         qemu_devicedel($vmid, $deviceid);
3610         qemu_devicedelverify($vmid, $deviceid);
3611         qemu_netdevdel($vmid, $deviceid);
3612
3613     } else {
3614         die "can't unplug device '$deviceid'\n";
3615     }
3616
3617     return 1;
3618 }
3619
3620 sub qemu_deviceadd {
3621     my ($vmid, $devicefull) = @_;
3622
3623     $devicefull = "driver=".$devicefull;
3624     my %options =  split(/[=,]/, $devicefull);
3625
3626     vm_mon_cmd($vmid, "device_add" , %options);
3627 }
3628
3629 sub qemu_devicedel {
3630     my ($vmid, $deviceid) = @_;
3631
3632     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3633 }
3634
3635 sub qemu_iothread_add {
3636     my($vmid, $deviceid, $device) = @_;
3637
3638     if ($device->{iothread}) {
3639         my $iothreads = vm_iothreads_list($vmid);
3640         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3641     }
3642 }
3643
3644 sub qemu_iothread_del {
3645     my($conf, $vmid, $deviceid) = @_;
3646
3647     my $device = parse_drive($deviceid, $conf->{$deviceid});
3648     if ($device->{iothread}) {
3649         my $iothreads = vm_iothreads_list($vmid);
3650         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3651     }
3652 }
3653
3654 sub qemu_objectadd {
3655     my($vmid, $objectid, $qomtype) = @_;
3656
3657     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3658
3659     return 1;
3660 }
3661
3662 sub qemu_objectdel {
3663     my($vmid, $objectid) = @_;
3664
3665     vm_mon_cmd($vmid, "object-del", id => $objectid);
3666
3667     return 1;
3668 }
3669
3670 sub qemu_driveadd {
3671     my ($storecfg, $vmid, $device) = @_;
3672
3673     my $drive = print_drive_full($storecfg, $vmid, $device);
3674     $drive =~ s/\\/\\\\/g;
3675     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3676
3677     # If the command succeeds qemu prints: "OK"
3678     return 1 if $ret =~ m/OK/s;
3679
3680     die "adding drive failed: $ret\n";
3681 }
3682
3683 sub qemu_drivedel {
3684     my($vmid, $deviceid) = @_;
3685
3686     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3687     $ret =~ s/^\s+//;
3688
3689     return 1 if $ret eq "";
3690
3691     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3692     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3693
3694     die "deleting drive $deviceid failed : $ret\n";
3695 }
3696
3697 sub qemu_deviceaddverify {
3698     my ($vmid, $deviceid) = @_;
3699
3700     for (my $i = 0; $i <= 5; $i++) {
3701          my $devices_list = vm_devices_list($vmid);
3702          return 1 if defined($devices_list->{$deviceid});
3703          sleep 1;
3704     }
3705
3706     die "error on hotplug device '$deviceid'\n";
3707 }
3708
3709
3710 sub qemu_devicedelverify {
3711     my ($vmid, $deviceid) = @_;
3712
3713     # need to verify that the device is correctly removed as device_del
3714     # is async and empty return is not reliable
3715
3716     for (my $i = 0; $i <= 5; $i++) {
3717          my $devices_list = vm_devices_list($vmid);
3718          return 1 if !defined($devices_list->{$deviceid});
3719          sleep 1;
3720     }
3721
3722     die "error on hot-unplugging device '$deviceid'\n";
3723 }
3724
3725 sub qemu_findorcreatescsihw {
3726     my ($storecfg, $conf, $vmid, $device) = @_;
3727
3728     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3729
3730     my $scsihwid="$controller_prefix$controller";
3731     my $devices_list = vm_devices_list($vmid);
3732
3733     if(!defined($devices_list->{$scsihwid})) {
3734         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3735     }
3736
3737     return 1;
3738 }
3739
3740 sub qemu_deletescsihw {
3741     my ($conf, $vmid, $opt) = @_;
3742
3743     my $device = parse_drive($opt, $conf->{$opt});
3744
3745     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3746         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3747         return 1;
3748     }
3749
3750     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3751
3752     my $devices_list = vm_devices_list($vmid);
3753     foreach my $opt (keys %{$devices_list}) {
3754         if (PVE::QemuServer::valid_drivename($opt)) {
3755             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3756             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3757                 return 1;
3758             }
3759         }
3760     }
3761
3762     my $scsihwid="scsihw$controller";
3763
3764     vm_deviceunplug($vmid, $conf, $scsihwid);
3765
3766     return 1;
3767 }
3768
3769 sub qemu_add_pci_bridge {
3770     my ($storecfg, $conf, $vmid, $device) = @_;
3771
3772     my $bridges = {};
3773
3774     my $bridgeid;
3775
3776     print_pci_addr($device, $bridges);
3777
3778     while (my ($k, $v) = each %$bridges) {
3779         $bridgeid = $k;
3780     }
3781     return 1 if !defined($bridgeid) || $bridgeid < 1;
3782
3783     my $bridge = "pci.$bridgeid";
3784     my $devices_list = vm_devices_list($vmid);
3785
3786     if (!defined($devices_list->{$bridge})) {
3787         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3788     }
3789
3790     return 1;
3791 }
3792
3793 sub qemu_set_link_status {
3794     my ($vmid, $device, $up) = @_;
3795
3796     vm_mon_cmd($vmid, "set_link", name => $device,
3797                up => $up ? JSON::true : JSON::false);
3798 }
3799
3800 sub qemu_netdevadd {
3801     my ($vmid, $conf, $device, $deviceid) = @_;
3802
3803     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3804     my %options =  split(/[=,]/, $netdev);
3805
3806     vm_mon_cmd($vmid, "netdev_add",  %options);
3807     return 1;
3808 }
3809
3810 sub qemu_netdevdel {
3811     my ($vmid, $deviceid) = @_;
3812
3813     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3814 }
3815
3816 sub qemu_cpu_hotplug {
3817     my ($vmid, $conf, $vcpus) = @_;
3818
3819     my $sockets = 1;
3820     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3821     $sockets = $conf->{sockets} if  $conf->{sockets};
3822     my $cores = $conf->{cores} || 1;
3823     my $maxcpus = $sockets * $cores;
3824
3825     $vcpus = $maxcpus if !$vcpus;
3826
3827     die "you can't add more vcpus than maxcpus\n"
3828         if $vcpus > $maxcpus;
3829
3830     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3831     die "online cpu unplug is not yet possible\n"
3832         if $vcpus < $currentvcpus;
3833
3834     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3835     die "vcpus in running vm is different than configuration\n"
3836         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3837
3838     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3839         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3840     }
3841 }
3842
3843 sub qemu_memory_hotplug {
3844     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3845
3846     return $value if !check_running($vmid);
3847
3848     my $memory = $conf->{memory} || $defaults->{memory};
3849     $value = $defaults->{memory} if !$value;
3850     return $value if $value == $memory;
3851
3852     my $static_memory = $STATICMEM;
3853     my $dimm_memory = $memory - $static_memory;
3854
3855     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3856     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3857
3858
3859     my $sockets = 1;
3860     $sockets = $conf->{sockets} if $conf->{sockets};
3861
3862     if($value > $memory) {
3863
3864         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3865             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3866
3867                 return if $current_size <= $conf->{memory};
3868
3869                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3870                 if (my $err = $@) {
3871                     eval { qemu_objectdel($vmid, "mem-$name"); };
3872                     die $err;
3873                 }
3874
3875                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3876                 if (my $err = $@) {
3877                     eval { qemu_objectdel($vmid, "mem-$name"); };
3878                     die $err;
3879                 }
3880                 #update conf after each succesful module hotplug
3881                 $conf->{memory} = $current_size;
3882                 write_config($vmid, $conf);
3883         });
3884
3885     } else {
3886
3887         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3888             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3889
3890                 return if $current_size >= $conf->{memory};
3891                 print "try to unplug memory dimm $name\n";
3892
3893                 my $retry = 0;
3894                 while (1) {
3895                     eval { qemu_devicedel($vmid, $name) };
3896                     sleep 3;
3897                     my $dimm_list = qemu_dimm_list($vmid);
3898                     last if !$dimm_list->{$name};
3899                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3900                     $retry++;
3901                 }
3902
3903                 #update conf after each succesful module unplug
3904                 $conf->{memory} = $current_size;
3905
3906                 eval { qemu_objectdel($vmid, "mem-$name"); };
3907                 write_config($vmid, $conf);
3908         });
3909     }
3910 }
3911
3912 sub qemu_dimm_list {
3913     my ($vmid) = @_;
3914
3915     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3916     my $dimms = {};
3917
3918     foreach my $dimm (@$dimmarray) {
3919
3920         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3921         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3922         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3923         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3924         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3925     }
3926     return $dimms;
3927 }
3928
3929 sub qemu_block_set_io_throttle {
3930     my ($vmid, $deviceid,
3931         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3932         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3933
3934     return if !check_running($vmid) ;
3935
3936     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3937         bps => int($bps),
3938         bps_rd => int($bps_rd),
3939         bps_wr => int($bps_wr),
3940         iops => int($iops),
3941         iops_rd => int($iops_rd),
3942         iops_wr => int($iops_wr),
3943         bps_max => int($bps_max),
3944         bps_rd_max => int($bps_rd_max),
3945         bps_wr_max => int($bps_wr_max),
3946         iops_max => int($iops_max),
3947         iops_rd_max => int($iops_rd_max),
3948         iops_wr_max => int($iops_wr_max)
3949     );
3950
3951 }
3952
3953 # old code, only used to shutdown old VM after update
3954 sub __read_avail {
3955     my ($fh, $timeout) = @_;
3956
3957     my $sel = new IO::Select;
3958     $sel->add($fh);
3959
3960     my $res = '';
3961     my $buf;
3962
3963     my @ready;
3964     while (scalar (@ready = $sel->can_read($timeout))) {
3965         my $count;
3966         if ($count = $fh->sysread($buf, 8192)) {
3967             if ($buf =~ /^(.*)\(qemu\) $/s) {
3968                 $res .= $1;
3969                 last;
3970             } else {
3971                 $res .= $buf;
3972             }
3973         } else {
3974             if (!defined($count)) {
3975                 die "$!\n";
3976             }
3977             last;
3978         }
3979     }
3980
3981     die "monitor read timeout\n" if !scalar(@ready);
3982
3983     return $res;
3984 }
3985
3986 # old code, only used to shutdown old VM after update
3987 sub vm_monitor_command {
3988     my ($vmid, $cmdstr, $nocheck) = @_;
3989
3990     my $res;
3991
3992     eval {
3993         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3994
3995         my $sname = "${var_run_tmpdir}/$vmid.mon";
3996
3997         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3998             die "unable to connect to VM $vmid socket - $!\n";
3999
4000         my $timeout = 3;
4001
4002         # hack: migrate sometime blocks the monitor (when migrate_downtime
4003         # is set)
4004         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4005             $timeout = 60*60; # 1 hour
4006         }
4007
4008         # read banner;
4009         my $data = __read_avail($sock, $timeout);
4010
4011         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4012             die "got unexpected qemu monitor banner\n";
4013         }
4014
4015         my $sel = new IO::Select;
4016         $sel->add($sock);
4017
4018         if (!scalar(my @ready = $sel->can_write($timeout))) {
4019             die "monitor write error - timeout";
4020         }
4021
4022         my $fullcmd = "$cmdstr\r";
4023
4024         # syslog('info', "VM $vmid monitor command: $cmdstr");
4025
4026         my $b;
4027         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4028             die "monitor write error - $!";
4029         }
4030
4031         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4032
4033         $timeout = 20;
4034
4035         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4036             $timeout = 60*60; # 1 hour
4037         } elsif ($cmdstr =~ m/^(eject|change)/) {
4038             $timeout = 60; # note: cdrom mount command is slow
4039         }
4040         if ($res = __read_avail($sock, $timeout)) {
4041
4042             my @lines = split("\r?\n", $res);
4043
4044             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4045
4046             $res = join("\n", @lines);
4047             $res .= "\n";
4048         }
4049     };
4050
4051     my $err = $@;
4052
4053     if ($err) {
4054         syslog("err", "VM $vmid monitor command failed - $err");
4055         die $err;
4056     }
4057
4058     return $res;
4059 }
4060
4061 sub qemu_block_resize {
4062     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4063
4064     my $running = check_running($vmid);
4065
4066     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4067
4068     return if !$running;
4069
4070     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4071
4072 }
4073
4074 sub qemu_volume_snapshot {
4075     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4076
4077     my $running = check_running($vmid);
4078
4079     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4080         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4081     } else {
4082         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4083     }
4084 }
4085
4086 sub qemu_volume_snapshot_delete {
4087     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4088
4089     my $running = check_running($vmid);
4090
4091     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4092
4093     return if !$running;
4094
4095     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4096 }
4097
4098 sub set_migration_caps {
4099     my ($vmid) = @_;
4100
4101     my $cap_ref = [];
4102
4103     my $enabled_cap = {
4104         "auto-converge" => 1,
4105         "xbzrle" => 1,
4106         "x-rdma-pin-all" => 0,
4107         "zero-blocks" => 0,
4108         "compress" => 0
4109     };
4110
4111     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4112
4113     for my $supported_capability (@$supported_capabilities) {
4114         push @$cap_ref, {
4115             capability => $supported_capability->{capability},
4116             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4117         };
4118     }
4119
4120     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4121 }
4122
4123 my $fast_plug_option = {
4124     'lock' => 1,
4125     'name' => 1,
4126     'onboot' => 1,
4127     'shares' => 1,
4128     'startup' => 1,
4129     'description' => 1,
4130 };
4131
4132 # hotplug changes in [PENDING]
4133 # $selection hash can be used to only apply specified options, for
4134 # example: { cores => 1 } (only apply changed 'cores')
4135 # $errors ref is used to return error messages
4136 sub vmconfig_hotplug_pending {
4137     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4138
4139     my $defaults = load_defaults();
4140
4141     # commit values which do not have any impact on running VM first
4142     # Note: those option cannot raise errors, we we do not care about
4143     # $selection and always apply them.
4144
4145     my $add_error = sub {
4146         my ($opt, $msg) = @_;
4147         $errors->{$opt} = "hotplug problem - $msg";
4148     };
4149
4150     my $changes = 0;
4151     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4152         if ($fast_plug_option->{$opt}) {
4153             $conf->{$opt} = $conf->{pending}->{$opt};
4154             delete $conf->{pending}->{$opt};
4155             $changes = 1;
4156         }
4157     }
4158
4159     if ($changes) {
4160         write_config($vmid, $conf);
4161         $conf = load_config($vmid); # update/reload
4162     }
4163
4164     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4165
4166     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4167     while (my ($opt, $force) = each %$pending_delete_hash) {
4168         next if $selection && !$selection->{$opt};
4169         eval {
4170             if ($opt eq 'hotplug') {
4171                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4172             } elsif ($opt eq 'tablet') {
4173                 die "skip\n" if !$hotplug_features->{usb};
4174                 if ($defaults->{tablet}) {
4175                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4176                 } else {
4177                     vm_deviceunplug($vmid, $conf, $opt);
4178                 }
4179             } elsif ($opt eq 'vcpus') {
4180                 die "skip\n" if !$hotplug_features->{cpu};
4181                 qemu_cpu_hotplug($vmid, $conf, undef);
4182             } elsif ($opt eq 'balloon') {
4183                 # enable balloon device is not hotpluggable
4184                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4185             } elsif ($fast_plug_option->{$opt}) {
4186                 # do nothing
4187             } elsif ($opt =~ m/^net(\d+)$/) {
4188                 die "skip\n" if !$hotplug_features->{network};
4189                 vm_deviceunplug($vmid, $conf, $opt);
4190             } elsif (valid_drivename($opt)) {
4191                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4192                 vm_deviceunplug($vmid, $conf, $opt);
4193                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4194             } elsif ($opt =~ m/^memory$/) {
4195                 die "skip\n" if !$hotplug_features->{memory};
4196                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4197             } elsif ($opt eq 'cpuunits') {
4198                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4199             } elsif ($opt eq 'cpulimit') {
4200                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4201             } else {
4202                 die "skip\n";
4203             }
4204         };
4205         if (my $err = $@) {
4206             &$add_error($opt, $err) if $err ne "skip\n";
4207         } else {
4208             # save new config if hotplug was successful
4209             delete $conf->{$opt};
4210             vmconfig_undelete_pending_option($conf, $opt);
4211             write_config($vmid, $conf);
4212             $conf = load_config($vmid); # update/reload
4213         }
4214     }
4215
4216     foreach my $opt (keys %{$conf->{pending}}) {
4217         next if $selection && !$selection->{$opt};
4218         my $value = $conf->{pending}->{$opt};
4219         eval {
4220             if ($opt eq 'hotplug') {
4221                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4222             } elsif ($opt eq 'tablet') {
4223                 die "skip\n" if !$hotplug_features->{usb};
4224                 if ($value == 1) {
4225                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4226                 } elsif ($value == 0) {
4227                     vm_deviceunplug($vmid, $conf, $opt);
4228                 }
4229             } elsif ($opt eq 'vcpus') {
4230                 die "skip\n" if !$hotplug_features->{cpu};
4231                 qemu_cpu_hotplug($vmid, $conf, $value);
4232             } elsif ($opt eq 'balloon') {
4233                 # enable/disable balloning device is not hotpluggable
4234                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4235                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4236                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4237
4238                 # allow manual ballooning if shares is set to zero
4239                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4240                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4241                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4242                 }
4243             } elsif ($opt =~ m/^net(\d+)$/) {
4244                 # some changes can be done without hotplug
4245                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4246                                     $vmid, $opt, $value);
4247             } elsif (valid_drivename($opt)) {
4248                 # some changes can be done without hotplug
4249                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4250                                      $vmid, $opt, $value, 1);
4251             } elsif ($opt =~ m/^memory$/) { #dimms
4252                 die "skip\n" if !$hotplug_features->{memory};
4253                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4254             } elsif ($opt eq 'cpuunits') {
4255                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4256             } elsif ($opt eq 'cpulimit') {
4257                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4258                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4259             } else {
4260                 die "skip\n";  # skip non-hot-pluggable options
4261             }
4262         };
4263         if (my $err = $@) {
4264             &$add_error($opt, $err) if $err ne "skip\n";
4265         } else {
4266             # save new config if hotplug was successful
4267             $conf->{$opt} = $value;
4268             delete $conf->{pending}->{$opt};
4269             write_config($vmid, $conf);
4270             $conf = load_config($vmid); # update/reload
4271         }
4272     }
4273 }
4274
4275 sub try_deallocate_drive {
4276     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4277
4278     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4279         my $volid = $drive->{file};
4280         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4281             my $sid = PVE::Storage::parse_volume_id($volid);
4282             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4283
4284             # check if the disk is really unused
4285             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4286                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4287             PVE::Storage::vdisk_free($storecfg, $volid);
4288             return 1;
4289         } else {
4290             # If vm is not owner of this disk remove from config
4291             return 1;
4292         }
4293     }
4294
4295     return undef;
4296 }
4297
4298 sub vmconfig_delete_or_detach_drive {
4299     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4300
4301     my $drive = parse_drive($opt, $conf->{$opt});
4302
4303     my $rpcenv = PVE::RPCEnvironment::get();
4304     my $authuser = $rpcenv->get_user();
4305
4306     if ($force) {
4307         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4308         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4309     } else {
4310         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4311     }
4312 }
4313
4314 sub vmconfig_apply_pending {
4315     my ($vmid, $conf, $storecfg) = @_;
4316
4317     # cold plug
4318
4319     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4320     while (my ($opt, $force) = each %$pending_delete_hash) {
4321         die "internal error" if $opt =~ m/^unused/;
4322         $conf = load_config($vmid); # update/reload
4323         if (!defined($conf->{$opt})) {
4324             vmconfig_undelete_pending_option($conf, $opt);
4325             write_config($vmid, $conf);
4326         } elsif (valid_drivename($opt)) {
4327             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4328             vmconfig_undelete_pending_option($conf, $opt);
4329             delete $conf->{$opt};
4330             write_config($vmid, $conf);
4331         } else {
4332             vmconfig_undelete_pending_option($conf, $opt);
4333             delete $conf->{$opt};
4334             write_config($vmid, $conf);
4335         }
4336     }
4337
4338     $conf = load_config($vmid); # update/reload
4339
4340     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4341         $conf = load_config($vmid); # update/reload
4342
4343         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4344             # skip if nothing changed
4345         } elsif (valid_drivename($opt)) {
4346             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4347                 if defined($conf->{$opt});
4348             $conf->{$opt} = $conf->{pending}->{$opt};
4349         } else {
4350             $conf->{$opt} = $conf->{pending}->{$opt};
4351         }
4352
4353         delete $conf->{pending}->{$opt};
4354         write_config($vmid, $conf);
4355     }
4356 }
4357
4358 my $safe_num_ne = sub {
4359     my ($a, $b) = @_;
4360
4361     return 0 if !defined($a) && !defined($b);
4362     return 1 if !defined($a);
4363     return 1 if !defined($b);
4364
4365     return $a != $b;
4366 };
4367
4368 my $safe_string_ne = sub {
4369     my ($a, $b) = @_;
4370
4371     return 0 if !defined($a) && !defined($b);
4372     return 1 if !defined($a);
4373     return 1 if !defined($b);
4374
4375     return $a ne $b;
4376 };
4377
4378 sub vmconfig_update_net {
4379     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4380
4381     my $newnet = parse_net($value);
4382
4383     if ($conf->{$opt}) {
4384         my $oldnet = parse_net($conf->{$opt});
4385
4386         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4387             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4388             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4389             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4390
4391             # for non online change, we try to hot-unplug
4392             die "skip\n" if !$hotplug;
4393             vm_deviceunplug($vmid, $conf, $opt);
4394         } else {
4395
4396             die "internal error" if $opt !~ m/net(\d+)/;
4397             my $iface = "tap${vmid}i$1";
4398
4399             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4400                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4401             }
4402
4403             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4404                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4405                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4406                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4407                 PVE::Network::tap_unplug($iface);
4408                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4409             }
4410
4411             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4412                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4413             }
4414
4415             return 1;
4416         }
4417     }
4418
4419     if ($hotplug) {
4420         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4421     } else {
4422         die "skip\n";
4423     }
4424 }
4425
4426 sub vmconfig_update_disk {
4427     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4428
4429     # fixme: do we need force?
4430
4431     my $drive = parse_drive($opt, $value);
4432
4433     if ($conf->{$opt}) {
4434
4435         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4436
4437             my $media = $drive->{media} || 'disk';
4438             my $oldmedia = $old_drive->{media} || 'disk';
4439             die "unable to change media type\n" if $media ne $oldmedia;
4440
4441             if (!drive_is_cdrom($old_drive)) {
4442
4443                 if ($drive->{file} ne $old_drive->{file}) {
4444
4445                     die "skip\n" if !$hotplug;
4446
4447                     # unplug and register as unused
4448                     vm_deviceunplug($vmid, $conf, $opt);
4449                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4450
4451                 } else {
4452                     # update existing disk
4453
4454                     # skip non hotpluggable value
4455                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4456                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4457                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4458                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4459                         die "skip\n";
4460                     }
4461
4462                     # apply throttle
4463                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4464                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4465                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4466                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4467                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4468                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4469                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4470                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4471                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4472                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4473                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4474                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4475
4476                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4477                                                    ($drive->{mbps} || 0)*1024*1024,
4478                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4479                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4480                                                    $drive->{iops} || 0,
4481                                                    $drive->{iops_rd} || 0,
4482                                                    $drive->{iops_wr} || 0,
4483                                                    ($drive->{mbps_max} || 0)*1024*1024,
4484                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4485                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4486                                                    $drive->{iops_max} || 0,
4487                                                    $drive->{iops_rd_max} || 0,
4488                                                    $drive->{iops_wr_max} || 0);
4489
4490                     }
4491
4492                     return 1;
4493                 }
4494
4495             } else { # cdrom
4496
4497                 if ($drive->{file} eq 'none') {
4498                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4499                 } else {
4500                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4501                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4502                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4503                 }
4504
4505                 return 1;
4506             }
4507         }
4508     }
4509
4510     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4511     # hotplug new disks
4512     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4513     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4514 }
4515
4516 sub vm_start {
4517     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4518         $forcemachine, $spice_ticket) = @_;
4519
4520     lock_config($vmid, sub {
4521         my $conf = load_config($vmid, $migratedfrom);
4522
4523         die "you can't start a vm if it's a template\n" if is_template($conf);
4524
4525         check_lock($conf) if !$skiplock;
4526
4527         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4528
4529         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4530             vmconfig_apply_pending($vmid, $conf, $storecfg);
4531             $conf = load_config($vmid); # update/reload
4532         }
4533
4534         my $defaults = load_defaults();
4535
4536         # set environment variable useful inside network script
4537         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4538
4539         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4540
4541         my $migrate_port = 0;
4542         my $migrate_uri;
4543         if ($statefile) {
4544             if ($statefile eq 'tcp') {
4545                 my $localip = "localhost";
4546                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4547                 my $nodename = PVE::INotify::nodename();
4548                 if ($datacenterconf->{migration_unsecure}) {
4549                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4550                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4551                 }
4552                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4553                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4554                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4555                 push @$cmd, '-incoming', $migrate_uri;
4556                 push @$cmd, '-S';
4557             } else {
4558                 push @$cmd, '-loadstate', $statefile;
4559             }
4560         } elsif ($paused) {
4561             push @$cmd, '-S';
4562         }
4563
4564         # host pci devices
4565         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4566           my $d = parse_hostpci($conf->{"hostpci$i"});
4567           next if !$d;
4568           my $pcidevices = $d->{pciid};
4569           foreach my $pcidevice (@$pcidevices) {
4570                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4571
4572                 my $info = pci_device_info("0000:$pciid");
4573                 die "IOMMU not present\n" if !check_iommu_support();
4574                 die "no pci device info for device '$pciid'\n" if !$info;
4575                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4576                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4577           }
4578         }
4579
4580         PVE::Storage::activate_volumes($storecfg, $vollist);
4581
4582         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4583                     umask => 0077); };
4584
4585         if (my $err = $@) {
4586             # deactivate volumes if start fails
4587             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4588             die "start failed: $err";
4589         }
4590
4591         print "migration listens on $migrate_uri\n" if $migrate_uri;
4592
4593         if ($statefile && $statefile ne 'tcp')  {
4594             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4595             warn $@ if $@;
4596         }
4597
4598         if ($migratedfrom) {
4599
4600             eval {
4601                 set_migration_caps($vmid);
4602             };
4603             warn $@ if $@;
4604
4605             if ($spice_port) {
4606                 print "spice listens on port $spice_port\n";
4607                 if ($spice_ticket) {
4608                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4609                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4610                 }
4611             }
4612
4613         } else {
4614
4615             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4616                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4617                     if $conf->{balloon};
4618             }
4619
4620             foreach my $opt (keys %$conf) {
4621                 next if $opt !~  m/^net\d+$/;
4622                 my $nicconf = parse_net($conf->{$opt});
4623                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4624             }
4625         }
4626
4627         vm_mon_cmd_nocheck($vmid, 'qom-set',
4628                     path => "machine/peripheral/balloon0",
4629                     property => "guest-stats-polling-interval",
4630                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4631
4632     });
4633 }
4634
4635 sub vm_mon_cmd {
4636     my ($vmid, $execute, %params) = @_;
4637
4638     my $cmd = { execute => $execute, arguments => \%params };
4639     vm_qmp_command($vmid, $cmd);
4640 }
4641
4642 sub vm_mon_cmd_nocheck {
4643     my ($vmid, $execute, %params) = @_;
4644
4645     my $cmd = { execute => $execute, arguments => \%params };
4646     vm_qmp_command($vmid, $cmd, 1);
4647 }
4648
4649 sub vm_qmp_command {
4650     my ($vmid, $cmd, $nocheck) = @_;
4651
4652     my $res;
4653
4654     my $timeout;
4655     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4656         $timeout = $cmd->{arguments}->{timeout};
4657         delete $cmd->{arguments}->{timeout};
4658     }
4659
4660     eval {
4661         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4662         my $sname = qmp_socket($vmid);
4663         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4664             my $qmpclient = PVE::QMPClient->new();
4665
4666             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4667         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4668             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4669                 if scalar(%{$cmd->{arguments}});
4670             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4671         } else {
4672             die "unable to open monitor socket\n";
4673         }
4674     };
4675     if (my $err = $@) {
4676         syslog("err", "VM $vmid qmp command failed - $err");
4677         die $err;
4678     }
4679
4680     return $res;
4681 }
4682
4683 sub vm_human_monitor_command {
4684     my ($vmid, $cmdline) = @_;
4685
4686     my $res;
4687
4688     my $cmd = {
4689         execute => 'human-monitor-command',
4690         arguments => { 'command-line' => $cmdline},
4691     };
4692
4693     return vm_qmp_command($vmid, $cmd);
4694 }
4695
4696 sub vm_commandline {
4697     my ($storecfg, $vmid) = @_;
4698
4699     my $conf = load_config($vmid);
4700
4701     my $defaults = load_defaults();
4702
4703     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4704
4705     return join(' ', @$cmd);
4706 }
4707
4708 sub vm_reset {
4709     my ($vmid, $skiplock) = @_;
4710
4711     lock_config($vmid, sub {
4712
4713         my $conf = load_config($vmid);
4714
4715         check_lock($conf) if !$skiplock;
4716
4717         vm_mon_cmd($vmid, "system_reset");
4718     });
4719 }
4720
4721 sub get_vm_volumes {
4722     my ($conf) = @_;
4723
4724     my $vollist = [];
4725     foreach_volid($conf, sub {
4726         my ($volid, $is_cdrom) = @_;
4727
4728         return if $volid =~ m|^/|;
4729
4730         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4731         return if !$sid;
4732
4733         push @$vollist, $volid;
4734     });
4735
4736     return $vollist;
4737 }
4738
4739 sub vm_stop_cleanup {
4740     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4741
4742     eval {
4743
4744         if (!$keepActive) {
4745             my $vollist = get_vm_volumes($conf);
4746             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4747         }
4748
4749         foreach my $ext (qw(mon qmp pid vnc qga)) {
4750             unlink "/var/run/qemu-server/${vmid}.$ext";
4751         }
4752
4753         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4754     };
4755     warn $@ if $@; # avoid errors - just warn
4756 }
4757
4758 # Note: use $nockeck to skip tests if VM configuration file exists.
4759 # We need that when migration VMs to other nodes (files already moved)
4760 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4761 sub vm_stop {
4762     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4763
4764     $force = 1 if !defined($force) && !$shutdown;
4765
4766     if ($migratedfrom){
4767         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4768         kill 15, $pid if $pid;
4769         my $conf = load_config($vmid, $migratedfrom);
4770         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4771         return;
4772     }
4773
4774     lock_config($vmid, sub {
4775
4776         my $pid = check_running($vmid, $nocheck);
4777         return if !$pid;
4778
4779         my $conf;
4780         if (!$nocheck) {
4781             $conf = load_config($vmid);
4782             check_lock($conf) if !$skiplock;
4783             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4784                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4785                 $timeout = $opts->{down} if $opts->{down};
4786             }
4787         }
4788
4789         $timeout = 60 if !defined($timeout);
4790
4791         eval {
4792             if ($shutdown) {
4793                 if (defined($conf) && $conf->{agent}) {
4794                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4795                 } else {
4796                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4797                 }
4798             } else {
4799                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4800             }
4801         };
4802         my $err = $@;
4803
4804         if (!$err) {
4805             my $count = 0;
4806             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4807                 $count++;
4808                 sleep 1;
4809             }
4810
4811             if ($count >= $timeout) {
4812                 if ($force) {
4813                     warn "VM still running - terminating now with SIGTERM\n";
4814                     kill 15, $pid;
4815                 } else {
4816                     die "VM quit/powerdown failed - got timeout\n";
4817                 }
4818             } else {
4819                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4820                 return;
4821             }
4822         } else {
4823             if ($force) {
4824                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4825                 kill 15, $pid;
4826             } else {
4827                 die "VM quit/powerdown failed\n";
4828             }
4829         }
4830
4831         # wait again
4832         $timeout = 10;
4833
4834         my $count = 0;
4835         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4836             $count++;
4837             sleep 1;
4838         }
4839
4840         if ($count >= $timeout) {
4841             warn "VM still running - terminating now with SIGKILL\n";
4842             kill 9, $pid;
4843             sleep 1;
4844         }
4845
4846         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4847    });
4848 }
4849
4850 sub vm_suspend {
4851     my ($vmid, $skiplock) = @_;
4852
4853     lock_config($vmid, sub {
4854
4855         my $conf = load_config($vmid);
4856
4857         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4858
4859         vm_mon_cmd($vmid, "stop");
4860     });
4861 }
4862
4863 sub vm_resume {
4864     my ($vmid, $skiplock, $nocheck) = @_;
4865
4866     lock_config($vmid, sub {
4867
4868         if (!$nocheck) {
4869
4870             my $conf = load_config($vmid);
4871
4872             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4873
4874             vm_mon_cmd($vmid, "cont");
4875
4876         } else {
4877             vm_mon_cmd_nocheck($vmid, "cont");
4878         }
4879     });
4880 }
4881
4882 sub vm_sendkey {
4883     my ($vmid, $skiplock, $key) = @_;
4884
4885     lock_config($vmid, sub {
4886
4887         my $conf = load_config($vmid);
4888
4889         # there is no qmp command, so we use the human monitor command
4890         vm_human_monitor_command($vmid, "sendkey $key");
4891     });
4892 }
4893
4894 sub vm_destroy {
4895     my ($storecfg, $vmid, $skiplock) = @_;
4896
4897     lock_config($vmid, sub {
4898
4899         my $conf = load_config($vmid);
4900
4901         if (!check_running($vmid)) {
4902             destroy_vm($storecfg, $vmid, undef, $skiplock);
4903         } else {
4904             die "VM $vmid is running - destroy failed\n";
4905         }
4906     });
4907 }
4908
4909 # pci helpers
4910
4911 sub file_write {
4912     my ($filename, $buf) = @_;
4913
4914     my $fh = IO::File->new($filename, "w");
4915     return undef if !$fh;
4916
4917     my $res = print $fh $buf;
4918
4919     $fh->close();
4920
4921     return $res;
4922 }
4923
4924 sub pci_device_info {
4925     my ($name) = @_;
4926
4927     my $res;
4928
4929     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4930     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4931
4932     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4933     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4934
4935     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4936     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4937
4938     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4939     return undef if !defined($product) || $product !~ s/^0x//;
4940
4941     $res = {
4942         name => $name,
4943         vendor => $vendor,
4944         product => $product,
4945         domain => $domain,
4946         bus => $bus,
4947         slot => $slot,
4948         func => $func,
4949         irq => $irq,
4950         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4951     };
4952
4953     return $res;
4954 }
4955
4956 sub pci_dev_reset {
4957     my ($dev) = @_;
4958
4959     my $name = $dev->{name};
4960
4961     my $fn = "$pcisysfs/devices/$name/reset";
4962
4963     return file_write($fn, "1");
4964 }
4965
4966 sub pci_dev_bind_to_vfio {
4967     my ($dev) = @_;
4968
4969     my $name = $dev->{name};
4970
4971     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4972
4973     if (!-d $vfio_basedir) {
4974         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4975     }
4976     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4977
4978     my $testdir = "$vfio_basedir/$name";
4979     return 1 if -d $testdir;
4980
4981     my $data = "$dev->{vendor} $dev->{product}";
4982     return undef if !file_write("$vfio_basedir/new_id", $data);
4983
4984     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4985     if (!file_write($fn, $name)) {
4986         return undef if -f $fn;
4987     }
4988
4989     $fn = "$vfio_basedir/bind";
4990     if (! -d $testdir) {
4991         return undef if !file_write($fn, $name);
4992     }
4993
4994     return -d $testdir;
4995 }
4996
4997 sub pci_dev_group_bind_to_vfio {
4998     my ($pciid) = @_;
4999
5000     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5001
5002     if (!-d $vfio_basedir) {
5003         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5004     }
5005     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5006
5007     # get IOMMU group devices
5008     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5009       my @devs = grep /^0000:/, readdir($D);
5010     closedir($D);
5011
5012     foreach my $pciid (@devs) {
5013         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5014
5015         # pci bridges, switches or root ports are not supported
5016         # they have a pci_bus subdirectory so skip them
5017         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5018
5019         my $info = pci_device_info($1);
5020         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5021     }
5022
5023     return 1;
5024 }
5025
5026 sub print_pci_addr {
5027     my ($id, $bridges) = @_;
5028
5029     my $res = '';
5030     my $devices = {
5031         piix3 => { bus => 0, addr => 1 },
5032         #addr2 : first videocard
5033         balloon0 => { bus => 0, addr => 3 },
5034         watchdog => { bus => 0, addr => 4 },
5035         scsihw0 => { bus => 0, addr => 5 },
5036         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5037         scsihw1 => { bus => 0, addr => 6 },
5038         ahci0 => { bus => 0, addr => 7 },
5039         qga0 => { bus => 0, addr => 8 },
5040         spice => { bus => 0, addr => 9 },
5041         virtio0 => { bus => 0, addr => 10 },
5042         virtio1 => { bus => 0, addr => 11 },
5043         virtio2 => { bus => 0, addr => 12 },
5044         virtio3 => { bus => 0, addr => 13 },
5045         virtio4 => { bus => 0, addr => 14 },
5046         virtio5 => { bus => 0, addr => 15 },
5047         hostpci0 => { bus => 0, addr => 16 },
5048         hostpci1 => { bus => 0, addr => 17 },
5049         net0 => { bus => 0, addr => 18 },
5050         net1 => { bus => 0, addr => 19 },
5051         net2 => { bus => 0, addr => 20 },
5052         net3 => { bus => 0, addr => 21 },
5053         net4 => { bus => 0, addr => 22 },
5054         net5 => { bus => 0, addr => 23 },
5055         vga1 => { bus => 0, addr => 24 },
5056         vga2 => { bus => 0, addr => 25 },
5057         vga3 => { bus => 0, addr => 26 },
5058         hostpci2 => { bus => 0, addr => 27 },
5059         hostpci3 => { bus => 0, addr => 28 },
5060         #addr29 : usb-host (pve-usb.cfg)
5061         'pci.1' => { bus => 0, addr => 30 },
5062         'pci.2' => { bus => 0, addr => 31 },
5063         'net6' => { bus => 1, addr => 1 },
5064         'net7' => { bus => 1, addr => 2 },
5065         'net8' => { bus => 1, addr => 3 },
5066         'net9' => { bus => 1, addr => 4 },
5067         'net10' => { bus => 1, addr => 5 },
5068         'net11' => { bus => 1, addr => 6 },
5069         'net12' => { bus => 1, addr => 7 },
5070         'net13' => { bus => 1, addr => 8 },
5071         'net14' => { bus => 1, addr => 9 },
5072         'net15' => { bus => 1, addr => 10 },
5073         'net16' => { bus => 1, addr => 11 },
5074         'net17' => { bus => 1, addr => 12 },
5075         'net18' => { bus => 1, addr => 13 },
5076         'net19' => { bus => 1, addr => 14 },
5077         'net20' => { bus => 1, addr => 15 },
5078         'net21' => { bus => 1, addr => 16 },
5079         'net22' => { bus => 1, addr => 17 },
5080         'net23' => { bus => 1, addr => 18 },
5081         'net24' => { bus => 1, addr => 19 },
5082         'net25' => { bus => 1, addr => 20 },
5083         'net26' => { bus => 1, addr => 21 },
5084         'net27' => { bus => 1, addr => 22 },
5085         'net28' => { bus => 1, addr => 23 },
5086         'net29' => { bus => 1, addr => 24 },
5087         'net30' => { bus => 1, addr => 25 },
5088         'net31' => { bus => 1, addr => 26 },
5089         'xhci' => { bus => 1, addr => 27 },
5090         'virtio6' => { bus => 2, addr => 1 },
5091         'virtio7' => { bus => 2, addr => 2 },
5092         'virtio8' => { bus => 2, addr => 3 },
5093         'virtio9' => { bus => 2, addr => 4 },
5094         'virtio10' => { bus => 2, addr => 5 },
5095         'virtio11' => { bus => 2, addr => 6 },
5096         'virtio12' => { bus => 2, addr => 7 },
5097         'virtio13' => { bus => 2, addr => 8 },
5098         'virtio14' => { bus => 2, addr => 9 },
5099         'virtio15' => { bus => 2, addr => 10 },
5100         'virtioscsi0' => { bus => 3, addr => 1 },
5101         'virtioscsi1' => { bus => 3, addr => 2 },
5102         'virtioscsi2' => { bus => 3, addr => 3 },
5103         'virtioscsi3' => { bus => 3, addr => 4 },
5104         'virtioscsi4' => { bus => 3, addr => 5 },
5105         'virtioscsi5' => { bus => 3, addr => 6 },
5106         'virtioscsi6' => { bus => 3, addr => 7 },
5107         'virtioscsi7' => { bus => 3, addr => 8 },
5108         'virtioscsi8' => { bus => 3, addr => 9 },
5109         'virtioscsi9' => { bus => 3, addr => 10 },
5110         'virtioscsi10' => { bus => 3, addr => 11 },
5111         'virtioscsi11' => { bus => 3, addr => 12 },
5112         'virtioscsi12' => { bus => 3, addr => 13 },
5113         'virtioscsi13' => { bus => 3, addr => 14 },
5114         'virtioscsi14' => { bus => 3, addr => 15 },
5115         'virtioscsi15' => { bus => 3, addr => 16 },
5116         'virtioscsi16' => { bus => 3, addr => 17 },
5117         'virtioscsi17' => { bus => 3, addr => 18 },
5118         'virtioscsi18' => { bus => 3, addr => 19 },
5119         'virtioscsi19' => { bus => 3, addr => 20 },
5120         'virtioscsi20' => { bus => 3, addr => 21 },
5121         'virtioscsi21' => { bus => 3, addr => 22 },
5122         'virtioscsi22' => { bus => 3, addr => 23 },
5123         'virtioscsi23' => { bus => 3, addr => 24 },
5124         'virtioscsi24' => { bus => 3, addr => 25 },
5125         'virtioscsi25' => { bus => 3, addr => 26 },
5126         'virtioscsi26' => { bus => 3, addr => 27 },
5127         'virtioscsi27' => { bus => 3, addr => 28 },
5128         'virtioscsi28' => { bus => 3, addr => 29 },
5129         'virtioscsi29' => { bus => 3, addr => 30 },
5130         'virtioscsi30' => { bus => 3, addr => 31 },
5131
5132     };
5133
5134     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5135            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5136            my $bus = $devices->{$id}->{bus};
5137            $res = ",bus=pci.$bus,addr=$addr";
5138            $bridges->{$bus} = 1 if $bridges;
5139     }
5140     return $res;
5141
5142 }
5143
5144 sub print_pcie_addr {
5145     my ($id) = @_;
5146
5147     my $res = '';
5148     my $devices = {
5149         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5150         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5151         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5152         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5153     };
5154
5155     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5156            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5157            my $bus = $devices->{$id}->{bus};
5158            $res = ",bus=$bus,addr=$addr";
5159     }
5160     return $res;
5161
5162 }
5163
5164 # vzdump restore implementaion
5165
5166 sub tar_archive_read_firstfile {
5167     my $archive = shift;
5168
5169     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5170
5171     # try to detect archive type first
5172     my $pid = open (TMP, "tar tf '$archive'|") ||
5173         die "unable to open file '$archive'\n";
5174     my $firstfile = <TMP>;
5175     kill 15, $pid;
5176     close TMP;
5177
5178     die "ERROR: archive contaions no data\n" if !$firstfile;
5179     chomp $firstfile;
5180
5181     return $firstfile;
5182 }
5183
5184 sub tar_restore_cleanup {
5185     my ($storecfg, $statfile) = @_;
5186
5187     print STDERR "starting cleanup\n";
5188
5189     if (my $fd = IO::File->new($statfile, "r")) {
5190         while (defined(my $line = <$fd>)) {
5191             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5192                 my $volid = $2;
5193                 eval {
5194                     if ($volid =~ m|^/|) {
5195                         unlink $volid || die 'unlink failed\n';
5196                     } else {
5197                         PVE::Storage::vdisk_free($storecfg, $volid);
5198                     }
5199                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5200                 };
5201                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5202             } else {
5203                 print STDERR "unable to parse line in statfile - $line";
5204             }
5205         }
5206         $fd->close();
5207     }
5208 }
5209
5210 sub restore_archive {
5211     my ($archive, $vmid, $user, $opts) = @_;
5212
5213     my $format = $opts->{format};
5214     my $comp;
5215
5216     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5217         $format = 'tar' if !$format;
5218         $comp = 'gzip';
5219     } elsif ($archive =~ m/\.tar$/) {
5220         $format = 'tar' if !$format;
5221     } elsif ($archive =~ m/.tar.lzo$/) {
5222         $format = 'tar' if !$format;
5223         $comp = 'lzop';
5224     } elsif ($archive =~ m/\.vma$/) {
5225         $format = 'vma' if !$format;
5226     } elsif ($archive =~ m/\.vma\.gz$/) {
5227         $format = 'vma' if !$format;
5228         $comp = 'gzip';
5229     } elsif ($archive =~ m/\.vma\.lzo$/) {
5230         $format = 'vma' if !$format;
5231         $comp = 'lzop';
5232     } else {
5233         $format = 'vma' if !$format; # default
5234     }
5235
5236     # try to detect archive format
5237     if ($format eq 'tar') {
5238         return restore_tar_archive($archive, $vmid, $user, $opts);
5239     } else {
5240         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5241     }
5242 }
5243
5244 sub restore_update_config_line {
5245     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5246
5247     return if $line =~ m/^\#qmdump\#/;
5248     return if $line =~ m/^\#vzdump\#/;
5249     return if $line =~ m/^lock:/;
5250     return if $line =~ m/^unused\d+:/;
5251     return if $line =~ m/^parent:/;
5252     return if $line =~ m/^template:/; # restored VM is never a template
5253
5254     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5255         # try to convert old 1.X settings
5256         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5257         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5258             my ($model, $macaddr) = split(/\=/, $devconfig);
5259             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5260             my $net = {
5261                 model => $model,
5262                 bridge => "vmbr$ind",
5263                 macaddr => $macaddr,
5264             };
5265             my $netstr = print_net($net);
5266
5267             print $outfd "net$cookie->{netcount}: $netstr\n";
5268             $cookie->{netcount}++;
5269         }
5270     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5271         my ($id, $netstr) = ($1, $2);
5272         my $net = parse_net($netstr);
5273         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5274         $netstr = print_net($net);
5275         print $outfd "$id: $netstr\n";
5276     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5277         my $virtdev = $1;
5278         my $value = $3;
5279         my $di = parse_drive($virtdev, $value);
5280         if (defined($di->{backup}) && !$di->{backup}) {
5281             print $outfd "#$line";
5282         } elsif ($map->{$virtdev}) {
5283             delete $di->{format}; # format can change on restore
5284             $di->{file} = $map->{$virtdev};
5285             $value = print_drive($vmid, $di);
5286             print $outfd "$virtdev: $value\n";
5287         } else {
5288             print $outfd $line;
5289         }
5290     } else {
5291         print $outfd $line;
5292     }
5293 }
5294
5295 sub scan_volids {
5296     my ($cfg, $vmid) = @_;
5297
5298     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5299
5300     my $volid_hash = {};
5301     foreach my $storeid (keys %$info) {
5302         foreach my $item (@{$info->{$storeid}}) {
5303             next if !($item->{volid} && $item->{size});
5304             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5305             $volid_hash->{$item->{volid}} = $item;
5306         }
5307     }
5308
5309     return $volid_hash;
5310 }
5311
5312 sub is_volume_in_use {
5313     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5314
5315     my $path = PVE::Storage::path($storecfg, $volid);
5316
5317     my $scan_config = sub {
5318         my ($cref, $snapname) = @_;
5319
5320         foreach my $key (keys %$cref) {
5321             my $value = $cref->{$key};
5322             if (valid_drivename($key)) {
5323                 next if $skip_drive && $key eq $skip_drive;
5324                 my $drive = parse_drive($key, $value);
5325                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5326                 return 1 if $volid eq $drive->{file};
5327                 if ($drive->{file} =~ m!^/!) {
5328                     return 1 if $drive->{file} eq $path;
5329                 } else {
5330                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5331                     next if !$storeid;
5332                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5333                     next if !$scfg;
5334                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5335                 }
5336             }
5337         }
5338
5339         return 0;
5340     };
5341
5342     return 1 if &$scan_config($conf);
5343
5344     undef $skip_drive;
5345
5346     foreach my $snapname (keys %{$conf->{snapshots}}) {
5347         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5348     }
5349
5350     return 0;
5351 }
5352
5353 sub update_disksize {
5354     my ($vmid, $conf, $volid_hash) = @_;
5355
5356     my $changes;
5357
5358     my $used = {};
5359
5360     # Note: it is allowed to define multiple storages with same path (alias), so
5361     # we need to check both 'volid' and real 'path' (two different volid can point
5362     # to the same path).
5363
5364     my $usedpath = {};
5365
5366     # update size info
5367     foreach my $opt (keys %$conf) {
5368         if (valid_drivename($opt)) {
5369             my $drive = parse_drive($opt, $conf->{$opt});
5370             my $volid = $drive->{file};
5371             next if !$volid;
5372
5373             $used->{$volid} = 1;
5374             if ($volid_hash->{$volid} &&
5375                 (my $path = $volid_hash->{$volid}->{path})) {
5376                 $usedpath->{$path} = 1;
5377             }
5378
5379             next if drive_is_cdrom($drive);
5380             next if !$volid_hash->{$volid};
5381
5382             $drive->{size} = $volid_hash->{$volid}->{size};
5383             my $new = print_drive($vmid, $drive);
5384             if ($new ne $conf->{$opt}) {
5385                 $changes = 1;
5386                 $conf->{$opt} = $new;
5387             }
5388         }
5389     }
5390
5391     # remove 'unusedX' entry if volume is used
5392     foreach my $opt (keys %$conf) {
5393         next if $opt !~ m/^unused\d+$/;
5394         my $volid = $conf->{$opt};
5395         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5396         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5397             $changes = 1;
5398             delete $conf->{$opt};
5399         }
5400     }
5401
5402     foreach my $volid (sort keys %$volid_hash) {
5403         next if $volid =~ m/vm-$vmid-state-/;
5404         next if $used->{$volid};
5405         my $path = $volid_hash->{$volid}->{path};
5406         next if !$path; # just to be sure
5407         next if $usedpath->{$path};
5408         $changes = 1;
5409         add_unused_volume($conf, $volid);
5410         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5411     }
5412
5413     return $changes;
5414 }
5415
5416 sub rescan {
5417     my ($vmid, $nolock) = @_;
5418
5419     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5420
5421     my $volid_hash = scan_volids($cfg, $vmid);
5422
5423     my $updatefn =  sub {
5424         my ($vmid) = @_;
5425
5426         my $conf = load_config($vmid);
5427
5428         check_lock($conf);
5429
5430         my $vm_volids = {};
5431         foreach my $volid (keys %$volid_hash) {
5432             my $info = $volid_hash->{$volid};
5433             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5434         }
5435
5436         my $changes = update_disksize($vmid, $conf, $vm_volids);
5437
5438         write_config($vmid, $conf) if $changes;
5439     };
5440
5441     if (defined($vmid)) {
5442         if ($nolock) {
5443             &$updatefn($vmid);
5444         } else {
5445             lock_config($vmid, $updatefn, $vmid);
5446         }
5447     } else {
5448         my $vmlist = config_list();
5449         foreach my $vmid (keys %$vmlist) {
5450             if ($nolock) {
5451                 &$updatefn($vmid);
5452             } else {
5453                 lock_config($vmid, $updatefn, $vmid);
5454             }
5455         }
5456     }
5457 }
5458
5459 sub restore_vma_archive {
5460     my ($archive, $vmid, $user, $opts, $comp) = @_;
5461
5462     my $input = $archive eq '-' ? "<&STDIN" : undef;
5463     my $readfrom = $archive;
5464
5465     my $uncomp = '';
5466     if ($comp) {
5467         $readfrom = '-';
5468         my $qarchive = PVE::Tools::shellquote($archive);
5469         if ($comp eq 'gzip') {
5470             $uncomp = "zcat $qarchive|";
5471         } elsif ($comp eq 'lzop') {
5472             $uncomp = "lzop -d -c $qarchive|";
5473         } else {
5474             die "unknown compression method '$comp'\n";
5475         }
5476
5477     }
5478
5479     my $tmpdir = "/var/tmp/vzdumptmp$$";
5480     rmtree $tmpdir;
5481
5482     # disable interrupts (always do cleanups)
5483     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5484         warn "got interrupt - ignored\n";
5485     };
5486
5487     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5488     POSIX::mkfifo($mapfifo, 0600);
5489     my $fifofh;
5490
5491     my $openfifo = sub {
5492         open($fifofh, '>', $mapfifo) || die $!;
5493     };
5494
5495     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5496
5497     my $oldtimeout;
5498     my $timeout = 5;
5499
5500     my $devinfo = {};
5501
5502     my $rpcenv = PVE::RPCEnvironment::get();
5503
5504     my $conffile = config_file($vmid);
5505     my $tmpfn = "$conffile.$$.tmp";
5506
5507     # Note: $oldconf is undef if VM does not exists
5508     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5509
5510     my $print_devmap = sub {
5511         my $virtdev_hash = {};
5512
5513         my $cfgfn = "$tmpdir/qemu-server.conf";
5514
5515         # we can read the config - that is already extracted
5516         my $fh = IO::File->new($cfgfn, "r") ||
5517             "unable to read qemu-server.conf - $!\n";
5518
5519         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5520         if (-f $fwcfgfn) {
5521             my $pve_firewall_dir = '/etc/pve/firewall';
5522             mkdir $pve_firewall_dir; # make sure the dir exists
5523             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5524         }
5525
5526         while (defined(my $line = <$fh>)) {
5527             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5528                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5529                 die "archive does not contain data for drive '$virtdev'\n"
5530                     if !$devinfo->{$devname};
5531                 if (defined($opts->{storage})) {
5532                     $storeid = $opts->{storage} || 'local';
5533                 } elsif (!$storeid) {
5534                     $storeid = 'local';
5535                 }
5536                 $format = 'raw' if !$format;
5537                 $devinfo->{$devname}->{devname} = $devname;
5538                 $devinfo->{$devname}->{virtdev} = $virtdev;
5539                 $devinfo->{$devname}->{format} = $format;
5540                 $devinfo->{$devname}->{storeid} = $storeid;
5541
5542                 # check permission on storage
5543                 my $pool = $opts->{pool}; # todo: do we need that?
5544                 if ($user ne 'root@pam') {
5545                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5546                 }
5547
5548                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5549             }
5550         }
5551
5552         foreach my $devname (keys %$devinfo) {
5553             die "found no device mapping information for device '$devname'\n"
5554                 if !$devinfo->{$devname}->{virtdev};
5555         }
5556
5557         my $cfg = cfs_read_file('storage.cfg');
5558
5559         # create empty/temp config
5560         if ($oldconf) {
5561             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5562             foreach_drive($oldconf, sub {
5563                 my ($ds, $drive) = @_;
5564
5565                 return if drive_is_cdrom($drive);
5566
5567                 my $volid = $drive->{file};
5568
5569                 return if !$volid || $volid =~ m|^/|;
5570
5571                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5572                 return if !$path || !$owner || ($owner != $vmid);
5573
5574                 # Note: only delete disk we want to restore
5575                 # other volumes will become unused
5576                 if ($virtdev_hash->{$ds}) {
5577                     PVE::Storage::vdisk_free($cfg, $volid);
5578                 }
5579             });
5580
5581             # delete vmstate files
5582             # since after the restore we have no snapshots anymore
5583             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5584                 my $snap = $oldconf->{snapshots}->{$snapname};
5585                 if ($snap->{vmstate}) {
5586                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5587                     if (my $err = $@) {
5588                         warn $err;
5589                     }
5590                 }
5591             }
5592         }
5593
5594         my $map = {};
5595         foreach my $virtdev (sort keys %$virtdev_hash) {
5596             my $d = $virtdev_hash->{$virtdev};
5597             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5598             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5599
5600             # test if requested format is supported
5601             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5602             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5603             $d->{format} = $defFormat if !$supported;
5604
5605             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5606                                                   $d->{format}, undef, $alloc_size);
5607             print STDERR "new volume ID is '$volid'\n";
5608             $d->{volid} = $volid;
5609             my $path = PVE::Storage::path($cfg, $volid);
5610
5611             PVE::Storage::activate_volumes($cfg,[$volid]);
5612
5613             my $write_zeros = 1;
5614             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5615                 $write_zeros = 0;
5616             }
5617
5618             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5619
5620             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5621             $map->{$virtdev} = $volid;
5622         }
5623
5624         $fh->seek(0, 0) || die "seek failed - $!\n";
5625
5626         my $outfd = new IO::File ($tmpfn, "w") ||
5627             die "unable to write config for VM $vmid\n";
5628
5629         my $cookie = { netcount => 0 };
5630         while (defined(my $line = <$fh>)) {
5631             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5632         }
5633
5634         $fh->close();
5635         $outfd->close();
5636     };
5637
5638     eval {
5639         # enable interrupts
5640         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5641             die "interrupted by signal\n";
5642         };
5643         local $SIG{ALRM} = sub { die "got timeout\n"; };
5644
5645         $oldtimeout = alarm($timeout);
5646
5647         my $parser = sub {
5648             my $line = shift;
5649
5650             print "$line\n";
5651
5652             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5653                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5654                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5655             } elsif ($line =~ m/^CTIME: /) {
5656                 # we correctly received the vma config, so we can disable
5657                 # the timeout now for disk allocation (set to 10 minutes, so
5658                 # that we always timeout if something goes wrong)
5659                 alarm(600);
5660                 &$print_devmap();
5661                 print $fifofh "done\n";
5662                 my $tmp = $oldtimeout || 0;
5663                 $oldtimeout = undef;
5664                 alarm($tmp);
5665                 close($fifofh);
5666             }
5667         };
5668
5669         print "restore vma archive: $cmd\n";
5670         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5671     };
5672     my $err = $@;
5673
5674     alarm($oldtimeout) if $oldtimeout;
5675
5676     my $vollist = [];
5677     foreach my $devname (keys %$devinfo) {
5678         my $volid = $devinfo->{$devname}->{volid};
5679         push @$vollist, $volid if $volid;
5680     }
5681
5682     my $cfg = cfs_read_file('storage.cfg');
5683     PVE::Storage::deactivate_volumes($cfg, $vollist);
5684
5685     unlink $mapfifo;
5686
5687     if ($err) {
5688         rmtree $tmpdir;
5689         unlink $tmpfn;
5690
5691         foreach my $devname (keys %$devinfo) {
5692             my $volid = $devinfo->{$devname}->{volid};
5693             next if !$volid;
5694             eval {
5695                 if ($volid =~ m|^/|) {
5696                     unlink $volid || die 'unlink failed\n';
5697                 } else {
5698                     PVE::Storage::vdisk_free($cfg, $volid);
5699                 }
5700                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5701             };
5702             print STDERR "unable to cleanup '$volid' - $@" if $@;
5703         }
5704         die $err;
5705     }
5706
5707     rmtree $tmpdir;
5708
5709     rename($tmpfn, $conffile) ||
5710         die "unable to commit configuration file '$conffile'\n";
5711
5712     PVE::Cluster::cfs_update(); # make sure we read new file
5713
5714     eval { rescan($vmid, 1); };
5715     warn $@ if $@;
5716 }
5717
5718 sub restore_tar_archive {
5719     my ($archive, $vmid, $user, $opts) = @_;
5720
5721     if ($archive ne '-') {
5722         my $firstfile = tar_archive_read_firstfile($archive);
5723         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5724             if $firstfile ne 'qemu-server.conf';
5725     }
5726
5727     my $storecfg = cfs_read_file('storage.cfg');
5728
5729     # destroy existing data - keep empty config
5730     my $vmcfgfn = config_file($vmid);
5731     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5732
5733     my $tocmd = "/usr/lib/qemu-server/qmextract";
5734
5735     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5736     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5737     $tocmd .= ' --prealloc' if $opts->{prealloc};
5738     $tocmd .= ' --info' if $opts->{info};
5739
5740     # tar option "xf" does not autodetect compression when read from STDIN,
5741     # so we pipe to zcat
5742     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5743         PVE::Tools::shellquote("--to-command=$tocmd");
5744
5745     my $tmpdir = "/var/tmp/vzdumptmp$$";
5746     mkpath $tmpdir;
5747
5748     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5749     local $ENV{VZDUMP_VMID} = $vmid;
5750     local $ENV{VZDUMP_USER} = $user;
5751
5752     my $conffile = config_file($vmid);
5753     my $tmpfn = "$conffile.$$.tmp";
5754
5755     # disable interrupts (always do cleanups)
5756     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5757         print STDERR "got interrupt - ignored\n";
5758     };
5759
5760     eval {
5761         # enable interrupts
5762         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5763             die "interrupted by signal\n";
5764         };
5765
5766         if ($archive eq '-') {
5767             print "extracting archive from STDIN\n";
5768             run_command($cmd, input => "<&STDIN");
5769         } else {
5770             print "extracting archive '$archive'\n";
5771             run_command($cmd);
5772         }
5773
5774         return if $opts->{info};
5775
5776         # read new mapping
5777         my $map = {};
5778         my $statfile = "$tmpdir/qmrestore.stat";
5779         if (my $fd = IO::File->new($statfile, "r")) {
5780             while (defined (my $line = <$fd>)) {
5781                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5782                     $map->{$1} = $2 if $1;
5783                 } else {
5784                     print STDERR "unable to parse line in statfile - $line\n";
5785                 }
5786             }
5787             $fd->close();
5788         }
5789
5790         my $confsrc = "$tmpdir/qemu-server.conf";
5791
5792         my $srcfd = new IO::File($confsrc, "r") ||
5793             die "unable to open file '$confsrc'\n";
5794
5795         my $outfd = new IO::File ($tmpfn, "w") ||
5796             die "unable to write config for VM $vmid\n";
5797
5798         my $cookie = { netcount => 0 };
5799         while (defined (my $line = <$srcfd>)) {
5800             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5801         }
5802
5803         $srcfd->close();
5804         $outfd->close();
5805     };
5806     my $err = $@;
5807
5808     if ($err) {
5809
5810         unlink $tmpfn;
5811
5812         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5813
5814         die $err;
5815     }
5816
5817     rmtree $tmpdir;
5818
5819     rename $tmpfn, $conffile ||
5820         die "unable to commit configuration file '$conffile'\n";
5821
5822     PVE::Cluster::cfs_update(); # make sure we read new file
5823
5824     eval { rescan($vmid, 1); };
5825     warn $@ if $@;
5826 };
5827
5828
5829 # Internal snapshots
5830
5831 # NOTE: Snapshot create/delete involves several non-atomic
5832 # action, and can take a long time.
5833 # So we try to avoid locking the file and use 'lock' variable
5834 # inside the config file instead.
5835
5836 my $snapshot_copy_config = sub {
5837     my ($source, $dest) = @_;
5838
5839     foreach my $k (keys %$source) {
5840         next if $k eq 'snapshots';
5841         next if $k eq 'snapstate';
5842         next if $k eq 'snaptime';
5843         next if $k eq 'vmstate';
5844         next if $k eq 'lock';
5845         next if $k eq 'digest';
5846         next if $k eq 'description';
5847         next if $k =~ m/^unused\d+$/;
5848
5849         $dest->{$k} = $source->{$k};
5850     }
5851 };
5852
5853 my $snapshot_apply_config = sub {
5854     my ($conf, $snap) = @_;
5855
5856     # copy snapshot list
5857     my $newconf = {
5858         snapshots => $conf->{snapshots},
5859     };
5860
5861     # keep description and list of unused disks
5862     foreach my $k (keys %$conf) {
5863         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5864         $newconf->{$k} = $conf->{$k};
5865     }
5866
5867     &$snapshot_copy_config($snap, $newconf);
5868
5869     return $newconf;
5870 };
5871
5872 sub foreach_writable_storage {
5873     my ($conf, $func) = @_;
5874
5875     my $sidhash = {};
5876
5877     foreach my $ds (keys %$conf) {
5878         next if !valid_drivename($ds);
5879
5880         my $drive = parse_drive($ds, $conf->{$ds});
5881         next if !$drive;
5882         next if drive_is_cdrom($drive);
5883
5884         my $volid = $drive->{file};
5885
5886         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5887         $sidhash->{$sid} = $sid if $sid;
5888     }
5889
5890     foreach my $sid (sort keys %$sidhash) {
5891         &$func($sid);
5892     }
5893 }
5894
5895 my $alloc_vmstate_volid = sub {
5896     my ($storecfg, $vmid, $conf, $snapname) = @_;
5897
5898     # Note: we try to be smart when selecting a $target storage
5899
5900     my $target;
5901
5902     # search shared storage first
5903     foreach_writable_storage($conf, sub {
5904         my ($sid) = @_;
5905         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5906         return if !$scfg->{shared};
5907
5908         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5909     });
5910
5911     if (!$target) {
5912         # now search local storage
5913         foreach_writable_storage($conf, sub {
5914             my ($sid) = @_;
5915             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5916             return if $scfg->{shared};
5917
5918             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5919         });
5920     }
5921
5922     $target = 'local' if !$target;
5923
5924     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5925     # we abort live save after $conf->{memory}, so we need at max twice that space
5926     my $size = $conf->{memory}*2 + $driver_state_size;
5927
5928     my $name = "vm-$vmid-state-$snapname";
5929     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5930     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5931     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5932
5933     return $volid;
5934 };
5935
5936 sub snapshot_save_vmstate {
5937     my ($vmid, $conf, $snapname, $storecfg) = @_;
5938
5939     my $snap = $conf->{snapshots}->{$snapname};
5940
5941     $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5942     # always overwrite machine if we save vmstate. This makes sure we
5943     # can restore it later using correct machine type
5944     $snap->{machine} = get_current_qemu_machine($vmid);
5945 }
5946
5947 sub snapshot_prepare {
5948     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5949
5950     my $snap;
5951
5952     my $updatefn =  sub {
5953
5954         my $conf = load_config($vmid);
5955
5956         die "you can't take a snapshot if it's a template\n"
5957             if is_template($conf);
5958
5959         check_lock($conf);
5960
5961         $conf->{lock} = 'snapshot';
5962
5963         die "snapshot name '$snapname' already used\n"
5964             if defined($conf->{snapshots}->{$snapname});
5965
5966         my $storecfg = PVE::Storage::config();
5967         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
5968
5969         $snap = $conf->{snapshots}->{$snapname} = {};
5970
5971         if ($save_vmstate && check_running($vmid)) {
5972             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
5973         }
5974
5975         &$snapshot_copy_config($conf, $snap);
5976
5977         $snap->{snapstate} = "prepare";
5978         $snap->{snaptime} = time();
5979         $snap->{description} = $comment if $comment;
5980
5981         write_config($vmid, $conf);
5982     };
5983
5984     lock_config($vmid, $updatefn);
5985
5986     return $snap;
5987 }
5988
5989 sub snapshot_commit {
5990     my ($vmid, $snapname) = @_;
5991
5992     my $updatefn = sub {
5993
5994         my $conf = load_config($vmid);
5995
5996         die "missing snapshot lock\n"
5997             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5998
5999         my $has_machine_config = defined($conf->{machine});
6000
6001         my $snap = $conf->{snapshots}->{$snapname};
6002
6003         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6004
6005         die "wrong snapshot state\n"
6006             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
6007
6008         delete $snap->{snapstate};
6009         delete $conf->{lock};
6010
6011         my $newconf = &$snapshot_apply_config($conf, $snap);
6012
6013         delete $newconf->{machine} if !$has_machine_config;
6014
6015         $newconf->{parent} = $snapname;
6016
6017         write_config($vmid, $newconf);
6018     };
6019
6020     lock_config($vmid, $updatefn);
6021 }
6022
6023 sub snapshot_rollback {
6024     my ($vmid, $snapname) = @_;
6025
6026     my $prepare = 1;
6027
6028     my $storecfg = PVE::Storage::config();
6029
6030     my $conf = load_config($vmid);
6031
6032     my $get_snapshot_config = sub {
6033
6034         die "you can't rollback if vm is a template\n" if is_template($conf);
6035
6036         my $res = $conf->{snapshots}->{$snapname};
6037
6038         die "snapshot '$snapname' does not exist\n" if !defined($res);
6039
6040         return $res;
6041     };
6042
6043     my $snap = &$get_snapshot_config();
6044
6045     foreach_drive($snap, sub {
6046         my ($ds, $drive) = @_;
6047
6048         return if drive_is_cdrom($drive);
6049
6050         my $volid = $drive->{file};
6051
6052         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
6053     });
6054
6055     my $updatefn = sub {
6056
6057         $conf = load_config($vmid);
6058
6059         $snap = &$get_snapshot_config();
6060
6061         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6062             if $snap->{snapstate};
6063
6064         if ($prepare) {
6065             check_lock($conf);
6066             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6067         }
6068
6069         die "unable to rollback vm $vmid: vm is running\n"
6070             if check_running($vmid);
6071
6072         if ($prepare) {
6073             $conf->{lock} = 'rollback';
6074         } else {
6075             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6076             delete $conf->{lock};
6077         }
6078
6079         my $forcemachine;
6080
6081         if (!$prepare) {
6082             my $has_machine_config = defined($conf->{machine});
6083
6084             # copy snapshot config to current config
6085             $conf = &$snapshot_apply_config($conf, $snap);
6086             $conf->{parent} = $snapname;
6087
6088             # Note: old code did not store 'machine', so we try to be smart
6089             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6090             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6091             # we remove the 'machine' configuration if not explicitly specified
6092             # in the original config.
6093             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6094         }
6095
6096         write_config($vmid, $conf);
6097
6098         if (!$prepare && $snap->{vmstate}) {
6099             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6100             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6101         }
6102     };
6103
6104     lock_config($vmid, $updatefn);
6105
6106     foreach_drive($snap, sub {
6107         my ($ds, $drive) = @_;
6108
6109         return if drive_is_cdrom($drive);
6110
6111         my $volid = $drive->{file};
6112         my $device = "drive-$ds";
6113
6114         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6115     });
6116
6117     $prepare = 0;
6118     lock_config($vmid, $updatefn);
6119 }
6120
6121 my $savevm_wait = sub {
6122     my ($vmid) = @_;
6123
6124     for(;;) {
6125         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6126         if (!$stat->{status}) {
6127             die "savevm not active\n";
6128         } elsif ($stat->{status} eq 'active') {
6129             sleep(1);
6130             next;
6131         } elsif ($stat->{status} eq 'completed') {
6132             last;
6133         } else {
6134             die "query-savevm returned status '$stat->{status}'\n";
6135         }
6136     }
6137 };
6138
6139 sub do_snapshots_with_qemu {
6140     my ($storecfg, $volid) = @_;
6141
6142     my $storage_name = PVE::Storage::parse_volume_id($volid);
6143
6144     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6145         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6146         return 1;
6147     }
6148
6149     if ($volid =~ m/\.(qcow2|qed)$/){
6150         return 1;
6151     }
6152
6153     return undef;
6154 }
6155
6156 sub qga_check_running {
6157     my ($vmid) = @_;
6158
6159     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6160     if ($@) {
6161         warn "Qemu Guest Agent are not running - $@";
6162         return 0;
6163     }
6164     return 1;
6165 }
6166
6167 sub check_freeze_needed {
6168     my ($vmid, $config, $save_vmstate) = @_;
6169
6170     my $running = check_running($vmid);
6171     if ($save_vmstate) {
6172         return ($running, $running && $config->{agent} && qga_check_running($vmid));
6173     } else {
6174         return ($running, 0);
6175     }
6176 }
6177
6178 sub snapshot_create {
6179     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6180
6181     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6182
6183     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6184
6185     my $config = load_config($vmid);
6186
6187     my ($running, $freezefs) = check_freeze_needed($vmid, $config, $snap->{vmstate});
6188
6189     my $drivehash = {};
6190
6191     eval {
6192         if ($freezefs) {
6193             eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6194             warn "guest-fsfreeze-freeze problems - $@" if $@;
6195         }
6196
6197         # create internal snapshots of all drives
6198
6199         my $storecfg = PVE::Storage::config();
6200
6201         if ($running) {
6202             if ($snap->{vmstate}) {
6203                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6204                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6205                 &$savevm_wait($vmid);
6206             } else {
6207                 vm_mon_cmd($vmid, "savevm-start");
6208             }
6209         };
6210
6211         foreach_drive($snap, sub {
6212             my ($ds, $drive) = @_;
6213
6214             return if drive_is_cdrom($drive);
6215
6216             my $volid = $drive->{file};
6217             my $device = "drive-$ds";
6218
6219             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6220             $drivehash->{$ds} = 1;
6221        });
6222     };
6223     my $err = $@;
6224
6225     if ($running) {
6226         eval { vm_mon_cmd($vmid, "savevm-end")  };
6227         warn $@ if $@;
6228
6229         if ($freezefs) {
6230             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6231             warn "guest-fsfreeze-thaw problems - $@" if $@;
6232         }
6233
6234         # savevm-end is async, we need to wait
6235         for (;;) {
6236             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6237             if (!$stat->{bytes}) {
6238                 last;
6239             } else {
6240                 print "savevm not yet finished\n";
6241                 sleep(1);
6242                 next;
6243             }
6244         }
6245     }
6246
6247     if ($err) {
6248         warn "snapshot create failed: starting cleanup\n";
6249         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
6250         warn $@ if $@;
6251         die $err;
6252     }
6253
6254     snapshot_commit($vmid, $snapname);
6255 }
6256
6257 # Note: $drivehash is only set when called from snapshot_create.
6258 sub snapshot_delete {
6259     my ($vmid, $snapname, $force, $drivehash) = @_;
6260
6261     my $prepare = 1;
6262
6263     my $snap;
6264     my $unused = [];
6265
6266     my $unlink_parent = sub {
6267         my ($confref, $new_parent) = @_;
6268
6269         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6270             if ($new_parent) {
6271                 $confref->{parent} = $new_parent;
6272             } else {
6273                 delete $confref->{parent};
6274             }
6275         }
6276     };
6277
6278     my $updatefn =  sub {
6279         my ($remove_drive) = @_;
6280
6281         my $conf = load_config($vmid);
6282
6283         if (!$drivehash) {
6284             check_lock($conf);
6285             die "you can't delete a snapshot if vm is a template\n"
6286                 if is_template($conf);
6287         }
6288
6289         $snap = $conf->{snapshots}->{$snapname};
6290
6291         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6292
6293         # remove parent refs
6294         if (!$prepare) {
6295             &$unlink_parent($conf, $snap->{parent});
6296             foreach my $sn (keys %{$conf->{snapshots}}) {
6297                 next if $sn eq $snapname;
6298                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6299             }
6300         }
6301
6302         if ($remove_drive) {
6303             if ($remove_drive eq 'vmstate') {
6304                 delete $snap->{$remove_drive};
6305             } else {
6306                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6307                 my $volid = $drive->{file};
6308                 delete $snap->{$remove_drive};
6309                 add_unused_volume($conf, $volid);
6310             }
6311         }
6312
6313         if ($prepare) {
6314             $snap->{snapstate} = 'delete';
6315         } else {
6316             delete $conf->{snapshots}->{$snapname};
6317             delete $conf->{lock} if $drivehash;
6318             foreach my $volid (@$unused) {
6319                 add_unused_volume($conf, $volid);
6320             }
6321         }
6322
6323         write_config($vmid, $conf);
6324     };
6325
6326     lock_config($vmid, $updatefn);
6327
6328     # now remove vmstate file
6329
6330     my $storecfg = PVE::Storage::config();
6331
6332     if ($snap->{vmstate}) {
6333         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6334         if (my $err = $@) {
6335             die $err if !$force;
6336             warn $err;
6337         }
6338         # save changes (remove vmstate from snapshot)
6339         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6340     };
6341
6342     # now remove all internal snapshots
6343     foreach_drive($snap, sub {
6344         my ($ds, $drive) = @_;
6345
6346         return if drive_is_cdrom($drive);
6347
6348         my $volid = $drive->{file};
6349         my $device = "drive-$ds";
6350
6351         if (!$drivehash || $drivehash->{$ds}) {
6352             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6353             if (my $err = $@) {
6354                 die $err if !$force;
6355                 warn $err;
6356             }
6357         }
6358
6359         # save changes (remove drive fron snapshot)
6360         lock_config($vmid, $updatefn, $ds) if !$force;
6361         push @$unused, $volid;
6362     });
6363
6364     # now cleanup config
6365     $prepare = 0;
6366     lock_config($vmid, $updatefn);
6367 }
6368
6369 sub has_feature {
6370     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6371
6372     my $err;
6373     foreach_drive($conf, sub {
6374         my ($ds, $drive) = @_;
6375
6376         return if drive_is_cdrom($drive);
6377         my $volid = $drive->{file};
6378         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6379     });
6380
6381     return $err ? 0 : 1;
6382 }
6383
6384 sub template_create {
6385     my ($vmid, $conf, $disk) = @_;
6386
6387     my $storecfg = PVE::Storage::config();
6388
6389     foreach_drive($conf, sub {
6390         my ($ds, $drive) = @_;
6391
6392         return if drive_is_cdrom($drive);
6393         return if $disk && $ds ne $disk;
6394
6395         my $volid = $drive->{file};
6396         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6397
6398         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6399         $drive->{file} = $voliddst;
6400         $conf->{$ds} = print_drive($vmid, $drive);
6401         write_config($vmid, $conf);
6402     });
6403 }
6404
6405 sub is_template {
6406     my ($conf) = @_;
6407
6408     return 1 if defined $conf->{template} && $conf->{template} == 1;
6409 }
6410
6411 sub qemu_img_convert {
6412     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6413
6414     my $storecfg = PVE::Storage::config();
6415     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6416     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6417
6418     if ($src_storeid && $dst_storeid) {
6419
6420         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6421
6422         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6423         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6424
6425         my $src_format = qemu_img_format($src_scfg, $src_volname);
6426         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6427
6428         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6429         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6430
6431         my $cmd = [];
6432         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6433         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6434         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6435
6436         my $parser = sub {
6437             my $line = shift;
6438             if($line =~ m/\((\S+)\/100\%\)/){
6439                 my $percent = $1;
6440                 my $transferred = int($size * $percent / 100);
6441                 my $remaining = $size - $transferred;
6442
6443                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6444             }
6445
6446         };
6447
6448         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6449         my $err = $@;
6450         die "copy failed: $err" if $err;
6451     }
6452 }
6453
6454 sub qemu_img_format {
6455     my ($scfg, $volname) = @_;
6456
6457     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6458         return $1;
6459     } else {
6460         return "raw";
6461     }
6462 }
6463
6464 sub qemu_drive_mirror {
6465     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6466
6467     my $storecfg = PVE::Storage::config();
6468     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6469
6470     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6471
6472     my $format = qemu_img_format($dst_scfg, $dst_volname);
6473
6474     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6475
6476     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6477     $opts->{format} = $format if $format;
6478
6479     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6480
6481     eval {
6482     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6483         while (1) {
6484             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6485             my $stat = @$stats[0];
6486             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6487             die "error job is not mirroring" if $stat->{type} ne "mirror";
6488
6489             my $busy = $stat->{busy};
6490             my $ready = $stat->{ready};
6491
6492             if (my $total = $stat->{len}) {
6493                 my $transferred = $stat->{offset} || 0;
6494                 my $remaining = $total - $transferred;
6495                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6496
6497                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6498             }
6499
6500
6501             if ($stat->{ready} eq 'true') {
6502
6503                 last if $vmiddst != $vmid;
6504
6505                 # try to switch the disk if source and destination are on the same guest
6506                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6507                 last if !$@;
6508                 die $@ if $@ !~ m/cannot be completed/;
6509             }
6510             sleep 1;
6511         }
6512
6513
6514     };
6515     my $err = $@;
6516
6517     my $cancel_job = sub {
6518         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6519         while (1) {
6520             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6521             my $stat = @$stats[0];
6522             last if !$stat;
6523             sleep 1;
6524         }
6525     };
6526
6527     if ($err) {
6528         eval { &$cancel_job(); };
6529         die "mirroring error: $err";
6530     }
6531
6532     if ($vmiddst != $vmid) {
6533         # if we clone a disk for a new target vm, we don't switch the disk
6534         &$cancel_job(); # so we call block-job-cancel
6535     }
6536 }
6537
6538 sub clone_disk {
6539     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6540         $newvmid, $storage, $format, $full, $newvollist) = @_;
6541
6542     my $newvolid;
6543
6544     if (!$full) {
6545         print "create linked clone of drive $drivename ($drive->{file})\n";
6546         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6547         push @$newvollist, $newvolid;
6548     } else {
6549         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6550         $storeid = $storage if $storage;
6551
6552         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6553         if (!$format) {
6554             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6555             $format = qemu_img_format($scfg, $volname);
6556         }
6557
6558         # test if requested format is supported - else use default
6559         my $supported = grep { $_ eq $format } @$validFormats;
6560         $format = $defFormat if !$supported;
6561
6562         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6563
6564         print "create full clone of drive $drivename ($drive->{file})\n";
6565         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6566         push @$newvollist, $newvolid;
6567
6568         PVE::Storage::activate_volumes($storecfg, $newvollist);
6569
6570         if (!$running || $snapname) {
6571             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6572         } else {
6573             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6574         }
6575     }
6576
6577     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6578
6579     my $disk = $drive;
6580     $disk->{format} = undef;
6581     $disk->{file} = $newvolid;
6582     $disk->{size} = $size;
6583
6584     return $disk;
6585 }
6586
6587 # this only works if VM is running
6588 sub get_current_qemu_machine {
6589     my ($vmid) = @_;
6590
6591     my $cmd = { execute => 'query-machines', arguments => {} };
6592     my $res = vm_qmp_command($vmid, $cmd);
6593
6594     my ($current, $default);
6595     foreach my $e (@$res) {
6596         $default = $e->{name} if $e->{'is-default'};
6597         $current = $e->{name} if $e->{'is-current'};
6598     }
6599
6600     # fallback to the default machine if current is not supported by qemu
6601     return $current || $default || 'pc';
6602 }
6603
6604 sub qemu_machine_feature_enabled {
6605     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6606
6607     my $current_major;
6608     my $current_minor;
6609
6610     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6611
6612         $current_major = $3;
6613         $current_minor = $4;
6614
6615     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6616
6617         $current_major = $1;
6618         $current_minor = $2;
6619     }
6620
6621     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6622
6623
6624 }
6625
6626 sub qemu_machine_pxe {
6627     my ($vmid, $conf, $machine) = @_;
6628
6629     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6630
6631     foreach my $opt (keys %$conf) {
6632         next if $opt !~ m/^net(\d+)$/;
6633         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6634         next if !$net;
6635         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6636         return $machine.".pxe" if $romfile =~ m/pxe/;
6637         last;
6638     }
6639
6640     return $machine;
6641 }
6642
6643 sub qemu_use_old_bios_files {
6644     my ($machine_type) = @_;
6645
6646     return if !$machine_type;
6647
6648     my $use_old_bios_files = undef;
6649
6650     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6651         $machine_type = $1;
6652         $use_old_bios_files = 1;
6653     } else {
6654         my $kvmver = kvm_user_version();
6655         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6656         # load new efi bios files on migration. So this hack is required to allow
6657         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6658         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6659         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6660     }
6661
6662     return ($use_old_bios_files, $machine_type);
6663 }
6664
6665 sub lspci {
6666
6667     my $devices = {};
6668
6669     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6670             my (undef, $id, $function) = @_;
6671             my $res = { id => $id, function => $function};
6672             push @{$devices->{$id}}, $res;
6673     });
6674
6675     return $devices;
6676 }
6677
6678 sub vm_iothreads_list {
6679     my ($vmid) = @_;
6680
6681     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6682
6683     my $iothreads = {};
6684     foreach my $iothread (@$res) {
6685         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6686     }
6687
6688     return $iothreads;
6689 }
6690
6691 sub scsihw_infos {
6692     my ($conf, $drive) = @_;
6693
6694     my $maxdev = 0;
6695
6696     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6697         $maxdev = 7;
6698     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6699         $maxdev = 1;
6700     } else {
6701         $maxdev = 256;
6702     }
6703
6704     my $controller = int($drive->{index} / $maxdev);
6705     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6706
6707     return ($maxdev, $controller, $controller_prefix);
6708 }
6709
6710 # bash completion helper
6711
6712 sub complete_backup_archives {
6713     my ($cmdname, $pname, $cvalue) = @_;
6714
6715     my $cfg = PVE::Storage::config();
6716
6717     my $storeid;
6718
6719     if ($cvalue =~ m/^([^:]+):/) {
6720         $storeid = $1;
6721     }
6722
6723     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6724
6725     my $res = [];
6726     foreach my $id (keys %$data) {
6727         foreach my $item (@{$data->{$id}}) {
6728             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6729             push @$res, $item->{volid} if defined($item->{volid});
6730         }
6731     }
6732
6733     return $res;
6734 }
6735
6736 my $complete_vmid_full = sub {
6737     my ($running) = @_;
6738
6739     my $idlist = vmstatus();
6740
6741     my $res = [];
6742
6743     foreach my $id (keys %$idlist) {
6744         my $d = $idlist->{$id};
6745         if (defined($running)) {
6746             next if $d->{template};
6747             next if $running && $d->{status} ne 'running';
6748             next if !$running && $d->{status} eq 'running';
6749         }
6750         push @$res, $id;
6751
6752     }
6753     return $res;
6754 };
6755
6756 sub complete_vmid {
6757     return &$complete_vmid_full();
6758 }
6759
6760 sub complete_vmid_stopped {
6761     return &$complete_vmid_full(0);
6762 }
6763
6764 sub complete_vmid_running {
6765     return &$complete_vmid_full(1);
6766 }
6767
6768 sub complete_storage {
6769
6770     my $cfg = PVE::Storage::config();
6771     my $ids = $cfg->{ids};
6772
6773     my $res = [];
6774     foreach my $sid (keys %$ids) {
6775         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6776         next if !$ids->{$sid}->{content}->{images};
6777         push @$res, $sid;
6778     }
6779
6780     return $res;
6781 }
6782
6783 1;