PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 unless(defined(&_VZSYSCALLS_H_)) {
  71     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  72     require 'sys/syscall.ph';
  73     if(defined(&__x86_64__)) {
  74         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  75         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  76         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  77         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  78         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  79         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  80         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  81         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  82     }
  83     elsif(defined( &__i386__) ) {
  84         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  85         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  86         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  87         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  88         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  89         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  90         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  91         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  92     } else {
  93         die("no fairsched syscall for this arch");
  94     }
  95     require 'asm/ioctl.ph';
  96     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  97 }
  98
  99 sub fairsched_mknod {
 100     my ($parent, $weight, $desired) = @_;
 101
 102     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 103 }
 104
 105 sub fairsched_rmnod {
 106     my ($id) = @_;
 107
 108     return syscall(&__NR_fairsched_rmnod, int($id));
 109 }
 110
 111 sub fairsched_mvpr {
 112     my ($pid, $newid) = @_;
 113
 114     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 115 }
 116
 117 sub fairsched_vcpus {
 118     my ($id, $vcpus) = @_;
 119
 120     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 121 }
 122
 123 sub fairsched_rate {
 124     my ($id, $op, $rate) = @_;
 125
 126     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 127 }
 128
 129 use constant FAIRSCHED_SET_RATE  => 0;
 130 use constant FAIRSCHED_DROP_RATE => 1;
 131 use constant FAIRSCHED_GET_RATE  => 2;
 132
 133 sub fairsched_cpulimit {
 134     my ($id, $limit) = @_;
 135
 136     my $cpulim1024 = int($limit * 1024 / 100);
 137     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 138
 139     return fairsched_rate($id, $op, $cpulim1024);
 140 }
 141
 142 my $nodename = PVE::INotify::nodename();
 143
 144 mkdir "/etc/pve/nodes/$nodename";
 145 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 146 mkdir $confdir;
 147
 148 my $var_run_tmpdir = "/var/run/qemu-server";
 149 mkdir $var_run_tmpdir;
 150
 151 my $lock_dir = "/var/lock/qemu-server";
 152 mkdir $lock_dir;
 153
 154 my $pcisysfs = "/sys/bus/pci";
 155
 156 my $confdesc = {
 157     onboot => {
 158         optional => 1,
 159         type => 'boolean',
 160         description => "Specifies whether a VM will be started during system bootup.",
 161         default => 0,
 162     },
 163     autostart => {
 164         optional => 1,
 165         type => 'boolean',
 166         description => "Automatic restart after crash (currently ignored).",
 167         default => 0,
 168     },
 169     hotplug => {
 170         optional => 1,
 171         type => 'string', format => 'pve-hotplug-features',
 172         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 173         default => 'network,disk,usb',
 174     },
 175     reboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 179         default => 1,
 180     },
 181     lock => {
 182         optional => 1,
 183         type => 'string',
 184         description => "Lock/unlock the VM.",
 185         enum => [qw(migrate backup snapshot rollback)],
 186     },
 187     cpulimit => {
 188         optional => 1,
 189         type => 'integer',
 190         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 191         minimum => 0,
 192         default => 0,
 193     },
 194     cpuunits => {
 195         optional => 1,
 196         type => 'integer',
 197         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 198         minimum => 0,
 199         maximum => 500000,
 200         default => 1000,
 201     },
 202     memory => {
 203         optional => 1,
 204         type => 'integer',
 205         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 206         minimum => 16,
 207         default => 512,
 208     },
 209     balloon => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 213         minimum => 0,
 214     },
 215     shares => {
 216         optional => 1,
 217         type => 'integer',
 218         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 219         minimum => 0,
 220         maximum => 50000,
 221         default => 1000,
 222     },
 223     keyboard => {
 224         optional => 1,
 225         type => 'string',
 226         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 227         enum => PVE::Tools::kvmkeymaplist(),
 228         default => 'en-us',
 229     },
 230     name => {
 231         optional => 1,
 232         type => 'string', format => 'dns-name',
 233         description => "Set a name for the VM. Only used on the configuration web interface.",
 234     },
 235     scsihw => {
 236         optional => 1,
 237         type => 'string',
 238         description => "scsi controller model",
 239         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 240         default => 'lsi',
 241     },
 242     description => {
 243         optional => 1,
 244         type => 'string',
 245         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 246     },
 247     ostype => {
 248         optional => 1,
 249         type => 'string',
 250         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 251         description => <<EODESC,
 252 Used to enable special optimization/features for specific
 253 operating systems:
 254
 255 other  => unspecified OS
 256 wxp    => Microsoft Windows XP
 257 w2k    => Microsoft Windows 2000
 258 w2k3   => Microsoft Windows 2003
 259 w2k8   => Microsoft Windows 2008
 260 wvista => Microsoft Windows Vista
 261 win7   => Microsoft Windows 7
 262 win8   => Microsoft Windows 8/2012
 263 l24    => Linux 2.4 Kernel
 264 l26    => Linux 2.6/3.X Kernel
 265 solaris => solaris/opensolaris/openindiania kernel
 266
 267 other|l24|l26|solaris                       ... no special behaviour
 268 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 269 EODESC
 270     },
 271     boot => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 275         pattern => '[acdn]{1,4}',
 276         default => 'cdn',
 277     },
 278     bootdisk => {
 279         optional => 1,
 280         type => 'string', format => 'pve-qm-bootdisk',
 281         description => "Enable booting from specified disk.",
 282         pattern => '(ide|sata|scsi|virtio)\d+',
 283     },
 284     smp => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "The number of CPUs. Please use option -sockets instead.",
 288         minimum => 1,
 289         default => 1,
 290     },
 291     sockets => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "The number of CPU sockets.",
 295         minimum => 1,
 296         default => 1,
 297     },
 298     cores => {
 299         optional => 1,
 300         type => 'integer',
 301         description => "The number of cores per socket.",
 302         minimum => 1,
 303         default => 1,
 304     },
 305     numa => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Enable/disable Numa.",
 309         default => 0,
 310     },
 311     vcpus => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "Number of hotplugged vcpus.",
 315         minimum => 1,
 316         default => 0,
 317     },
 318     acpi => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Enable/disable ACPI.",
 322         default => 1,
 323     },
 324     agent => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable Qemu GuestAgent.",
 328         default => 0,
 329     },
 330     kvm => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable KVM hardware virtualization.",
 334         default => 1,
 335     },
 336     tdf => {
 337         optional => 1,
 338         type => 'boolean',
 339         description => "Enable/disable time drift fix.",
 340         default => 0,
 341     },
 342     localtime => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 346     },
 347     freeze => {
 348         optional => 1,
 349         type => 'boolean',
 350         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 351     },
 352     vga => {
 353         optional => 1,
 354         type => 'string',
 355         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 356         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 357     },
 358     watchdog => {
 359         optional => 1,
 360         type => 'string', format => 'pve-qm-watchdog',
 361         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 362         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 363     },
 364     startdate => {
 365         optional => 1,
 366         type => 'string',
 367         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 368         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 369         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 370         default => 'now',
 371     },
 372     startup =>  get_standard_option('pve-startup-order'),
 373     template => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable Template.",
 377         default => 0,
 378     },
 379     args => {
 380         optional => 1,
 381         type => 'string',
 382         description => <<EODESCR,
 383 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 384
 385 args: -no-reboot -no-hpet
 386 EODESCR
 387     },
 388     tablet => {
 389         optional => 1,
 390         type => 'boolean',
 391         default => 1,
 392         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 393     },
 394     migrate_speed => {
 395         optional => 1,
 396         type => 'integer',
 397         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 398         minimum => 0,
 399         default => 0,
 400     },
 401     migrate_downtime => {
 402         optional => 1,
 403         type => 'number',
 404         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 405         minimum => 0,
 406         default => 0.1,
 407     },
 408     cdrom => {
 409         optional => 1,
 410         type => 'string', format => 'pve-qm-drive',
 411         typetext => 'volume',
 412         description => "This is an alias for option -ide2",
 413     },
 414     cpu => {
 415         optional => 1,
 416         description => "Emulated CPU type.",
 417         type => 'string',
 418         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 419         default => 'kvm64',
 420     },
 421     parent => get_standard_option('pve-snapshot-name', {
 422         optional => 1,
 423         description => "Parent snapshot name. This is used internally, and should not be modified.",
 424     }),
 425     snaptime => {
 426         optional => 1,
 427         description => "Timestamp for snapshots.",
 428         type => 'integer',
 429         minimum => 0,
 430     },
 431     vmstate => {
 432         optional => 1,
 433         type => 'string', format => 'pve-volume-id',
 434         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 435     },
 436     machine => {
 437         description => "Specific the Qemu machine type.",
 438         type => 'string',
 439         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 440         maxLength => 40,
 441         optional => 1,
 442     },
 443     smbios1 => {
 444         description => "Specify SMBIOS type 1 fields.",
 445         type => 'string', format => 'pve-qm-smbios1',
 446         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 447         maxLength => 256,
 448         optional => 1,
 449     },
 450 };
 451
 452 # what about other qemu settings ?
 453 #cpu => 'string',
 454 #machine => 'string',
 455 #fda => 'file',
 456 #fdb => 'file',
 457 #mtdblock => 'file',
 458 #sd => 'file',
 459 #pflash => 'file',
 460 #snapshot => 'bool',
 461 #bootp => 'file',
 462 ##tftp => 'dir',
 463 ##smb => 'dir',
 464 #kernel => 'file',
 465 #append => 'string',
 466 #initrd => 'file',
 467 ##soundhw => 'string',
 468
 469 while (my ($k, $v) = each %$confdesc) {
 470     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 471 }
 472
 473 my $MAX_IDE_DISKS = 4;
 474 my $MAX_SCSI_DISKS = 14;
 475 my $MAX_VIRTIO_DISKS = 16;
 476 my $MAX_SATA_DISKS = 6;
 477 my $MAX_USB_DEVICES = 5;
 478 my $MAX_NETS = 32;
 479 my $MAX_UNUSED_DISKS = 8;
 480 my $MAX_HOSTPCI_DEVICES = 4;
 481 my $MAX_SERIAL_PORTS = 4;
 482 my $MAX_PARALLEL_PORTS = 3;
 483 my $MAX_NUMA = 8;
 484 my $MAX_MEM = 4194304;
 485 my $STATICMEM = 1024;
 486
 487 my $numadesc = {
 488     optional => 1,
 489     type => 'string', format => 'pve-qm-numanode',
 490     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 491     description => "numa topology",
 492 };
 493 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 494
 495 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 496     $confdesc->{"numa$i"} = $numadesc;
 497 }
 498
 499 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 500                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 501                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 502 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 503
 504 my $netdesc = {
 505     optional => 1,
 506     type => 'string', format => 'pve-qm-net',
 507     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 508     description => <<EODESCR,
 509 Specify network devices.
 510
 511 MODEL is one of: $nic_model_list_txt
 512
 513 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 514 automatically generated if not specified.
 515
 516 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 517
 518 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 519
 520 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 521
 522 10.0.2.2   Gateway
 523 10.0.2.3   DNS Server
 524 10.0.2.4   SMB Server
 525
 526 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 527
 528 EODESCR
 529 };
 530 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 531
 532 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 533     $confdesc->{"net$i"} = $netdesc;
 534 }
 535
 536 my $drivename_hash;
 537
 538 my $idedesc = {
 539     optional => 1,
 540     type => 'string', format => 'pve-qm-drive',
 541     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 542     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 543 };
 544 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 545
 546 my $scsidesc = {
 547     optional => 1,
 548     type => 'string', format => 'pve-qm-drive',
 549     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 550     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 553
 554 my $satadesc = {
 555     optional => 1,
 556     type => 'string', format => 'pve-qm-drive',
 557     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 558     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 559 };
 560 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 561
 562 my $virtiodesc = {
 563     optional => 1,
 564     type => 'string', format => 'pve-qm-drive',
 565     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 566     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 567 };
 568 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 569
 570 my $usbdesc = {
 571     optional => 1,
 572     type => 'string', format => 'pve-qm-usb-device',
 573     typetext => 'host=HOSTUSBDEVICE|spice',
 574     description => <<EODESCR,
 575 Configure an USB device (n is 0 to 4). This can be used to
 576 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 577
 578 'bus-port(.port)*' (decimal numbers) or
 579 'vendor_id:product_id' (hexadeciaml numbers)
 580
 581 You can use the 'lsusb -t' command to list existing usb devices.
 582
 583 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 584
 585 The value 'spice' can be used to add a usb redirection devices for spice.
 586
 587 EODESCR
 588 };
 589 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 590
 591 my $hostpcidesc = {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-hostpci',
 594         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 595         description => <<EODESCR,
 596 Map host pci devices. HOSTPCIDEVICE syntax is:
 597
 598 'bus:dev.func' (hexadecimal numbers)
 599
 600 You can us the 'lspci' command to list existing pci devices.
 601
 602 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 603
 604 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 605
 606 Experimental: user reported problems with this option.
 607 EODESCR
 608 };
 609 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 610
 611 my $serialdesc = {
 612         optional => 1,
 613         type => 'string',
 614         pattern => '(/dev/.+|socket)',
 615         description =>  <<EODESCR,
 616 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 617
 618 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 619
 620 Experimental: user reported problems with this option.
 621 EODESCR
 622 };
 623
 624 my $paralleldesc= {
 625         optional => 1,
 626         type => 'string',
 627         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 628         description =>  <<EODESCR,
 629 Map host parallel devices (n is 0 to 2).
 630
 631 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 632
 633 Experimental: user reported problems with this option.
 634 EODESCR
 635 };
 636
 637 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 638     $confdesc->{"parallel$i"} = $paralleldesc;
 639 }
 640
 641 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 642     $confdesc->{"serial$i"} = $serialdesc;
 643 }
 644
 645 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 646     $confdesc->{"hostpci$i"} = $hostpcidesc;
 647 }
 648
 649 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 650     $drivename_hash->{"ide$i"} = 1;
 651     $confdesc->{"ide$i"} = $idedesc;
 652 }
 653
 654 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 655     $drivename_hash->{"sata$i"} = 1;
 656     $confdesc->{"sata$i"} = $satadesc;
 657 }
 658
 659 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 660     $drivename_hash->{"scsi$i"} = 1;
 661     $confdesc->{"scsi$i"} = $scsidesc ;
 662 }
 663
 664 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 665     $drivename_hash->{"virtio$i"} = 1;
 666     $confdesc->{"virtio$i"} = $virtiodesc;
 667 }
 668
 669 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 670     $confdesc->{"usb$i"} = $usbdesc;
 671 }
 672
 673 my $unuseddesc = {
 674     optional => 1,
 675     type => 'string', format => 'pve-volume-id',
 676     description => "Reference to unused volumes.",
 677 };
 678
 679 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 680     $confdesc->{"unused$i"} = $unuseddesc;
 681 }
 682
 683 my $kvm_api_version = 0;
 684
 685 sub kvm_version {
 686
 687     return $kvm_api_version if $kvm_api_version;
 688
 689     my $fh = IO::File->new("</dev/kvm") ||
 690         return 0;
 691
 692     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 693         $kvm_api_version = $v;
 694     }
 695
 696     $fh->close();
 697
 698     return  $kvm_api_version;
 699 }
 700
 701 my $kvm_user_version;
 702
 703 sub kvm_user_version {
 704
 705     return $kvm_user_version if $kvm_user_version;
 706
 707     $kvm_user_version = 'unknown';
 708
 709     my $tmp = `kvm -help 2>/dev/null`;
 710
 711     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 712         $kvm_user_version = $2;
 713     }
 714
 715     return $kvm_user_version;
 716
 717 }
 718
 719 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 720
 721 sub disknames {
 722     # order is important - used to autoselect boot disk
 723     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 724             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 725             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 726             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 727 }
 728
 729 sub valid_drivename {
 730     my $dev = shift;
 731
 732     return defined($drivename_hash->{$dev});
 733 }
 734
 735 sub option_exists {
 736     my $key = shift;
 737     return defined($confdesc->{$key});
 738 }
 739
 740 sub nic_models {
 741     return $nic_model_list;
 742 }
 743
 744 sub os_list_description {
 745
 746     return {
 747         other => 'Other',
 748         wxp => 'Windows XP',
 749         w2k => 'Windows 2000',
 750         w2k3 =>, 'Windows 2003',
 751         w2k8 => 'Windows 2008',
 752         wvista => 'Windows Vista',
 753         win7 => 'Windows 7',
 754         win8 => 'Windows 8/2012',
 755         l24 => 'Linux 2.4',
 756         l26 => 'Linux 2.6',
 757     };
 758 }
 759
 760 my $cdrom_path;
 761
 762 sub get_cdrom_path {
 763
 764     return  $cdrom_path if $cdrom_path;
 765
 766     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 767     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 768     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 769 }
 770
 771 sub get_iso_path {
 772     my ($storecfg, $vmid, $cdrom) = @_;
 773
 774     if ($cdrom eq 'cdrom') {
 775         return get_cdrom_path();
 776     } elsif ($cdrom eq 'none') {
 777         return '';
 778     } elsif ($cdrom =~ m|^/|) {
 779         return $cdrom;
 780     } else {
 781         return PVE::Storage::path($storecfg, $cdrom);
 782     }
 783 }
 784
 785 # try to convert old style file names to volume IDs
 786 sub filename_to_volume_id {
 787     my ($vmid, $file, $media) = @_;
 788
 789     if (!($file eq 'none' || $file eq 'cdrom' ||
 790           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 791
 792         return undef if $file =~ m|/|;
 793
 794         if ($media && $media eq 'cdrom') {
 795             $file = "local:iso/$file";
 796         } else {
 797             $file = "local:$vmid/$file";
 798         }
 799     }
 800
 801     return $file;
 802 }
 803
 804 sub verify_media_type {
 805     my ($opt, $vtype, $media) = @_;
 806
 807     return if !$media;
 808
 809     my $etype;
 810     if ($media eq 'disk') {
 811         $etype = 'images';
 812     } elsif ($media eq 'cdrom') {
 813         $etype = 'iso';
 814     } else {
 815         die "internal error";
 816     }
 817
 818     return if ($vtype eq $etype);
 819
 820     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 821 }
 822
 823 sub cleanup_drive_path {
 824     my ($opt, $storecfg, $drive) = @_;
 825
 826     # try to convert filesystem paths to volume IDs
 827
 828     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 829         ($drive->{file} !~ m|^/dev/.+|) &&
 830         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 831         ($drive->{file} !~ m/^\d+$/)) {
 832         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 833         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 834         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 835         verify_media_type($opt, $vtype, $drive->{media});
 836         $drive->{file} = $volid;
 837     }
 838
 839     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 840 }
 841
 842 sub create_conf_nolock {
 843     my ($vmid, $settings) = @_;
 844
 845     my $filename = config_file($vmid);
 846
 847     die "configuration file '$filename' already exists\n" if -f $filename;
 848
 849     my $defaults = load_defaults();
 850
 851     $settings->{name} = "vm$vmid" if !$settings->{name};
 852     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 853
 854     my $data = '';
 855     foreach my $opt (keys %$settings) {
 856         next if !$confdesc->{$opt};
 857
 858         my $value = $settings->{$opt};
 859         next if !$value;
 860
 861         $data .= "$opt: $value\n";
 862     }
 863
 864     PVE::Tools::file_set_contents($filename, $data);
 865 }
 866
 867 sub parse_hotplug_features {
 868     my ($data) = @_;
 869
 870     my $res = {};
 871
 872     return $res if $data eq '0';
 873
 874     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 875
 876     foreach my $feature (PVE::Tools::split_list($data)) {
 877         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 878             $res->{$1} = 1;
 879         } else {
 880             warn "ignoring unknown hotplug feature '$feature'\n";
 881         }
 882     }
 883     return $res;
 884 }
 885
 886 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 887 sub pve_verify_hotplug_features {
 888     my ($value, $noerr) = @_;
 889
 890     return $value if parse_hotplug_features($value);
 891
 892     return undef if $noerr;
 893
 894     die "unable to parse hotplug option\n";
 895 }
 896
 897 my $parse_size = sub {
 898     my ($value) = @_;
 899
 900     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 901     my ($size, $unit) = ($1, $3);
 902     if ($unit) {
 903         if ($unit eq 'K') {
 904             $size = $size * 1024;
 905         } elsif ($unit eq 'M') {
 906             $size = $size * 1024 * 1024;
 907         } elsif ($unit eq 'G') {
 908             $size = $size * 1024 * 1024 * 1024;
 909         }
 910     }
 911     return int($size);
 912 };
 913
 914 my $format_size = sub {
 915     my ($size) = @_;
 916
 917     $size = int($size);
 918
 919     my $kb = int($size/1024);
 920     return $size if $kb*1024 != $size;
 921
 922     my $mb = int($kb/1024);
 923     return "${kb}K" if $mb*1024 != $kb;
 924
 925     my $gb = int($mb/1024);
 926     return "${mb}M" if $gb*1024 != $mb;
 927
 928     return "${gb}G";
 929 };
 930
 931 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 932 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 933 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 934 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 935
 936 sub parse_drive {
 937     my ($key, $data) = @_;
 938
 939     my $res = {};
 940
 941     # $key may be undefined - used to verify JSON parameters
 942     if (!defined($key)) {
 943         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 944         $res->{index} = 0;
 945     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 946         $res->{interface} = $1;
 947         $res->{index} = $2;
 948     } else {
 949         return undef;
 950     }
 951
 952     foreach my $p (split (/,/, $data)) {
 953         next if $p =~ m/^\s*$/;
 954
 955         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 956             my ($k, $v) = ($1, $2);
 957
 958             $k = 'file' if $k eq 'volume';
 959
 960             return undef if defined $res->{$k};
 961
 962             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 963                 return undef if !$v || $v !~ m/^\d+/;
 964                 $k = "m$k";
 965                 $v = sprintf("%.3f", $v / (1024*1024));
 966             }
 967             $res->{$k} = $v;
 968         } else {
 969             if (!$res->{file} && $p !~ m/=/) {
 970                 $res->{file} = $p;
 971             } else {
 972                 return undef;
 973             }
 974         }
 975     }
 976
 977     return undef if !$res->{file};
 978
 979     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 980         $res->{format} = $1;
 981     }
 982
 983     return undef if $res->{cache} &&
 984         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 985     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 986     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 987     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 988     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 989     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 990     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 991     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 992     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 993     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 994     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 995     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 996     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 997     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 998     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 999
1000     return undef if $res->{mbps_rd} && $res->{mbps};
1001     return undef if $res->{mbps_wr} && $res->{mbps};
1002
1003     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1004     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1005     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1006     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1007     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1008     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1009
1010     return undef if $res->{iops_rd} && $res->{iops};
1011     return undef if $res->{iops_wr} && $res->{iops};
1012
1013
1014     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1015     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1016     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1017     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1018     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1019     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1020
1021
1022     if ($res->{size}) {
1023         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1024     }
1025
1026     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1027         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1028         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1029         return undef if $res->{interface} eq 'virtio';
1030     }
1031
1032     # rerror does not work with scsi drives
1033     if ($res->{rerror}) {
1034         return undef if $res->{interface} eq 'scsi';
1035     }
1036
1037     return $res;
1038 }
1039
1040 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1041
1042 sub print_drive {
1043     my ($vmid, $drive) = @_;
1044
1045     my $opts = '';
1046     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
1047         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1048     }
1049
1050     if ($drive->{size}) {
1051         $opts .= ",size=" . &$format_size($drive->{size});
1052     }
1053
1054     return "$drive->{file}$opts";
1055 }
1056
1057 sub scsi_inquiry {
1058     my($fh, $noerr) = @_;
1059
1060     my $SG_IO = 0x2285;
1061     my $SG_GET_VERSION_NUM = 0x2282;
1062
1063     my $versionbuf = "\x00" x 8;
1064     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1065     if (!$ret) {
1066         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1067         return undef;
1068     }
1069     my $version = unpack("I", $versionbuf);
1070     if ($version < 30000) {
1071         die "scsi generic interface too old\n"  if !$noerr;
1072         return undef;
1073     }
1074
1075     my $buf = "\x00" x 36;
1076     my $sensebuf = "\x00" x 8;
1077     my $cmd = pack("C x3 C x1", 0x12, 36);
1078
1079     # see /usr/include/scsi/sg.h
1080     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1081
1082     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1083                       length($sensebuf), 0, length($buf), $buf,
1084                       $cmd, $sensebuf, 6000);
1085
1086     $ret = ioctl($fh, $SG_IO, $packet);
1087     if (!$ret) {
1088         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1089         return undef;
1090     }
1091
1092     my @res = unpack($sg_io_hdr_t, $packet);
1093     if ($res[17] || $res[18]) {
1094         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1095         return undef;
1096     }
1097
1098     my $res = {};
1099     (my $byte0, my $byte1, $res->{vendor},
1100      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1101
1102     $res->{removable} = $byte1 & 128 ? 1 : 0;
1103     $res->{type} = $byte0 & 31;
1104
1105     return $res;
1106 }
1107
1108 sub path_is_scsi {
1109     my ($path) = @_;
1110
1111     my $fh = IO::File->new("+<$path") || return undef;
1112     my $res = scsi_inquiry($fh, 1);
1113     close($fh);
1114
1115     return $res;
1116 }
1117
1118 sub machine_type_is_q35 {
1119     my ($conf) = @_;
1120
1121     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1122 }
1123
1124 sub print_tabletdevice_full {
1125     my ($conf) = @_;
1126
1127     my $q35 = machine_type_is_q35($conf);
1128
1129     # we use uhci for old VMs because tablet driver was buggy in older qemu
1130     my $usbbus = $q35 ? "ehci" : "uhci";
1131
1132     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1133 }
1134
1135 sub print_drivedevice_full {
1136     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1137
1138     my $device = '';
1139     my $maxdev = 0;
1140
1141     if ($drive->{interface} eq 'virtio') {
1142         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1143         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1144         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1145     } elsif ($drive->{interface} eq 'scsi') {
1146
1147         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1148         my $unit = $drive->{index} % $maxdev;
1149         my $devicetype = 'hd';
1150         my $path = '';
1151         if (drive_is_cdrom($drive)) {
1152               $devicetype = 'cd';
1153         } else {
1154               if ($drive->{file} =~ m|^/|) {
1155                   $path = $drive->{file};
1156               } else {
1157                   $path = PVE::Storage::path($storecfg, $drive->{file});
1158               }
1159
1160               if($path =~ m/^iscsi\:\/\//){
1161                   $devicetype = 'generic';
1162               } else {
1163                   if (my $info = path_is_scsi($path)) {
1164                       if ($info->{type} == 0) {
1165                           $devicetype = 'block';
1166                       } elsif ($info->{type} == 1) { # tape
1167                           $devicetype = 'generic';
1168                       }
1169                   }
1170               }
1171          }
1172
1173         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1174             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1175         } else {
1176             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1177         }
1178
1179     } elsif ($drive->{interface} eq 'ide'){
1180         $maxdev = 2;
1181         my $controller = int($drive->{index} / $maxdev);
1182         my $unit = $drive->{index} % $maxdev;
1183         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1184
1185         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1186     } elsif ($drive->{interface} eq 'sata'){
1187         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1188         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1189         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1190     } elsif ($drive->{interface} eq 'usb') {
1191         die "implement me";
1192         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1193     } else {
1194         die "unsupported interface type";
1195     }
1196
1197     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1198
1199     return $device;
1200 }
1201
1202 sub get_initiator_name {
1203     my $initiator;
1204
1205     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1206     while (defined(my $line = <$fh>)) {
1207         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1208         $initiator = $1;
1209         last;
1210     }
1211     $fh->close();
1212
1213     return $initiator;
1214 }
1215
1216 sub print_drive_full {
1217     my ($storecfg, $vmid, $drive) = @_;
1218
1219     my $opts = '';
1220     foreach my $o (@qemu_drive_options) {
1221         next if $o eq 'bootindex';
1222         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1223     }
1224
1225     foreach my $o (qw(bps bps_rd bps_wr)) {
1226         my $v = $drive->{"m$o"};
1227         $opts .= ",$o=" . int($v*1024*1024) if $v;
1228     }
1229
1230     # aio native works only with O_DIRECT
1231     if (!$drive->{aio}) {
1232         if(!$drive->{cache} || $drive->{cache} eq 'none' || $drive->{cache} eq 'directsync') {
1233             $opts .= ",aio=native";
1234         } else {
1235             $opts .= ",aio=threads";
1236         }
1237     }
1238
1239
1240     my $path;
1241     my $volid = $drive->{file};
1242     if (drive_is_cdrom($drive)) {
1243         $path = get_iso_path($storecfg, $vmid, $volid);
1244     } else {
1245         if ($volid =~ m|^/|) {
1246             $path = $volid;
1247         } else {
1248             $path = PVE::Storage::path($storecfg, $volid);
1249         }
1250     }
1251
1252     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1253
1254     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1255     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1256
1257     my $pathinfo = $path ? "file=$path," : '';
1258
1259     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1260 }
1261
1262 sub print_netdevice_full {
1263     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1264
1265     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1266
1267     my $device = $net->{model};
1268     if ($net->{model} eq 'virtio') {
1269          $device = 'virtio-net-pci';
1270      };
1271
1272     my $pciaddr = print_pci_addr("$netid", $bridges);
1273     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1274     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1275         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1276         my $vectors = $net->{queues} * 2 + 2;
1277         $tmpstr .= ",vectors=$vectors,mq=on";
1278     }
1279     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1280     return $tmpstr;
1281 }
1282
1283 sub print_netdev_full {
1284     my ($vmid, $conf, $net, $netid) = @_;
1285
1286     my $i = '';
1287     if ($netid =~ m/^net(\d+)$/) {
1288         $i = int($1);
1289     }
1290
1291     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1292
1293     my $ifname = "tap${vmid}i$i";
1294
1295     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1296     die "interface name '$ifname' is too long (max 15 character)\n"
1297         if length($ifname) >= 16;
1298
1299     my $vhostparam = '';
1300     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1301
1302     my $vmname = $conf->{name} || "vm$vmid";
1303
1304     my $netdev = "";
1305
1306     if ($net->{bridge}) {
1307         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1308     } else {
1309         $netdev = "type=user,id=$netid,hostname=$vmname";
1310     }
1311
1312     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1313
1314     return $netdev;
1315 }
1316
1317 sub drive_is_cdrom {
1318     my ($drive) = @_;
1319
1320     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1321
1322 }
1323
1324 sub parse_numa {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     foreach my $kvp (split(/,/, $data)) {
1330
1331         if ($kvp =~ m/^memory=(\S+)$/) {
1332             $res->{memory} = $1;
1333         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1334             $res->{policy} = $1;
1335         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1336             $res->{cpus}->{start} = $1;
1337             $res->{cpus}->{end} = $3;
1338         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1339             $res->{hostnodes}->{start} = $1;
1340             $res->{hostnodes}->{end} = $3;
1341         } else {
1342             return undef;
1343         }
1344     }
1345
1346     return $res;
1347 }
1348
1349 sub parse_hostpci {
1350     my ($value) = @_;
1351
1352     return undef if !$value;
1353
1354
1355     my @list = split(/,/, $value);
1356     my $found;
1357
1358     my $res = {};
1359     foreach my $kv (@list) {
1360
1361         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1362             $found = 1;
1363             if(defined($4)){
1364                 push @{$res->{pciid}}, { id => $2 , function => $4};
1365
1366             }else{
1367                 my $pcidevices = lspci($2);
1368                 $res->{pciid} = $pcidevices->{$2};
1369             }
1370         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1371             $res->{driver} = $1;
1372         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1373             $res->{rombar} = $1;
1374         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1375             $res->{'x-vga'} = $1;
1376         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1377             $res->{pcie} = 1 if $1 == 1;
1378         } else {
1379             warn "unknown hostpci setting '$kv'\n";
1380         }
1381     }
1382
1383     return undef if !$found;
1384
1385     return $res;
1386 }
1387
1388 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1389 sub parse_net {
1390     my ($data) = @_;
1391
1392     my $res = {};
1393
1394     foreach my $kvp (split(/,/, $data)) {
1395
1396         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1397             my $model = lc($1);
1398             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1399             $res->{model} = $model;
1400             $res->{macaddr} = $mac;
1401         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1402             $res->{bridge} = $1;
1403         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1404             $res->{queues} = $1;
1405         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1406             $res->{rate} = $1;
1407         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1408             $res->{tag} = $1;
1409         } elsif ($kvp =~ m/^firewall=([01])$/) {
1410             $res->{firewall} = $1;
1411         } elsif ($kvp =~ m/^link_down=([01])$/) {
1412             $res->{link_down} = $1;
1413         } else {
1414             return undef;
1415         }
1416
1417     }
1418
1419     return undef if !$res->{model};
1420
1421     return $res;
1422 }
1423
1424 sub print_net {
1425     my $net = shift;
1426
1427     my $res = "$net->{model}";
1428     $res .= "=$net->{macaddr}" if $net->{macaddr};
1429     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1430     $res .= ",rate=$net->{rate}" if $net->{rate};
1431     $res .= ",tag=$net->{tag}" if $net->{tag};
1432     $res .= ",firewall=1" if $net->{firewall};
1433     $res .= ",link_down=1" if $net->{link_down};
1434     $res .= ",queues=$net->{queues}" if $net->{queues};
1435
1436     return $res;
1437 }
1438
1439 sub add_random_macs {
1440     my ($settings) = @_;
1441
1442     foreach my $opt (keys %$settings) {
1443         next if $opt !~ m/^net(\d+)$/;
1444         my $net = parse_net($settings->{$opt});
1445         next if !$net;
1446         $settings->{$opt} = print_net($net);
1447     }
1448 }
1449
1450 sub add_unused_volume {
1451     my ($config, $volid) = @_;
1452
1453     my $key;
1454     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1455         my $test = "unused$ind";
1456         if (my $vid = $config->{$test}) {
1457             return if $vid eq $volid; # do not add duplicates
1458         } else {
1459             $key = $test;
1460         }
1461     }
1462
1463     die "To many unused volume - please delete them first.\n" if !$key;
1464
1465     $config->{$key} = $volid;
1466
1467     return $key;
1468 }
1469
1470 sub vm_is_volid_owner {
1471     my ($storecfg, $vmid, $volid) = @_;
1472
1473     if ($volid !~  m|^/|) {
1474         my ($path, $owner);
1475         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1476         if ($owner && ($owner == $vmid)) {
1477             return 1;
1478         }
1479     }
1480
1481     return undef;
1482 }
1483
1484 sub vmconfig_delete_pending_option {
1485     my ($conf, $key) = @_;
1486
1487     delete $conf->{pending}->{$key};
1488     my $pending_delete_hash = { $key => 1 };
1489     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1490         $pending_delete_hash->{$opt} = 1;
1491     }
1492     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1493 }
1494
1495 sub vmconfig_undelete_pending_option {
1496     my ($conf, $key) = @_;
1497
1498     my $pending_delete_hash = {};
1499     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1500         $pending_delete_hash->{$opt} = 1;
1501     }
1502     delete $pending_delete_hash->{$key};
1503
1504     my @keylist = keys %$pending_delete_hash;
1505     if (scalar(@keylist)) {
1506         $conf->{pending}->{delete} = join(',', @keylist);
1507     } else {
1508         delete $conf->{pending}->{delete};
1509     }
1510 }
1511
1512 sub vmconfig_register_unused_drive {
1513     my ($storecfg, $vmid, $conf, $drive) = @_;
1514
1515     if (!drive_is_cdrom($drive)) {
1516         my $volid = $drive->{file};
1517         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1518             add_unused_volume($conf, $volid, $vmid);
1519         }
1520     }
1521 }
1522
1523 sub vmconfig_cleanup_pending {
1524     my ($conf) = @_;
1525
1526     # remove pending changes when nothing changed
1527     my $changes;
1528     foreach my $opt (keys %{$conf->{pending}}) {
1529         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1530             $changes = 1;
1531             delete $conf->{pending}->{$opt};
1532         }
1533     }
1534
1535     # remove delete if option is not set
1536     my $pending_delete_hash = {};
1537     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1538         if (defined($conf->{$opt})) {
1539             $pending_delete_hash->{$opt} = 1;
1540         } else {
1541             $changes = 1;
1542         }
1543     }
1544
1545     my @keylist = keys %$pending_delete_hash;
1546     if (scalar(@keylist)) {
1547         $conf->{pending}->{delete} = join(',', @keylist);
1548     } else {
1549         delete $conf->{pending}->{delete};
1550     }
1551
1552     return $changes;
1553 }
1554
1555 my $valid_smbios1_options = {
1556     manufacturer => '\S+',
1557     product => '\S+',
1558     version => '\S+',
1559     serial => '\S+',
1560     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1561     sku => '\S+',
1562     family => '\S+',
1563 };
1564
1565 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1566 sub parse_smbios1 {
1567     my ($data) = @_;
1568
1569     my $res = {};
1570
1571     foreach my $kvp (split(/,/, $data)) {
1572         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1573         my ($k, $v) = split(/=/, $kvp);
1574         return undef if !defined($k) || !defined($v);
1575         return undef if !$valid_smbios1_options->{$k};
1576         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1577         $res->{$k} = $v;
1578     }
1579
1580     return $res;
1581 }
1582
1583 sub print_smbios1 {
1584     my ($smbios1) = @_;
1585
1586     my $data = '';
1587     foreach my $k (keys %$smbios1) {
1588         next if !defined($smbios1->{$k});
1589         next if !$valid_smbios1_options->{$k};
1590         $data .= ',' if $data;
1591         $data .= "$k=$smbios1->{$k}";
1592     }
1593     return $data;
1594 }
1595
1596 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1597 sub verify_smbios1 {
1598     my ($value, $noerr) = @_;
1599
1600     return $value if parse_smbios1($value);
1601
1602     return undef if $noerr;
1603
1604     die "unable to parse smbios (type 1) options\n";
1605 }
1606
1607 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1608 sub verify_bootdisk {
1609     my ($value, $noerr) = @_;
1610
1611     return $value if valid_drivename($value);
1612
1613     return undef if $noerr;
1614
1615     die "invalid boot disk '$value'\n";
1616 }
1617
1618 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1619 sub verify_numa {
1620     my ($value, $noerr) = @_;
1621
1622     return $value if parse_numa($value);
1623
1624     return undef if $noerr;
1625
1626     die "unable to parse numa options\n";
1627 }
1628
1629 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1630 sub verify_net {
1631     my ($value, $noerr) = @_;
1632
1633     return $value if parse_net($value);
1634
1635     return undef if $noerr;
1636
1637     die "unable to parse network options\n";
1638 }
1639
1640 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1641 sub verify_drive {
1642     my ($value, $noerr) = @_;
1643
1644     return $value if parse_drive(undef, $value);
1645
1646     return undef if $noerr;
1647
1648     die "unable to parse drive options\n";
1649 }
1650
1651 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1652 sub verify_hostpci {
1653     my ($value, $noerr) = @_;
1654
1655     return $value if parse_hostpci($value);
1656
1657     return undef if $noerr;
1658
1659     die "unable to parse pci id\n";
1660 }
1661
1662 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1663 sub verify_watchdog {
1664     my ($value, $noerr) = @_;
1665
1666     return $value if parse_watchdog($value);
1667
1668     return undef if $noerr;
1669
1670     die "unable to parse watchdog options\n";
1671 }
1672
1673 sub parse_watchdog {
1674     my ($value) = @_;
1675
1676     return undef if !$value;
1677
1678     my $res = {};
1679
1680     foreach my $p (split(/,/, $value)) {
1681         next if $p =~ m/^\s*$/;
1682
1683         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1684             $res->{model} = $2;
1685         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1686             $res->{action} = $2;
1687         } else {
1688             return undef;
1689         }
1690     }
1691
1692     return $res;
1693 }
1694
1695 sub parse_usb_device {
1696     my ($value) = @_;
1697
1698     return undef if !$value;
1699
1700     my @dl = split(/,/, $value);
1701     my $found;
1702
1703     my $res = {};
1704     foreach my $v (@dl) {
1705         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1706             $found = 1;
1707             $res->{vendorid} = $2;
1708             $res->{productid} = $4;
1709         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1710             $found = 1;
1711             $res->{hostbus} = $1;
1712             $res->{hostport} = $2;
1713         } elsif ($v =~ m/^spice$/) {
1714             $found = 1;
1715             $res->{spice} = 1;
1716         } else {
1717             return undef;
1718         }
1719     }
1720     return undef if !$found;
1721
1722     return $res;
1723 }
1724
1725 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1726 sub verify_usb_device {
1727     my ($value, $noerr) = @_;
1728
1729     return $value if parse_usb_device($value);
1730
1731     return undef if $noerr;
1732
1733     die "unable to parse usb device\n";
1734 }
1735
1736 # add JSON properties for create and set function
1737 sub json_config_properties {
1738     my $prop = shift;
1739
1740     foreach my $opt (keys %$confdesc) {
1741         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1742         $prop->{$opt} = $confdesc->{$opt};
1743     }
1744
1745     return $prop;
1746 }
1747
1748 sub check_type {
1749     my ($key, $value) = @_;
1750
1751     die "unknown setting '$key'\n" if !$confdesc->{$key};
1752
1753     my $type = $confdesc->{$key}->{type};
1754
1755     if (!defined($value)) {
1756         die "got undefined value\n";
1757     }
1758
1759     if ($value =~ m/[\n\r]/) {
1760         die "property contains a line feed\n";
1761     }
1762
1763     if ($type eq 'boolean') {
1764         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1765         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1766         die "type check ('boolean') failed - got '$value'\n";
1767     } elsif ($type eq 'integer') {
1768         return int($1) if $value =~ m/^(\d+)$/;
1769         die "type check ('integer') failed - got '$value'\n";
1770     } elsif ($type eq 'number') {
1771         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1772         die "type check ('number') failed - got '$value'\n";
1773     } elsif ($type eq 'string') {
1774         if (my $fmt = $confdesc->{$key}->{format}) {
1775             if ($fmt eq 'pve-qm-drive') {
1776                 # special case - we need to pass $key to parse_drive()
1777                 my $drive = parse_drive($key, $value);
1778                 return $value if $drive;
1779                 die "unable to parse drive options\n";
1780             }
1781             PVE::JSONSchema::check_format($fmt, $value);
1782             return $value;
1783         }
1784         $value =~ s/^\"(.*)\"$/$1/;
1785         return $value;
1786     } else {
1787         die "internal error"
1788     }
1789 }
1790
1791 sub lock_config_full {
1792     my ($vmid, $timeout, $code, @param) = @_;
1793
1794     my $filename = config_file_lock($vmid);
1795
1796     my $res = lock_file($filename, $timeout, $code, @param);
1797
1798     die $@ if $@;
1799
1800     return $res;
1801 }
1802
1803 sub lock_config_mode {
1804     my ($vmid, $timeout, $shared, $code, @param) = @_;
1805
1806     my $filename = config_file_lock($vmid);
1807
1808     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1809
1810     die $@ if $@;
1811
1812     return $res;
1813 }
1814
1815 sub lock_config {
1816     my ($vmid, $code, @param) = @_;
1817
1818     return lock_config_full($vmid, 10, $code, @param);
1819 }
1820
1821 sub cfs_config_path {
1822     my ($vmid, $node) = @_;
1823
1824     $node = $nodename if !$node;
1825     return "nodes/$node/qemu-server/$vmid.conf";
1826 }
1827
1828 sub check_iommu_support{
1829     #fixme : need to check IOMMU support
1830     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1831
1832     my $iommu=1;
1833     return $iommu;
1834
1835 }
1836
1837 sub config_file {
1838     my ($vmid, $node) = @_;
1839
1840     my $cfspath = cfs_config_path($vmid, $node);
1841     return "/etc/pve/$cfspath";
1842 }
1843
1844 sub config_file_lock {
1845     my ($vmid) = @_;
1846
1847     return "$lock_dir/lock-$vmid.conf";
1848 }
1849
1850 sub touch_config {
1851     my ($vmid) = @_;
1852
1853     my $conf = config_file($vmid);
1854     utime undef, undef, $conf;
1855 }
1856
1857 sub destroy_vm {
1858     my ($storecfg, $vmid, $keep_empty_config) = @_;
1859
1860     my $conffile = config_file($vmid);
1861
1862     my $conf = load_config($vmid);
1863
1864     check_lock($conf);
1865
1866     # only remove disks owned by this VM
1867     foreach_drive($conf, sub {
1868         my ($ds, $drive) = @_;
1869
1870         return if drive_is_cdrom($drive);
1871
1872         my $volid = $drive->{file};
1873
1874         return if !$volid || $volid =~ m|^/|;
1875
1876         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1877         return if !$path || !$owner || ($owner != $vmid);
1878
1879         PVE::Storage::vdisk_free($storecfg, $volid);
1880     });
1881
1882     if ($keep_empty_config) {
1883         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1884     } else {
1885         unlink $conffile;
1886     }
1887
1888     # also remove unused disk
1889     eval {
1890         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1891
1892         eval {
1893             PVE::Storage::foreach_volid($dl, sub {
1894                 my ($volid, $sid, $volname, $d) = @_;
1895                 PVE::Storage::vdisk_free($storecfg, $volid);
1896             });
1897         };
1898         warn $@ if $@;
1899
1900     };
1901     warn $@ if $@;
1902 }
1903
1904 sub load_config {
1905     my ($vmid, $node) = @_;
1906
1907     my $cfspath = cfs_config_path($vmid, $node);
1908
1909     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1910
1911     die "no such VM ('$vmid')\n" if !defined($conf);
1912
1913     return $conf;
1914 }
1915
1916 sub parse_vm_config {
1917     my ($filename, $raw) = @_;
1918
1919     return undef if !defined($raw);
1920
1921     my $res = {
1922         digest => Digest::SHA::sha1_hex($raw),
1923         snapshots => {},
1924         pending => {},
1925     };
1926
1927     $filename =~ m|/qemu-server/(\d+)\.conf$|
1928         || die "got strange filename '$filename'";
1929
1930     my $vmid = $1;
1931
1932     my $conf = $res;
1933     my $descr = '';
1934     my $section = '';
1935
1936     my @lines = split(/\n/, $raw);
1937     foreach my $line (@lines) {
1938         next if $line =~ m/^\s*$/;
1939
1940         if ($line =~ m/^\[PENDING\]\s*$/i) {
1941             $section = 'pending';
1942             $conf->{description} = $descr if $descr;
1943             $descr = '';
1944             $conf = $res->{$section} = {};
1945             next;
1946
1947         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1948             $section = $1;
1949             $conf->{description} = $descr if $descr;
1950             $descr = '';
1951             $conf = $res->{snapshots}->{$section} = {};
1952             next;
1953         }
1954
1955         if ($line =~ m/^\#(.*)\s*$/) {
1956             $descr .= PVE::Tools::decode_text($1) . "\n";
1957             next;
1958         }
1959
1960         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1961             $descr .= PVE::Tools::decode_text($2);
1962         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1963             $conf->{snapstate} = $1;
1964         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1965             my $key = $1;
1966             my $value = $2;
1967             $conf->{$key} = $value;
1968         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1969             my $value = $1;
1970             if ($section eq 'pending') {
1971                 $conf->{delete} = $value; # we parse this later
1972             } else {
1973                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1974             }
1975         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1976             my $key = $1;
1977             my $value = $2;
1978             eval { $value = check_type($key, $value); };
1979             if ($@) {
1980                 warn "vm $vmid - unable to parse value of '$key' - $@";
1981             } else {
1982                 my $fmt = $confdesc->{$key}->{format};
1983                 if ($fmt && $fmt eq 'pve-qm-drive') {
1984                     my $v = parse_drive($key, $value);
1985                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1986                         $v->{file} = $volid;
1987                         $value = print_drive($vmid, $v);
1988                     } else {
1989                         warn "vm $vmid - unable to parse value of '$key'\n";
1990                         next;
1991                     }
1992                 }
1993
1994                 if ($key eq 'cdrom') {
1995                     $conf->{ide2} = $value;
1996                 } else {
1997                     $conf->{$key} = $value;
1998                 }
1999             }
2000         }
2001     }
2002
2003     $conf->{description} = $descr if $descr;
2004
2005     delete $res->{snapstate}; # just to be sure
2006
2007     return $res;
2008 }
2009
2010 sub write_vm_config {
2011     my ($filename, $conf) = @_;
2012
2013     delete $conf->{snapstate}; # just to be sure
2014
2015     if ($conf->{cdrom}) {
2016         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2017         $conf->{ide2} = $conf->{cdrom};
2018         delete $conf->{cdrom};
2019     }
2020
2021     # we do not use 'smp' any longer
2022     if ($conf->{sockets}) {
2023         delete $conf->{smp};
2024     } elsif ($conf->{smp}) {
2025         $conf->{sockets} = $conf->{smp};
2026         delete $conf->{cores};
2027         delete $conf->{smp};
2028     }
2029
2030     my $used_volids = {};
2031
2032     my $cleanup_config = sub {
2033         my ($cref, $pending, $snapname) = @_;
2034
2035         foreach my $key (keys %$cref) {
2036             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2037                 $key eq 'snapstate' || $key eq 'pending';
2038             my $value = $cref->{$key};
2039             if ($key eq 'delete') {
2040                 die "propertry 'delete' is only allowed in [PENDING]\n"
2041                     if !$pending;
2042                 # fixme: check syntax?
2043                 next;
2044             }
2045             eval { $value = check_type($key, $value); };
2046             die "unable to parse value of '$key' - $@" if $@;
2047
2048             $cref->{$key} = $value;
2049
2050             if (!$snapname && valid_drivename($key)) {
2051                 my $drive = parse_drive($key, $value);
2052                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2053             }
2054         }
2055     };
2056
2057     &$cleanup_config($conf);
2058
2059     &$cleanup_config($conf->{pending}, 1);
2060
2061     foreach my $snapname (keys %{$conf->{snapshots}}) {
2062         die "internal error" if $snapname eq 'pending';
2063         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2064     }
2065
2066     # remove 'unusedX' settings if we re-add a volume
2067     foreach my $key (keys %$conf) {
2068         my $value = $conf->{$key};
2069         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2070             delete $conf->{$key};
2071         }
2072     }
2073
2074     my $generate_raw_config = sub {
2075         my ($conf) = @_;
2076
2077         my $raw = '';
2078
2079         # add description as comment to top of file
2080         my $descr = $conf->{description} || '';
2081         foreach my $cl (split(/\n/, $descr)) {
2082             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2083         }
2084
2085         foreach my $key (sort keys %$conf) {
2086             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2087             $raw .= "$key: $conf->{$key}\n";
2088         }
2089         return $raw;
2090     };
2091
2092     my $raw = &$generate_raw_config($conf);
2093
2094     if (scalar(keys %{$conf->{pending}})){
2095         $raw .= "\n[PENDING]\n";
2096         $raw .= &$generate_raw_config($conf->{pending});
2097     }
2098
2099     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2100         $raw .= "\n[$snapname]\n";
2101         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2102     }
2103
2104     return $raw;
2105 }
2106
2107 sub update_config_nolock {
2108     my ($vmid, $conf, $skiplock) = @_;
2109
2110     check_lock($conf) if !$skiplock;
2111
2112     my $cfspath = cfs_config_path($vmid);
2113
2114     PVE::Cluster::cfs_write_file($cfspath, $conf);
2115 }
2116
2117 sub update_config {
2118     my ($vmid, $conf, $skiplock) = @_;
2119
2120     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2121 }
2122
2123 sub load_defaults {
2124
2125     my $res = {};
2126
2127     # we use static defaults from our JSON schema configuration
2128     foreach my $key (keys %$confdesc) {
2129         if (defined(my $default = $confdesc->{$key}->{default})) {
2130             $res->{$key} = $default;
2131         }
2132     }
2133
2134     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2135     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2136
2137     return $res;
2138 }
2139
2140 sub config_list {
2141     my $vmlist = PVE::Cluster::get_vmlist();
2142     my $res = {};
2143     return $res if !$vmlist || !$vmlist->{ids};
2144     my $ids = $vmlist->{ids};
2145
2146     foreach my $vmid (keys %$ids) {
2147         my $d = $ids->{$vmid};
2148         next if !$d->{node} || $d->{node} ne $nodename;
2149         next if !$d->{type} || $d->{type} ne 'qemu';
2150         $res->{$vmid}->{exists} = 1;
2151     }
2152     return $res;
2153 }
2154
2155 # test if VM uses local resources (to prevent migration)
2156 sub check_local_resources {
2157     my ($conf, $noerr) = @_;
2158
2159     my $loc_res = 0;
2160
2161     $loc_res = 1 if $conf->{hostusb}; # old syntax
2162     $loc_res = 1 if $conf->{hostpci}; # old syntax
2163
2164     foreach my $k (keys %$conf) {
2165         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2166         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2167     }
2168
2169     die "VM uses local resources\n" if $loc_res && !$noerr;
2170
2171     return $loc_res;
2172 }
2173
2174 # check if used storages are available on all nodes (use by migrate)
2175 sub check_storage_availability {
2176     my ($storecfg, $conf, $node) = @_;
2177
2178     foreach_drive($conf, sub {
2179         my ($ds, $drive) = @_;
2180
2181         my $volid = $drive->{file};
2182         return if !$volid;
2183
2184         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2185         return if !$sid;
2186
2187         # check if storage is available on both nodes
2188         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2189         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2190    });
2191 }
2192
2193 # list nodes where all VM images are available (used by has_feature API)
2194 sub shared_nodes {
2195     my ($conf, $storecfg) = @_;
2196
2197     my $nodelist = PVE::Cluster::get_nodelist();
2198     my $nodehash = { map { $_ => 1 } @$nodelist };
2199     my $nodename = PVE::INotify::nodename();
2200
2201     foreach_drive($conf, sub {
2202         my ($ds, $drive) = @_;
2203
2204         my $volid = $drive->{file};
2205         return if !$volid;
2206
2207         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2208         if ($storeid) {
2209             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2210             if ($scfg->{disable}) {
2211                 $nodehash = {};
2212             } elsif (my $avail = $scfg->{nodes}) {
2213                 foreach my $node (keys %$nodehash) {
2214                     delete $nodehash->{$node} if !$avail->{$node};
2215                 }
2216             } elsif (!$scfg->{shared}) {
2217                 foreach my $node (keys %$nodehash) {
2218                     delete $nodehash->{$node} if $node ne $nodename
2219                 }
2220             }
2221         }
2222     });
2223
2224     return $nodehash
2225 }
2226
2227 sub check_lock {
2228     my ($conf) = @_;
2229
2230     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2231 }
2232
2233 sub check_cmdline {
2234     my ($pidfile, $pid) = @_;
2235
2236     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2237     if (defined($fh)) {
2238         my $line = <$fh>;
2239         $fh->close;
2240         return undef if !$line;
2241         my @param = split(/\0/, $line);
2242
2243         my $cmd = $param[0];
2244         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2245
2246         for (my $i = 0; $i < scalar (@param); $i++) {
2247             my $p = $param[$i];
2248             next if !$p;
2249             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2250                 my $p = $param[$i+1];
2251                 return 1 if $p && ($p eq $pidfile);
2252                 return undef;
2253             }
2254         }
2255     }
2256     return undef;
2257 }
2258
2259 sub check_running {
2260     my ($vmid, $nocheck, $node) = @_;
2261
2262     my $filename = config_file($vmid, $node);
2263
2264     die "unable to find configuration file for VM $vmid - no such machine\n"
2265         if !$nocheck && ! -f $filename;
2266
2267     my $pidfile = pidfile_name($vmid);
2268
2269     if (my $fd = IO::File->new("<$pidfile")) {
2270         my $st = stat($fd);
2271         my $line = <$fd>;
2272         close($fd);
2273
2274         my $mtime = $st->mtime;
2275         if ($mtime > time()) {
2276             warn "file '$filename' modified in future\n";
2277         }
2278
2279         if ($line =~ m/^(\d+)$/) {
2280             my $pid = $1;
2281             if (check_cmdline($pidfile, $pid)) {
2282                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2283                     return $pid;
2284                 }
2285             }
2286         }
2287     }
2288
2289     return undef;
2290 }
2291
2292 sub vzlist {
2293
2294     my $vzlist = config_list();
2295
2296     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2297
2298     while (defined(my $de = $fd->read)) {
2299         next if $de !~ m/^(\d+)\.pid$/;
2300         my $vmid = $1;
2301         next if !defined($vzlist->{$vmid});
2302         if (my $pid = check_running($vmid)) {
2303             $vzlist->{$vmid}->{pid} = $pid;
2304         }
2305     }
2306
2307     return $vzlist;
2308 }
2309
2310 sub disksize {
2311     my ($storecfg, $conf) = @_;
2312
2313     my $bootdisk = $conf->{bootdisk};
2314     return undef if !$bootdisk;
2315     return undef if !valid_drivename($bootdisk);
2316
2317     return undef if !$conf->{$bootdisk};
2318
2319     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2320     return undef if !defined($drive);
2321
2322     return undef if drive_is_cdrom($drive);
2323
2324     my $volid = $drive->{file};
2325     return undef if !$volid;
2326
2327     return $drive->{size};
2328 }
2329
2330 my $last_proc_pid_stat;
2331
2332 # get VM status information
2333 # This must be fast and should not block ($full == false)
2334 # We only query KVM using QMP if $full == true (this can be slow)
2335 sub vmstatus {
2336     my ($opt_vmid, $full) = @_;
2337
2338     my $res = {};
2339
2340     my $storecfg = PVE::Storage::config();
2341
2342     my $list = vzlist();
2343     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2344
2345     my $cpucount = $cpuinfo->{cpus} || 1;
2346
2347     foreach my $vmid (keys %$list) {
2348         next if $opt_vmid && ($vmid ne $opt_vmid);
2349
2350         my $cfspath = cfs_config_path($vmid);
2351         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2352
2353         my $d = {};
2354         $d->{pid} = $list->{$vmid}->{pid};
2355
2356         # fixme: better status?
2357         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2358
2359         my $size = disksize($storecfg, $conf);
2360         if (defined($size)) {
2361             $d->{disk} = 0; # no info available
2362             $d->{maxdisk} = $size;
2363         } else {
2364             $d->{disk} = 0;
2365             $d->{maxdisk} = 0;
2366         }
2367
2368         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2369         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2370         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2371
2372         $d->{name} = $conf->{name} || "VM $vmid";
2373         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2374
2375         if ($conf->{balloon}) {
2376             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2377             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2378         }
2379
2380         $d->{uptime} = 0;
2381         $d->{cpu} = 0;
2382         $d->{mem} = 0;
2383
2384         $d->{netout} = 0;
2385         $d->{netin} = 0;
2386
2387         $d->{diskread} = 0;
2388         $d->{diskwrite} = 0;
2389
2390         $d->{template} = is_template($conf);
2391
2392         $res->{$vmid} = $d;
2393     }
2394
2395     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2396     foreach my $dev (keys %$netdev) {
2397         next if $dev !~ m/^tap([1-9]\d*)i/;
2398         my $vmid = $1;
2399         my $d = $res->{$vmid};
2400         next if !$d;
2401
2402         $d->{netout} += $netdev->{$dev}->{receive};
2403         $d->{netin} += $netdev->{$dev}->{transmit};
2404     }
2405
2406     my $ctime = gettimeofday;
2407
2408     foreach my $vmid (keys %$list) {
2409
2410         my $d = $res->{$vmid};
2411         my $pid = $d->{pid};
2412         next if !$pid;
2413
2414         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2415         next if !$pstat; # not running
2416
2417         my $used = $pstat->{utime} + $pstat->{stime};
2418
2419         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2420
2421         if ($pstat->{vsize}) {
2422             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2423         }
2424
2425         my $old = $last_proc_pid_stat->{$pid};
2426         if (!$old) {
2427             $last_proc_pid_stat->{$pid} = {
2428                 time => $ctime,
2429                 used => $used,
2430                 cpu => 0,
2431             };
2432             next;
2433         }
2434
2435         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2436
2437         if ($dtime > 1000) {
2438             my $dutime = $used -  $old->{used};
2439
2440             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2441             $last_proc_pid_stat->{$pid} = {
2442                 time => $ctime,
2443                 used => $used,
2444                 cpu => $d->{cpu},
2445             };
2446         } else {
2447             $d->{cpu} = $old->{cpu};
2448         }
2449     }
2450
2451     return $res if !$full;
2452
2453     my $qmpclient = PVE::QMPClient->new();
2454
2455     my $ballooncb = sub {
2456         my ($vmid, $resp) = @_;
2457
2458         my $info = $resp->{'return'};
2459         return if !$info->{max_mem};
2460
2461         my $d = $res->{$vmid};
2462
2463         # use memory assigned to VM
2464         $d->{maxmem} = $info->{max_mem};
2465         $d->{balloon} = $info->{actual};
2466
2467         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2468             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2469             $d->{freemem} = $info->{free_mem};
2470         }
2471
2472     };
2473
2474     my $blockstatscb = sub {
2475         my ($vmid, $resp) = @_;
2476         my $data = $resp->{'return'} || [];
2477         my $totalrdbytes = 0;
2478         my $totalwrbytes = 0;
2479         for my $blockstat (@$data) {
2480             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2481             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2482         }
2483         $res->{$vmid}->{diskread} = $totalrdbytes;
2484         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2485     };
2486
2487     my $statuscb = sub {
2488         my ($vmid, $resp) = @_;
2489
2490         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2491         # this fails if ballon driver is not loaded, so this must be
2492         # the last commnand (following command are aborted if this fails).
2493         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2494
2495         my $status = 'unknown';
2496         if (!defined($status = $resp->{'return'}->{status})) {
2497             warn "unable to get VM status\n";
2498             return;
2499         }
2500
2501         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2502     };
2503
2504     foreach my $vmid (keys %$list) {
2505         next if $opt_vmid && ($vmid ne $opt_vmid);
2506         next if !$res->{$vmid}->{pid}; # not running
2507         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2508     }
2509
2510     $qmpclient->queue_execute(undef, 1);
2511
2512     foreach my $vmid (keys %$list) {
2513         next if $opt_vmid && ($vmid ne $opt_vmid);
2514         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2515     }
2516
2517     return $res;
2518 }
2519
2520 sub foreach_dimm {
2521     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2522
2523     my $dimm_id = 0;
2524     my $current_size = 1024;
2525     my $dimm_size = 512;
2526     return if $current_size == $memory;
2527
2528     for (my $j = 0; $j < 8; $j++) {
2529         for (my $i = 0; $i < 32; $i++) {
2530             my $name = "dimm${dimm_id}";
2531             $dimm_id++;
2532             my $numanode = $i % $sockets;
2533             $current_size += $dimm_size;
2534             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2535             return  $current_size if $current_size >= $memory;
2536         }
2537         $dimm_size *= 2;
2538     }
2539 }
2540
2541 sub foreach_drive {
2542     my ($conf, $func) = @_;
2543
2544     foreach my $ds (keys %$conf) {
2545         next if !valid_drivename($ds);
2546
2547         my $drive = parse_drive($ds, $conf->{$ds});
2548         next if !$drive;
2549
2550         &$func($ds, $drive);
2551     }
2552 }
2553
2554 sub foreach_volid {
2555     my ($conf, $func) = @_;
2556
2557     my $volhash = {};
2558
2559     my $test_volid = sub {
2560         my ($volid, $is_cdrom) = @_;
2561
2562         return if !$volid;
2563
2564         $volhash->{$volid} = $is_cdrom || 0;
2565     };
2566
2567     foreach_drive($conf, sub {
2568         my ($ds, $drive) = @_;
2569         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2570     });
2571
2572     foreach my $snapname (keys %{$conf->{snapshots}}) {
2573         my $snap = $conf->{snapshots}->{$snapname};
2574         &$test_volid($snap->{vmstate}, 0);
2575         foreach_drive($snap, sub {
2576             my ($ds, $drive) = @_;
2577             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2578         });
2579     }
2580
2581     foreach my $volid (keys %$volhash) {
2582         &$func($volid, $volhash->{$volid});
2583     }
2584 }
2585
2586 sub vga_conf_has_spice {
2587     my ($vga) = @_;
2588
2589     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2590
2591     return $1 || 1;
2592 }
2593
2594 sub config_to_command {
2595     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2596
2597     my $cmd = [];
2598     my $globalFlags = [];
2599     my $machineFlags = [];
2600     my $rtcFlags = [];
2601     my $cpuFlags = [];
2602     my $devices = [];
2603     my $pciaddr = '';
2604     my $bridges = {};
2605     my $kvmver = kvm_user_version();
2606     my $vernum = 0; # unknown
2607     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2608         $vernum = $1*1000000+$2*1000;
2609     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2610         $vernum = $1*1000000+$2*1000+$3;
2611     }
2612
2613     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2614
2615     my $have_ovz = -f '/proc/vz/vestat';
2616
2617     my $q35 = machine_type_is_q35($conf);
2618     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2619     my $machine_type = $forcemachine || $conf->{machine};
2620
2621     push @$cmd, '/usr/bin/kvm';
2622
2623     push @$cmd, '-id', $vmid;
2624
2625     my $use_virtio = 0;
2626
2627     my $qmpsocket = qmp_socket($vmid);
2628     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2629     push @$cmd, '-mon', "chardev=qmp,mode=control";
2630
2631     my $socket = vnc_socket($vmid);
2632     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2633
2634     push @$cmd, '-pidfile' , pidfile_name($vmid);
2635
2636     push @$cmd, '-daemonize';
2637
2638     if ($conf->{smbios1}) {
2639         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2640     }
2641
2642     if ($q35) {
2643         # the q35 chipset support native usb2, so we enable usb controller
2644         # by default for this machine type
2645         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2646     } else {
2647         $pciaddr = print_pci_addr("piix3", $bridges);
2648         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2649
2650         my $use_usb2 = 0;
2651         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2652             next if !$conf->{"usb$i"};
2653             $use_usb2 = 1;
2654         }
2655         # include usb device config
2656         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2657     }
2658
2659     my $vga = $conf->{vga};
2660
2661     my $qxlnum = vga_conf_has_spice($vga);
2662     $vga = 'qxl' if $qxlnum;
2663
2664     if (!$vga) {
2665         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2666                                 $conf->{ostype} eq 'win7' ||
2667                                 $conf->{ostype} eq 'w2k8')) {
2668             $vga = 'std';
2669         } else {
2670             $vga = 'cirrus';
2671         }
2672     }
2673
2674     # enable absolute mouse coordinates (needed by vnc)
2675     my $tablet;
2676     if (defined($conf->{tablet})) {
2677         $tablet = $conf->{tablet};
2678     } else {
2679         $tablet = $defaults->{tablet};
2680         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2681         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2682     }
2683
2684     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2685
2686     # host pci devices
2687     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2688         my $d = parse_hostpci($conf->{"hostpci$i"});
2689         next if !$d;
2690
2691         my $pcie = $d->{pcie};
2692         if($pcie){
2693             die "q35 machine model is not enabled" if !$q35;
2694             $pciaddr = print_pcie_addr("hostpci$i");
2695         }else{
2696             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2697         }
2698
2699         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2700         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2701         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2702         if ($xvga && $xvga ne '') {
2703             push @$cpuFlags, 'kvm=off';
2704             $vga = 'none';
2705         }
2706         $driver = "vfio-pci" if $xvga ne '';
2707         my $pcidevices = $d->{pciid};
2708         my $multifunction = 1 if @$pcidevices > 1;
2709
2710         my $j=0;
2711         foreach my $pcidevice (@$pcidevices) {
2712
2713             my $id = "hostpci$i";
2714             $id .= ".$j" if $multifunction;
2715             my $addr = $pciaddr;
2716             $addr .= ".$j" if $multifunction;
2717             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2718
2719             if($j == 0){
2720                 $devicestr .= "$rombar$xvga";
2721                 $devicestr .= ",multifunction=on" if $multifunction;
2722             }
2723
2724             push @$devices, '-device', $devicestr;
2725             $j++;
2726         }
2727     }
2728
2729     # usb devices
2730     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2731         my $d = parse_usb_device($conf->{"usb$i"});
2732         next if !$d;
2733         if ($d->{vendorid} && $d->{productid}) {
2734             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2735         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2736             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2737         } elsif ($d->{spice}) {
2738             # usb redir support for spice
2739             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2740             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2741         }
2742     }
2743
2744     # serial devices
2745     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2746         if (my $path = $conf->{"serial$i"}) {
2747             if ($path eq 'socket') {
2748                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2749                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2750                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2751             } else {
2752                 die "no such serial device\n" if ! -c $path;
2753                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2754                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2755             }
2756         }
2757     }
2758
2759     # parallel devices
2760     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2761         if (my $path = $conf->{"parallel$i"}) {
2762             die "no such parallel device\n" if ! -c $path;
2763             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2764             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2765             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2766         }
2767     }
2768
2769     my $vmname = $conf->{name} || "vm$vmid";
2770
2771     push @$cmd, '-name', $vmname;
2772
2773     my $sockets = 1;
2774     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2775     $sockets = $conf->{sockets} if  $conf->{sockets};
2776
2777     my $cores = $conf->{cores} || 1;
2778
2779     my $maxcpus = $sockets * $cores;
2780
2781     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2782
2783     my $allowed_vcpus = $cpuinfo->{cpus};
2784
2785     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2786         if ($allowed_vcpus < $maxcpus);
2787
2788     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2789
2790     push @$cmd, '-nodefaults';
2791
2792     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2793
2794     my $bootindex_hash = {};
2795     my $i = 1;
2796     foreach my $o (split(//, $bootorder)) {
2797         $bootindex_hash->{$o} = $i*100;
2798         $i++;
2799     }
2800
2801     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2802
2803     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2804
2805     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2806
2807     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2808
2809     # time drift fix
2810     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2811
2812     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2813     my $useLocaltime = $conf->{localtime};
2814
2815     if (my $ost = $conf->{ostype}) {
2816         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2817
2818         if ($ost =~ m/^w/) { # windows
2819             $useLocaltime = 1 if !defined($conf->{localtime});
2820
2821             # use time drift fix when acpi is enabled
2822             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2823                 $tdf = 1 if !defined($conf->{tdf});
2824             }
2825         }
2826
2827         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2828             $ost eq 'wvista') {
2829             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2830             push @$cmd, '-no-hpet';
2831             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2832                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2833                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2834                 push @$cpuFlags , 'hv_time' if !$nokvm;
2835
2836             } else {
2837                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2838             }
2839         }
2840
2841         if ($ost eq 'win7' || $ost eq 'win8') {
2842             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2843         }
2844     }
2845
2846     push @$rtcFlags, 'driftfix=slew' if $tdf;
2847
2848     if ($nokvm) {
2849         push @$machineFlags, 'accel=tcg';
2850     } else {
2851         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2852     }
2853
2854     if ($machine_type) {
2855         push @$machineFlags, "type=${machine_type}";
2856     }
2857
2858     if ($conf->{startdate}) {
2859         push @$rtcFlags, "base=$conf->{startdate}";
2860     } elsif ($useLocaltime) {
2861         push @$rtcFlags, 'base=localtime';
2862     }
2863
2864     my $cpu = $nokvm ? "qemu64" : "kvm64";
2865     $cpu = $conf->{cpu} if $conf->{cpu};
2866
2867     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2868
2869     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2870
2871     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2872
2873     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2874
2875     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2876
2877         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2878         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2879     }
2880
2881     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2882
2883     push @$cmd, '-cpu', "$cpu,enforce";
2884
2885     my $memory = $conf->{memory} || $defaults->{memory};
2886     my $static_memory = 0;
2887     my $dimm_memory = 0;
2888
2889     if ($hotplug_features->{memory}) {
2890         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2891         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2892         $static_memory = $STATICMEM;
2893         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2894         $dimm_memory = $memory - $static_memory;
2895         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2896
2897     } else {
2898
2899         $static_memory = $memory;
2900         push @$cmd, '-m', $static_memory;
2901     }
2902
2903     if ($conf->{numa}) {
2904
2905         my $numa_totalmemory = undef;
2906         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2907             next if !$conf->{"numa$i"};
2908             my $numa = parse_numa($conf->{"numa$i"});
2909             next if !$numa;
2910             # memory
2911             die "missing numa node$i memory value\n" if !$numa->{memory};
2912             my $numa_memory = $numa->{memory};
2913             $numa_totalmemory += $numa_memory;
2914             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2915
2916             # cpus
2917             my $cpus_start = $numa->{cpus}->{start};
2918             die "missing numa node$i cpus\n" if !defined($cpus_start);
2919             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2920             my $cpus = $cpus_start;
2921             if (defined($cpus_end)) {
2922                 $cpus .= "-$cpus_end";
2923                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2924             }
2925
2926             # hostnodes
2927             my $hostnodes_start = $numa->{hostnodes}->{start};
2928             if (defined($hostnodes_start)) {
2929                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2930                 my $hostnodes = $hostnodes_start;
2931                 if (defined($hostnodes_end)) {
2932                     $hostnodes .= "-$hostnodes_end";
2933                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2934                 }
2935
2936                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2937                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2938                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2939                 }
2940
2941                 # policy
2942                 my $policy = $numa->{policy};
2943                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2944                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2945             }
2946
2947             push @$cmd, '-object', $numa_object;
2948             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2949         }
2950
2951         die "total memory for NUMA nodes must be equal to vm static memory\n"
2952             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2953
2954         #if no custom tology, we split memory and cores across numa nodes
2955         if(!$numa_totalmemory) {
2956
2957             my $numa_memory = ($static_memory / $sockets) . "M";
2958
2959             for (my $i = 0; $i < $sockets; $i++)  {
2960
2961                 my $cpustart = ($cores * $i);
2962                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2963                 my $cpus = $cpustart;
2964                 $cpus .= "-$cpuend" if $cpuend;
2965
2966                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2967                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2968             }
2969         }
2970     }
2971
2972     if ($hotplug_features->{memory}) {
2973         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2974             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2975             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2976             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2977
2978             #if dimm_memory is not aligned to dimm map
2979             if($current_size > $memory) {
2980                  $conf->{memory} = $current_size;
2981                  update_config_nolock($vmid, $conf, 1);
2982             }
2983         });
2984     }
2985
2986     push @$cmd, '-S' if $conf->{freeze};
2987
2988     # set keyboard layout
2989     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2990     push @$cmd, '-k', $kb if $kb;
2991
2992     # enable sound
2993     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2994     #push @$cmd, '-soundhw', 'es1370';
2995     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2996
2997     if($conf->{agent}) {
2998         my $qgasocket = qmp_socket($vmid, 1);
2999         my $pciaddr = print_pci_addr("qga0", $bridges);
3000         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3001         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3002         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3003     }
3004
3005     my $spice_port;
3006
3007     if ($qxlnum) {
3008         if ($qxlnum > 1) {
3009             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3010                 for(my $i = 1; $i < $qxlnum; $i++){
3011                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3012                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3013                 }
3014             } else {
3015                 # assume other OS works like Linux
3016                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3017                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3018             }
3019         }
3020
3021         my $pciaddr = print_pci_addr("spice", $bridges);
3022
3023         my $nodename = PVE::INotify::nodename();
3024         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3025         $spice_port = PVE::Tools::next_spice_port($pfamily);
3026
3027         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3028
3029         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3030         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3031         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3032     }
3033
3034     # enable balloon by default, unless explicitly disabled
3035     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3036         $pciaddr = print_pci_addr("balloon0", $bridges);
3037         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3038     }
3039
3040     if ($conf->{watchdog}) {
3041         my $wdopts = parse_watchdog($conf->{watchdog});
3042         $pciaddr = print_pci_addr("watchdog", $bridges);
3043         my $watchdog = $wdopts->{model} || 'i6300esb';
3044         push @$devices, '-device', "$watchdog$pciaddr";
3045         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3046     }
3047
3048     my $vollist = [];
3049     my $scsicontroller = {};
3050     my $ahcicontroller = {};
3051     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3052
3053     # Add iscsi initiator name if available
3054     if (my $initiator = get_initiator_name()) {
3055         push @$devices, '-iscsi', "initiator-name=$initiator";
3056     }
3057
3058     foreach_drive($conf, sub {
3059         my ($ds, $drive) = @_;
3060
3061         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3062             push @$vollist, $drive->{file};
3063         }
3064
3065         $use_virtio = 1 if $ds =~ m/^virtio/;
3066
3067         if (drive_is_cdrom ($drive)) {
3068             if ($bootindex_hash->{d}) {
3069                 $drive->{bootindex} = $bootindex_hash->{d};
3070                 $bootindex_hash->{d} += 1;
3071             }
3072         } else {
3073             if ($bootindex_hash->{c}) {
3074                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3075                 $bootindex_hash->{c} += 1;
3076             }
3077         }
3078
3079         if($drive->{interface} eq 'virtio'){
3080            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3081         }
3082
3083         if ($drive->{interface} eq 'scsi') {
3084
3085             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3086
3087             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3088             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3089
3090             my $iothread = '';
3091             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3092                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3093                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3094             }
3095
3096             my $queues = '';
3097             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3098                 $queues = ",num_queues=$drive->{queues}";
3099             }
3100
3101             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3102             $scsicontroller->{$controller}=1;
3103         }
3104
3105         if ($drive->{interface} eq 'sata') {
3106            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3107            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3108            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3109            $ahcicontroller->{$controller}=1;
3110         }
3111
3112         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3113         push @$devices, '-drive',$drive_cmd;
3114         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3115     });
3116
3117     for (my $i = 0; $i < $MAX_NETS; $i++) {
3118          next if !$conf->{"net$i"};
3119          my $d = parse_net($conf->{"net$i"});
3120          next if !$d;
3121
3122          $use_virtio = 1 if $d->{model} eq 'virtio';
3123
3124          if ($bootindex_hash->{n}) {
3125             $d->{bootindex} = $bootindex_hash->{n};
3126             $bootindex_hash->{n} += 1;
3127          }
3128
3129          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3130          push @$devices, '-netdev', $netdevfull;
3131
3132          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3133          push @$devices, '-device', $netdevicefull;
3134     }
3135
3136     if (!$q35) {
3137         # add pci bridges
3138         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3139            $bridges->{1} = 1;
3140            $bridges->{2} = 1;
3141         }
3142
3143         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3144
3145         while (my ($k, $v) = each %$bridges) {
3146             $pciaddr = print_pci_addr("pci.$k");
3147             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3148         }
3149     }
3150
3151     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3152     # when the VM uses virtio devices.
3153     if (!$use_virtio && $have_ovz) {
3154
3155         my $cpuunits = defined($conf->{cpuunits}) ?
3156             $conf->{cpuunits} : $defaults->{cpuunits};
3157
3158         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3159
3160         # fixme: cpulimit is currently ignored
3161         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3162     }
3163
3164     # add custom args
3165     if ($conf->{args}) {
3166         my $aa = PVE::Tools::split_args($conf->{args});
3167         push @$cmd, @$aa;
3168     }
3169
3170     push @$cmd, @$devices;
3171     push @$cmd, '-rtc', join(',', @$rtcFlags)
3172         if scalar(@$rtcFlags);
3173     push @$cmd, '-machine', join(',', @$machineFlags)
3174         if scalar(@$machineFlags);
3175     push @$cmd, '-global', join(',', @$globalFlags)
3176         if scalar(@$globalFlags);
3177
3178     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3179 }
3180
3181 sub vnc_socket {
3182     my ($vmid) = @_;
3183     return "${var_run_tmpdir}/$vmid.vnc";
3184 }
3185
3186 sub spice_port {
3187     my ($vmid) = @_;
3188
3189     my $res = vm_mon_cmd($vmid, 'query-spice');
3190
3191     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3192 }
3193
3194 sub qmp_socket {
3195     my ($vmid, $qga) = @_;
3196     my $sockettype = $qga ? 'qga' : 'qmp';
3197     return "${var_run_tmpdir}/$vmid.$sockettype";
3198 }
3199
3200 sub pidfile_name {
3201     my ($vmid) = @_;
3202     return "${var_run_tmpdir}/$vmid.pid";
3203 }
3204
3205 sub vm_devices_list {
3206     my ($vmid) = @_;
3207
3208     my $res = vm_mon_cmd($vmid, 'query-pci');
3209     my $devices = {};
3210     foreach my $pcibus (@$res) {
3211         foreach my $device (@{$pcibus->{devices}}) {
3212             next if !$device->{'qdev_id'};
3213             if ($device->{'pci_bridge'}) {
3214                 $devices->{$device->{'qdev_id'}} = 1;
3215                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3216                     next if !$bridge_device->{'qdev_id'};
3217                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3218                     $devices->{$device->{'qdev_id'}}++;
3219                 }
3220             } else {
3221                 $devices->{$device->{'qdev_id'}} = 1;
3222             }
3223         }
3224     }
3225
3226     my $resblock = vm_mon_cmd($vmid, 'query-block');
3227     foreach my $block (@$resblock) {
3228         if($block->{device} =~ m/^drive-(\S+)/){
3229                 $devices->{$1} = 1;
3230         }
3231     }
3232
3233     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3234     foreach my $mice (@$resmice) {
3235         if ($mice->{name} eq 'QEMU HID Tablet') {
3236             $devices->{tablet} = 1;
3237             last;
3238         }
3239     }
3240
3241     return $devices;
3242 }
3243
3244 sub vm_deviceplug {
3245     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3246
3247     my $q35 = machine_type_is_q35($conf);
3248
3249     my $devices_list = vm_devices_list($vmid);
3250     return 1 if defined($devices_list->{$deviceid});
3251
3252     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3253
3254     if ($deviceid eq 'tablet') {
3255
3256         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3257
3258     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3259
3260         qemu_iothread_add($vmid, $deviceid, $device);
3261
3262         qemu_driveadd($storecfg, $vmid, $device);
3263         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3264
3265         qemu_deviceadd($vmid, $devicefull);
3266         eval { qemu_deviceaddverify($vmid, $deviceid); };
3267         if (my $err = $@) {
3268             eval { qemu_drivedel($vmid, $deviceid); };
3269             warn $@ if $@;
3270             die $err;
3271         }
3272
3273     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3274
3275
3276         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3277         my $pciaddr = print_pci_addr($deviceid);
3278         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3279
3280         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3281
3282         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3283             qemu_iothread_add($vmid, $deviceid, $device);
3284             $devicefull .= ",iothread=iothread-$deviceid";
3285         }
3286
3287         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3288             $devicefull .= ",num_queues=$device->{queues}";
3289         }
3290
3291         qemu_deviceadd($vmid, $devicefull);
3292         qemu_deviceaddverify($vmid, $deviceid);
3293
3294     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3295
3296         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3297         qemu_driveadd($storecfg, $vmid, $device);
3298
3299         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3300         eval { qemu_deviceadd($vmid, $devicefull); };
3301         if (my $err = $@) {
3302             eval { qemu_drivedel($vmid, $deviceid); };
3303             warn $@ if $@;
3304             die $err;
3305         }
3306
3307     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3308
3309         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3310         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3311         qemu_deviceadd($vmid, $netdevicefull);
3312         eval { qemu_deviceaddverify($vmid, $deviceid); };
3313         if (my $err = $@) {
3314             eval { qemu_netdevdel($vmid, $deviceid); };
3315             warn $@ if $@;
3316             die $err;
3317         }
3318
3319     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3320
3321         my $bridgeid = $2;
3322         my $pciaddr = print_pci_addr($deviceid);
3323         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3324
3325         qemu_deviceadd($vmid, $devicefull);
3326         qemu_deviceaddverify($vmid, $deviceid);
3327
3328     } else {
3329         die "can't hotplug device '$deviceid'\n";
3330     }
3331
3332     return 1;
3333 }
3334
3335 # fixme: this should raise exceptions on error!
3336 sub vm_deviceunplug {
3337     my ($vmid, $conf, $deviceid) = @_;
3338
3339     my $devices_list = vm_devices_list($vmid);
3340     return 1 if !defined($devices_list->{$deviceid});
3341
3342     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3343
3344     if ($deviceid eq 'tablet') {
3345
3346         qemu_devicedel($vmid, $deviceid);
3347
3348     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3349
3350         qemu_devicedel($vmid, $deviceid);
3351         qemu_devicedelverify($vmid, $deviceid);
3352         qemu_drivedel($vmid, $deviceid);
3353         qemu_iothread_del($conf, $vmid, $deviceid);
3354
3355     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3356
3357         qemu_devicedel($vmid, $deviceid);
3358         qemu_devicedelverify($vmid, $deviceid);
3359         qemu_iothread_del($conf, $vmid, $deviceid);
3360
3361     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3362
3363         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3364         my $device = parse_drive($deviceid, $conf->{$deviceid});
3365         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3366
3367         qemu_devicedel($vmid, $deviceid);
3368         qemu_drivedel($vmid, $deviceid);
3369         qemu_deletescsihw($conf, $vmid, $deviceid);
3370
3371     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3372
3373         qemu_devicedel($vmid, $deviceid);
3374         qemu_devicedelverify($vmid, $deviceid);
3375         qemu_netdevdel($vmid, $deviceid);
3376
3377     } else {
3378         die "can't unplug device '$deviceid'\n";
3379     }
3380
3381     return 1;
3382 }
3383
3384 sub qemu_deviceadd {
3385     my ($vmid, $devicefull) = @_;
3386
3387     $devicefull = "driver=".$devicefull;
3388     my %options =  split(/[=,]/, $devicefull);
3389
3390     vm_mon_cmd($vmid, "device_add" , %options);
3391 }
3392
3393 sub qemu_devicedel {
3394     my ($vmid, $deviceid) = @_;
3395
3396     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3397 }
3398
3399 sub qemu_iothread_add {
3400     my($vmid, $deviceid, $device) = @_;
3401
3402     if ($device->{iothread}) {
3403         my $iothreads = vm_iothreads_list($vmid);
3404         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3405     }
3406 }
3407
3408 sub qemu_iothread_del {
3409     my($conf, $vmid, $deviceid) = @_;
3410
3411     my $device = parse_drive($deviceid, $conf->{$deviceid});
3412     if ($device->{iothread}) {
3413         my $iothreads = vm_iothreads_list($vmid);
3414         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3415     }
3416 }
3417
3418 sub qemu_objectadd {
3419     my($vmid, $objectid, $qomtype) = @_;
3420
3421     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3422
3423     return 1;
3424 }
3425
3426 sub qemu_objectdel {
3427     my($vmid, $objectid) = @_;
3428
3429     vm_mon_cmd($vmid, "object-del", id => $objectid);
3430
3431     return 1;
3432 }
3433
3434 sub qemu_driveadd {
3435     my ($storecfg, $vmid, $device) = @_;
3436
3437     my $drive = print_drive_full($storecfg, $vmid, $device);
3438     $drive =~ s/\\/\\\\/g;
3439     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3440
3441     # If the command succeeds qemu prints: "OK"
3442     return 1 if $ret =~ m/OK/s;
3443
3444     die "adding drive failed: $ret\n";
3445 }
3446
3447 sub qemu_drivedel {
3448     my($vmid, $deviceid) = @_;
3449
3450     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3451     $ret =~ s/^\s+//;
3452
3453     return 1 if $ret eq "";
3454
3455     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3456     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3457
3458     die "deleting drive $deviceid failed : $ret\n";
3459 }
3460
3461 sub qemu_deviceaddverify {
3462     my ($vmid, $deviceid) = @_;
3463
3464     for (my $i = 0; $i <= 5; $i++) {
3465          my $devices_list = vm_devices_list($vmid);
3466          return 1 if defined($devices_list->{$deviceid});
3467          sleep 1;
3468     }
3469
3470     die "error on hotplug device '$deviceid'\n";
3471 }
3472
3473
3474 sub qemu_devicedelverify {
3475     my ($vmid, $deviceid) = @_;
3476
3477     # need to verify that the device is correctly removed as device_del
3478     # is async and empty return is not reliable
3479
3480     for (my $i = 0; $i <= 5; $i++) {
3481          my $devices_list = vm_devices_list($vmid);
3482          return 1 if !defined($devices_list->{$deviceid});
3483          sleep 1;
3484     }
3485
3486     die "error on hot-unplugging device '$deviceid'\n";
3487 }
3488
3489 sub qemu_findorcreatescsihw {
3490     my ($storecfg, $conf, $vmid, $device) = @_;
3491
3492     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3493
3494     my $scsihwid="$controller_prefix$controller";
3495     my $devices_list = vm_devices_list($vmid);
3496
3497     if(!defined($devices_list->{$scsihwid})) {
3498         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3499     }
3500
3501     return 1;
3502 }
3503
3504 sub qemu_deletescsihw {
3505     my ($conf, $vmid, $opt) = @_;
3506
3507     my $device = parse_drive($opt, $conf->{$opt});
3508
3509     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3510         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3511         return 1;
3512     }
3513
3514     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3515
3516     my $devices_list = vm_devices_list($vmid);
3517     foreach my $opt (keys %{$devices_list}) {
3518         if (PVE::QemuServer::valid_drivename($opt)) {
3519             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3520             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3521                 return 1;
3522             }
3523         }
3524     }
3525
3526     my $scsihwid="scsihw$controller";
3527
3528     vm_deviceunplug($vmid, $conf, $scsihwid);
3529
3530     return 1;
3531 }
3532
3533 sub qemu_add_pci_bridge {
3534     my ($storecfg, $conf, $vmid, $device) = @_;
3535
3536     my $bridges = {};
3537
3538     my $bridgeid;
3539
3540     print_pci_addr($device, $bridges);
3541
3542     while (my ($k, $v) = each %$bridges) {
3543         $bridgeid = $k;
3544     }
3545     return 1 if !defined($bridgeid) || $bridgeid < 1;
3546
3547     my $bridge = "pci.$bridgeid";
3548     my $devices_list = vm_devices_list($vmid);
3549
3550     if (!defined($devices_list->{$bridge})) {
3551         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3552     }
3553
3554     return 1;
3555 }
3556
3557 sub qemu_set_link_status {
3558     my ($vmid, $device, $up) = @_;
3559
3560     vm_mon_cmd($vmid, "set_link", name => $device,
3561                up => $up ? JSON::true : JSON::false);
3562 }
3563
3564 sub qemu_netdevadd {
3565     my ($vmid, $conf, $device, $deviceid) = @_;
3566
3567     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3568     my %options =  split(/[=,]/, $netdev);
3569
3570     vm_mon_cmd($vmid, "netdev_add",  %options);
3571     return 1;
3572 }
3573
3574 sub qemu_netdevdel {
3575     my ($vmid, $deviceid) = @_;
3576
3577     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3578 }
3579
3580 sub qemu_cpu_hotplug {
3581     my ($vmid, $conf, $vcpus) = @_;
3582
3583     my $sockets = 1;
3584     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3585     $sockets = $conf->{sockets} if  $conf->{sockets};
3586     my $cores = $conf->{cores} || 1;
3587     my $maxcpus = $sockets * $cores;
3588
3589     $vcpus = $maxcpus if !$vcpus;
3590
3591     die "you can't add more vcpus than maxcpus\n"
3592         if $vcpus > $maxcpus;
3593
3594     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3595     die "online cpu unplug is not yet possible\n"
3596         if $vcpus < $currentvcpus;
3597
3598     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3599     die "vcpus in running vm is different than configuration\n"
3600         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3601
3602     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3603         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3604     }
3605 }
3606
3607 sub qemu_memory_hotplug {
3608     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3609
3610     return $value if !check_running($vmid);
3611
3612     my $memory = $conf->{memory} || $defaults->{memory};
3613     $value = $defaults->{memory} if !$value;
3614     return $value if $value == $memory;
3615
3616     my $static_memory = $STATICMEM;
3617     my $dimm_memory = $memory - $static_memory;
3618
3619     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3620     die "memory unplug is not yet available" if $value < $memory;
3621     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3622
3623
3624     my $sockets = 1;
3625     $sockets = $conf->{sockets} if $conf->{sockets};
3626
3627     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3628         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3629
3630             return if $current_size <= $conf->{memory};
3631
3632             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3633             if (my $err = $@) {
3634                 eval { qemu_objectdel($vmid, "mem-$name"); };
3635                 die $err;
3636             }
3637
3638             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3639             if (my $err = $@) {
3640                 eval { qemu_objectdel($vmid, "mem-$name"); };
3641                 die $err;
3642             }
3643             #update conf after each succesful module hotplug
3644             $conf->{memory} = $current_size;
3645             update_config_nolock($vmid, $conf, 1);
3646     });
3647 }
3648
3649 sub qemu_block_set_io_throttle {
3650     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3651
3652     return if !check_running($vmid) ;
3653
3654     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3655
3656 }
3657
3658 # old code, only used to shutdown old VM after update
3659 sub __read_avail {
3660     my ($fh, $timeout) = @_;
3661
3662     my $sel = new IO::Select;
3663     $sel->add($fh);
3664
3665     my $res = '';
3666     my $buf;
3667
3668     my @ready;
3669     while (scalar (@ready = $sel->can_read($timeout))) {
3670         my $count;
3671         if ($count = $fh->sysread($buf, 8192)) {
3672             if ($buf =~ /^(.*)\(qemu\) $/s) {
3673                 $res .= $1;
3674                 last;
3675             } else {
3676                 $res .= $buf;
3677             }
3678         } else {
3679             if (!defined($count)) {
3680                 die "$!\n";
3681             }
3682             last;
3683         }
3684     }
3685
3686     die "monitor read timeout\n" if !scalar(@ready);
3687
3688     return $res;
3689 }
3690
3691 # old code, only used to shutdown old VM after update
3692 sub vm_monitor_command {
3693     my ($vmid, $cmdstr, $nocheck) = @_;
3694
3695     my $res;
3696
3697     eval {
3698         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3699
3700         my $sname = "${var_run_tmpdir}/$vmid.mon";
3701
3702         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3703             die "unable to connect to VM $vmid socket - $!\n";
3704
3705         my $timeout = 3;
3706
3707         # hack: migrate sometime blocks the monitor (when migrate_downtime
3708         # is set)
3709         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3710             $timeout = 60*60; # 1 hour
3711         }
3712
3713         # read banner;
3714         my $data = __read_avail($sock, $timeout);
3715
3716         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3717             die "got unexpected qemu monitor banner\n";
3718         }
3719
3720         my $sel = new IO::Select;
3721         $sel->add($sock);
3722
3723         if (!scalar(my @ready = $sel->can_write($timeout))) {
3724             die "monitor write error - timeout";
3725         }
3726
3727         my $fullcmd = "$cmdstr\r";
3728
3729         # syslog('info', "VM $vmid monitor command: $cmdstr");
3730
3731         my $b;
3732         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3733             die "monitor write error - $!";
3734         }
3735
3736         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3737
3738         $timeout = 20;
3739
3740         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3741             $timeout = 60*60; # 1 hour
3742         } elsif ($cmdstr =~ m/^(eject|change)/) {
3743             $timeout = 60; # note: cdrom mount command is slow
3744         }
3745         if ($res = __read_avail($sock, $timeout)) {
3746
3747             my @lines = split("\r?\n", $res);
3748
3749             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3750
3751             $res = join("\n", @lines);
3752             $res .= "\n";
3753         }
3754     };
3755
3756     my $err = $@;
3757
3758     if ($err) {
3759         syslog("err", "VM $vmid monitor command failed - $err");
3760         die $err;
3761     }
3762
3763     return $res;
3764 }
3765
3766 sub qemu_block_resize {
3767     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3768
3769     my $running = check_running($vmid);
3770
3771     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3772
3773     return if !$running;
3774
3775     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3776
3777 }
3778
3779 sub qemu_volume_snapshot {
3780     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3781
3782     my $running = check_running($vmid);
3783
3784     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3785         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3786     } else {
3787         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3788     }
3789 }
3790
3791 sub qemu_volume_snapshot_delete {
3792     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3793
3794     my $running = check_running($vmid);
3795
3796     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3797
3798     return if !$running;
3799
3800     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3801 }
3802
3803 sub set_migration_caps {
3804     my ($vmid) = @_;
3805
3806     my $cap_ref = [];
3807
3808     my $enabled_cap = {
3809         "auto-converge" => 1,
3810         "xbzrle" => 0,
3811         "x-rdma-pin-all" => 0,
3812         "zero-blocks" => 0,
3813     };
3814
3815     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3816
3817     for my $supported_capability (@$supported_capabilities) {
3818         push @$cap_ref, {
3819             capability => $supported_capability->{capability},
3820             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3821         };
3822     }
3823
3824     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3825 }
3826
3827 my $fast_plug_option = {
3828     'lock' => 1,
3829     'name' => 1,
3830     'onboot' => 1,
3831     'shares' => 1,
3832     'startup' => 1,
3833 };
3834
3835 # hotplug changes in [PENDING]
3836 # $selection hash can be used to only apply specified options, for
3837 # example: { cores => 1 } (only apply changed 'cores')
3838 # $errors ref is used to return error messages
3839 sub vmconfig_hotplug_pending {
3840     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3841
3842     my $defaults = load_defaults();
3843
3844     # commit values which do not have any impact on running VM first
3845     # Note: those option cannot raise errors, we we do not care about
3846     # $selection and always apply them.
3847
3848     my $add_error = sub {
3849         my ($opt, $msg) = @_;
3850         $errors->{$opt} = "hotplug problem - $msg";
3851     };
3852
3853     my $changes = 0;
3854     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3855         if ($fast_plug_option->{$opt}) {
3856             $conf->{$opt} = $conf->{pending}->{$opt};
3857             delete $conf->{pending}->{$opt};
3858             $changes = 1;
3859         }
3860     }
3861
3862     if ($changes) {
3863         update_config_nolock($vmid, $conf, 1);
3864         $conf = load_config($vmid); # update/reload
3865     }
3866
3867     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3868
3869     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3870     foreach my $opt (@delete) {
3871         next if $selection && !$selection->{$opt};
3872         eval {
3873             if ($opt eq 'hotplug') {
3874                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3875             } elsif ($opt eq 'tablet') {
3876                 die "skip\n" if !$hotplug_features->{usb};
3877                 if ($defaults->{tablet}) {
3878                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3879                 } else {
3880                     vm_deviceunplug($vmid, $conf, $opt);
3881                 }
3882             } elsif ($opt eq 'vcpus') {
3883                 die "skip\n" if !$hotplug_features->{cpu};
3884                 qemu_cpu_hotplug($vmid, $conf, undef);
3885             } elsif ($opt eq 'balloon') {
3886                 # enable balloon device is not hotpluggable
3887                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3888             } elsif ($fast_plug_option->{$opt}) {
3889                 # do nothing
3890             } elsif ($opt =~ m/^net(\d+)$/) {
3891                 die "skip\n" if !$hotplug_features->{network};
3892                 vm_deviceunplug($vmid, $conf, $opt);
3893             } elsif (valid_drivename($opt)) {
3894                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3895                 vm_deviceunplug($vmid, $conf, $opt);
3896                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3897             } elsif ($opt =~ m/^memory$/) {
3898                 die "skip\n" if !$hotplug_features->{memory};
3899                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3900             } else {
3901                 die "skip\n";
3902             }
3903         };
3904         if (my $err = $@) {
3905             &$add_error($opt, $err) if $err ne "skip\n";
3906         } else {
3907             # save new config if hotplug was successful
3908             delete $conf->{$opt};
3909             vmconfig_undelete_pending_option($conf, $opt);
3910             update_config_nolock($vmid, $conf, 1);
3911             $conf = load_config($vmid); # update/reload
3912         }
3913     }
3914
3915     foreach my $opt (keys %{$conf->{pending}}) {
3916         next if $selection && !$selection->{$opt};
3917         my $value = $conf->{pending}->{$opt};
3918         eval {
3919             if ($opt eq 'hotplug') {
3920                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3921             } elsif ($opt eq 'tablet') {
3922                 die "skip\n" if !$hotplug_features->{usb};
3923                 if ($value == 1) {
3924                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3925                 } elsif ($value == 0) {
3926                     vm_deviceunplug($vmid, $conf, $opt);
3927                 }
3928             } elsif ($opt eq 'vcpus') {
3929                 die "skip\n" if !$hotplug_features->{cpu};
3930                 qemu_cpu_hotplug($vmid, $conf, $value);
3931             } elsif ($opt eq 'balloon') {
3932                 # enable/disable balloning device is not hotpluggable
3933                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3934                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3935                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3936
3937                 # allow manual ballooning if shares is set to zero
3938                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3939                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3940                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3941                 }
3942             } elsif ($opt =~ m/^net(\d+)$/) {
3943                 # some changes can be done without hotplug
3944                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3945                                     $vmid, $opt, $value);
3946             } elsif (valid_drivename($opt)) {
3947                 # some changes can be done without hotplug
3948                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3949                                      $vmid, $opt, $value, 1);
3950             } elsif ($opt =~ m/^memory$/) { #dimms
3951                 die "skip\n" if !$hotplug_features->{memory};
3952                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3953             } else {
3954                 die "skip\n";  # skip non-hot-pluggable options
3955             }
3956         };
3957         if (my $err = $@) {
3958             &$add_error($opt, $err) if $err ne "skip\n";
3959         } else {
3960             # save new config if hotplug was successful
3961             $conf->{$opt} = $value;
3962             delete $conf->{pending}->{$opt};
3963             update_config_nolock($vmid, $conf, 1);
3964             $conf = load_config($vmid); # update/reload
3965         }
3966     }
3967 }
3968
3969 sub vmconfig_apply_pending {
3970     my ($vmid, $conf, $storecfg) = @_;
3971
3972     # cold plug
3973
3974     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3975     foreach my $opt (@delete) { # delete
3976         die "internal error" if $opt =~ m/^unused/;
3977         $conf = load_config($vmid); # update/reload
3978         if (!defined($conf->{$opt})) {
3979             vmconfig_undelete_pending_option($conf, $opt);
3980             update_config_nolock($vmid, $conf, 1);
3981         } elsif (valid_drivename($opt)) {
3982             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3983             vmconfig_undelete_pending_option($conf, $opt);
3984             delete $conf->{$opt};
3985             update_config_nolock($vmid, $conf, 1);
3986         } else {
3987             vmconfig_undelete_pending_option($conf, $opt);
3988             delete $conf->{$opt};
3989             update_config_nolock($vmid, $conf, 1);
3990         }
3991     }
3992
3993     $conf = load_config($vmid); # update/reload
3994
3995     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3996         $conf = load_config($vmid); # update/reload
3997
3998         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3999             # skip if nothing changed
4000         } elsif (valid_drivename($opt)) {
4001             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4002                 if defined($conf->{$opt});
4003             $conf->{$opt} = $conf->{pending}->{$opt};
4004         } else {
4005             $conf->{$opt} = $conf->{pending}->{$opt};
4006         }
4007
4008         delete $conf->{pending}->{$opt};
4009         update_config_nolock($vmid, $conf, 1);
4010     }
4011 }
4012
4013 my $safe_num_ne = sub {
4014     my ($a, $b) = @_;
4015
4016     return 0 if !defined($a) && !defined($b);
4017     return 1 if !defined($a);
4018     return 1 if !defined($b);
4019
4020     return $a != $b;
4021 };
4022
4023 my $safe_string_ne = sub {
4024     my ($a, $b) = @_;
4025
4026     return 0 if !defined($a) && !defined($b);
4027     return 1 if !defined($a);
4028     return 1 if !defined($b);
4029
4030     return $a ne $b;
4031 };
4032
4033 sub vmconfig_update_net {
4034     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4035
4036     my $newnet = parse_net($value);
4037
4038     if ($conf->{$opt}) {
4039         my $oldnet = parse_net($conf->{$opt});
4040
4041         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4042             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4043             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4044             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4045
4046             # for non online change, we try to hot-unplug
4047             die "skip\n" if !$hotplug;
4048             vm_deviceunplug($vmid, $conf, $opt);
4049         } else {
4050
4051             die "internal error" if $opt !~ m/net(\d+)/;
4052             my $iface = "tap${vmid}i$1";
4053
4054             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4055                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4056             }
4057
4058             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4059                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4060                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4061                 PVE::Network::tap_unplug($iface);
4062                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4063             }
4064
4065             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4066                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4067             }
4068
4069             return 1;
4070         }
4071     }
4072
4073     if ($hotplug) {
4074         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4075     } else {
4076         die "skip\n";
4077     }
4078 }
4079
4080 sub vmconfig_update_disk {
4081     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4082
4083     # fixme: do we need force?
4084
4085     my $drive = parse_drive($opt, $value);
4086
4087     if ($conf->{$opt}) {
4088
4089         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4090
4091             my $media = $drive->{media} || 'disk';
4092             my $oldmedia = $old_drive->{media} || 'disk';
4093             die "unable to change media type\n" if $media ne $oldmedia;
4094
4095             if (!drive_is_cdrom($old_drive)) {
4096
4097                 if ($drive->{file} ne $old_drive->{file}) {
4098
4099                     die "skip\n" if !$hotplug;
4100
4101                     # unplug and register as unused
4102                     vm_deviceunplug($vmid, $conf, $opt);
4103                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4104
4105                 } else {
4106                     # update existing disk
4107
4108                     # skip non hotpluggable value
4109                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4110                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4111                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4112                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4113                         die "skip\n";
4114                     }
4115
4116                     # apply throttle
4117                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4118                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4119                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4120                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4121                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4122                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4123                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4124                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4125                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4126                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4127                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4128                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4129
4130                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4131                                                    ($drive->{mbps} || 0)*1024*1024,
4132                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4133                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4134                                                    $drive->{iops} || 0,
4135                                                    $drive->{iops_rd} || 0,
4136                                                    $drive->{iops_wr} || 0,
4137                                                    ($drive->{mbps_max} || 0)*1024*1024,
4138                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4139                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4140                                                    $drive->{iops_max} || 0,
4141                                                    $drive->{iops_rd_max} || 0,
4142                                                    $drive->{iops_wr_max} || 0);
4143
4144                     }
4145
4146                     return 1;
4147                 }
4148
4149             } else { # cdrom
4150
4151                 if ($drive->{file} eq 'none') {
4152                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4153                 } else {
4154                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4155                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4156                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4157                 }
4158
4159                 return 1;
4160             }
4161         }
4162     }
4163
4164     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4165     # hotplug new disks
4166     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4167 }
4168
4169 sub vm_start {
4170     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4171
4172     lock_config($vmid, sub {
4173         my $conf = load_config($vmid, $migratedfrom);
4174
4175         die "you can't start a vm if it's a template\n" if is_template($conf);
4176
4177         check_lock($conf) if !$skiplock;
4178
4179         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4180
4181         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4182             vmconfig_apply_pending($vmid, $conf, $storecfg);
4183             $conf = load_config($vmid); # update/reload
4184         }
4185
4186         my $defaults = load_defaults();
4187
4188         # set environment variable useful inside network script
4189         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4190
4191         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4192
4193         my $migrate_port = 0;
4194         my $migrate_uri;
4195         if ($statefile) {
4196             if ($statefile eq 'tcp') {
4197                 my $localip = "localhost";
4198                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4199                 my $nodename = PVE::INotify::nodename();
4200                 if ($datacenterconf->{migration_unsecure}) {
4201                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4202                 }
4203                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4204                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4205                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4206                 push @$cmd, '-incoming', $migrate_uri;
4207                 push @$cmd, '-S';
4208             } else {
4209                 push @$cmd, '-loadstate', $statefile;
4210             }
4211         } elsif ($paused) {
4212             push @$cmd, '-S';
4213         }
4214
4215         # host pci devices
4216         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4217           my $d = parse_hostpci($conf->{"hostpci$i"});
4218           next if !$d;
4219           my $pcidevices = $d->{pciid};
4220           foreach my $pcidevice (@$pcidevices) {
4221                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4222
4223                 my $info = pci_device_info("0000:$pciid");
4224                 die "IOMMU not present\n" if !check_iommu_support();
4225                 die "no pci device info for device '$pciid'\n" if !$info;
4226
4227                 if ($d->{driver} && $d->{driver} eq "vfio") {
4228                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4229                 } else {
4230                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4231                 }
4232
4233                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4234           }
4235         }
4236
4237         PVE::Storage::activate_volumes($storecfg, $vollist);
4238
4239         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4240                     umask => 0077); };
4241         my $err = $@;
4242         die "start failed: $err" if $err;
4243
4244         print "migration listens on $migrate_uri\n" if $migrate_uri;
4245
4246         if ($statefile && $statefile ne 'tcp')  {
4247             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4248             warn $@ if $@;
4249         }
4250
4251         if ($migratedfrom) {
4252
4253             eval {
4254                 set_migration_caps($vmid);
4255             };
4256             warn $@ if $@;
4257
4258             if ($spice_port) {
4259                 print "spice listens on port $spice_port\n";
4260                 if ($spice_ticket) {
4261                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4262                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4263                 }
4264             }
4265
4266         } else {
4267
4268             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4269                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4270                     if $conf->{balloon};
4271             }
4272
4273             foreach my $opt (keys %$conf) {
4274                 next if $opt !~  m/^net\d+$/;
4275                 my $nicconf = parse_net($conf->{$opt});
4276                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4277             }
4278         }
4279
4280         vm_mon_cmd_nocheck($vmid, 'qom-set',
4281                     path => "machine/peripheral/balloon0",
4282                     property => "guest-stats-polling-interval",
4283                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4284
4285     });
4286 }
4287
4288 sub vm_mon_cmd {
4289     my ($vmid, $execute, %params) = @_;
4290
4291     my $cmd = { execute => $execute, arguments => \%params };
4292     vm_qmp_command($vmid, $cmd);
4293 }
4294
4295 sub vm_mon_cmd_nocheck {
4296     my ($vmid, $execute, %params) = @_;
4297
4298     my $cmd = { execute => $execute, arguments => \%params };
4299     vm_qmp_command($vmid, $cmd, 1);
4300 }
4301
4302 sub vm_qmp_command {
4303     my ($vmid, $cmd, $nocheck) = @_;
4304
4305     my $res;
4306
4307     my $timeout;
4308     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4309         $timeout = $cmd->{arguments}->{timeout};
4310         delete $cmd->{arguments}->{timeout};
4311     }
4312
4313     eval {
4314         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4315         my $sname = qmp_socket($vmid);
4316         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4317             my $qmpclient = PVE::QMPClient->new();
4318
4319             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4320         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4321             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4322                 if scalar(%{$cmd->{arguments}});
4323             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4324         } else {
4325             die "unable to open monitor socket\n";
4326         }
4327     };
4328     if (my $err = $@) {
4329         syslog("err", "VM $vmid qmp command failed - $err");
4330         die $err;
4331     }
4332
4333     return $res;
4334 }
4335
4336 sub vm_human_monitor_command {
4337     my ($vmid, $cmdline) = @_;
4338
4339     my $res;
4340
4341     my $cmd = {
4342         execute => 'human-monitor-command',
4343         arguments => { 'command-line' => $cmdline},
4344     };
4345
4346     return vm_qmp_command($vmid, $cmd);
4347 }
4348
4349 sub vm_commandline {
4350     my ($storecfg, $vmid) = @_;
4351
4352     my $conf = load_config($vmid);
4353
4354     my $defaults = load_defaults();
4355
4356     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4357
4358     return join(' ', @$cmd);
4359 }
4360
4361 sub vm_reset {
4362     my ($vmid, $skiplock) = @_;
4363
4364     lock_config($vmid, sub {
4365
4366         my $conf = load_config($vmid);
4367
4368         check_lock($conf) if !$skiplock;
4369
4370         vm_mon_cmd($vmid, "system_reset");
4371     });
4372 }
4373
4374 sub get_vm_volumes {
4375     my ($conf) = @_;
4376
4377     my $vollist = [];
4378     foreach_volid($conf, sub {
4379         my ($volid, $is_cdrom) = @_;
4380
4381         return if $volid =~ m|^/|;
4382
4383         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4384         return if !$sid;
4385
4386         push @$vollist, $volid;
4387     });
4388
4389     return $vollist;
4390 }
4391
4392 sub vm_stop_cleanup {
4393     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4394
4395     eval {
4396         fairsched_rmnod($vmid); # try to destroy group
4397
4398         if (!$keepActive) {
4399             my $vollist = get_vm_volumes($conf);
4400             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4401         }
4402
4403         foreach my $ext (qw(mon qmp pid vnc qga)) {
4404             unlink "/var/run/qemu-server/${vmid}.$ext";
4405         }
4406
4407         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4408     };
4409     warn $@ if $@; # avoid errors - just warn
4410 }
4411
4412 # Note: use $nockeck to skip tests if VM configuration file exists.
4413 # We need that when migration VMs to other nodes (files already moved)
4414 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4415 sub vm_stop {
4416     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4417
4418     $force = 1 if !defined($force) && !$shutdown;
4419
4420     if ($migratedfrom){
4421         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4422         kill 15, $pid if $pid;
4423         my $conf = load_config($vmid, $migratedfrom);
4424         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4425         return;
4426     }
4427
4428     lock_config($vmid, sub {
4429
4430         my $pid = check_running($vmid, $nocheck);
4431         return if !$pid;
4432
4433         my $conf;
4434         if (!$nocheck) {
4435             $conf = load_config($vmid);
4436             check_lock($conf) if !$skiplock;
4437             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4438                 my $opts = parse_startup($conf->{startup});
4439                 $timeout = $opts->{down} if $opts->{down};
4440             }
4441         }
4442
4443         $timeout = 60 if !defined($timeout);
4444
4445         eval {
4446             if ($shutdown) {
4447                 if (defined($conf) && $conf->{agent}) {
4448                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4449                 } else {
4450                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4451                 }
4452             } else {
4453                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4454             }
4455         };
4456         my $err = $@;
4457
4458         if (!$err) {
4459             my $count = 0;
4460             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4461                 $count++;
4462                 sleep 1;
4463             }
4464
4465             if ($count >= $timeout) {
4466                 if ($force) {
4467                     warn "VM still running - terminating now with SIGTERM\n";
4468                     kill 15, $pid;
4469                 } else {
4470                     die "VM quit/powerdown failed - got timeout\n";
4471                 }
4472             } else {
4473                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4474                 return;
4475             }
4476         } else {
4477             if ($force) {
4478                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4479                 kill 15, $pid;
4480             } else {
4481                 die "VM quit/powerdown failed\n";
4482             }
4483         }
4484
4485         # wait again
4486         $timeout = 10;
4487
4488         my $count = 0;
4489         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4490             $count++;
4491             sleep 1;
4492         }
4493
4494         if ($count >= $timeout) {
4495             warn "VM still running - terminating now with SIGKILL\n";
4496             kill 9, $pid;
4497             sleep 1;
4498         }
4499
4500         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4501    });
4502 }
4503
4504 sub vm_suspend {
4505     my ($vmid, $skiplock) = @_;
4506
4507     lock_config($vmid, sub {
4508
4509         my $conf = load_config($vmid);
4510
4511         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4512
4513         vm_mon_cmd($vmid, "stop");
4514     });
4515 }
4516
4517 sub vm_resume {
4518     my ($vmid, $skiplock) = @_;
4519
4520     lock_config($vmid, sub {
4521
4522         my $conf = load_config($vmid);
4523
4524         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4525
4526         vm_mon_cmd($vmid, "cont");
4527     });
4528 }
4529
4530 sub vm_sendkey {
4531     my ($vmid, $skiplock, $key) = @_;
4532
4533     lock_config($vmid, sub {
4534
4535         my $conf = load_config($vmid);
4536
4537         # there is no qmp command, so we use the human monitor command
4538         vm_human_monitor_command($vmid, "sendkey $key");
4539     });
4540 }
4541
4542 sub vm_destroy {
4543     my ($storecfg, $vmid, $skiplock) = @_;
4544
4545     lock_config($vmid, sub {
4546
4547         my $conf = load_config($vmid);
4548
4549         check_lock($conf) if !$skiplock;
4550
4551         if (!check_running($vmid)) {
4552             fairsched_rmnod($vmid); # try to destroy group
4553             destroy_vm($storecfg, $vmid);
4554         } else {
4555             die "VM $vmid is running - destroy failed\n";
4556         }
4557     });
4558 }
4559
4560 # pci helpers
4561
4562 sub file_write {
4563     my ($filename, $buf) = @_;
4564
4565     my $fh = IO::File->new($filename, "w");
4566     return undef if !$fh;
4567
4568     my $res = print $fh $buf;
4569
4570     $fh->close();
4571
4572     return $res;
4573 }
4574
4575 sub pci_device_info {
4576     my ($name) = @_;
4577
4578     my $res;
4579
4580     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4581     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4582
4583     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4584     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4585
4586     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4587     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4588
4589     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4590     return undef if !defined($product) || $product !~ s/^0x//;
4591
4592     $res = {
4593         name => $name,
4594         vendor => $vendor,
4595         product => $product,
4596         domain => $domain,
4597         bus => $bus,
4598         slot => $slot,
4599         func => $func,
4600         irq => $irq,
4601         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4602     };
4603
4604     return $res;
4605 }
4606
4607 sub pci_dev_reset {
4608     my ($dev) = @_;
4609
4610     my $name = $dev->{name};
4611
4612     my $fn = "$pcisysfs/devices/$name/reset";
4613
4614     return file_write($fn, "1");
4615 }
4616
4617 sub pci_dev_bind_to_stub {
4618     my ($dev) = @_;
4619
4620     my $name = $dev->{name};
4621
4622     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4623     return 1 if -d $testdir;
4624
4625     my $data = "$dev->{vendor} $dev->{product}";
4626     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4627
4628     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4629     if (!file_write($fn, $name)) {
4630         return undef if -f $fn;
4631     }
4632
4633     $fn = "$pcisysfs/drivers/pci-stub/bind";
4634     if (! -d $testdir) {
4635         return undef if !file_write($fn, $name);
4636     }
4637
4638     return -d $testdir;
4639 }
4640
4641 sub pci_dev_bind_to_vfio {
4642     my ($dev) = @_;
4643
4644     my $name = $dev->{name};
4645
4646     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4647
4648     if (!-d $vfio_basedir) {
4649         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4650     }
4651     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4652
4653     my $testdir = "$vfio_basedir/$name";
4654     return 1 if -d $testdir;
4655
4656     my $data = "$dev->{vendor} $dev->{product}";
4657     return undef if !file_write("$vfio_basedir/new_id", $data);
4658
4659     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4660     if (!file_write($fn, $name)) {
4661         return undef if -f $fn;
4662     }
4663
4664     $fn = "$vfio_basedir/bind";
4665     if (! -d $testdir) {
4666         return undef if !file_write($fn, $name);
4667     }
4668
4669     return -d $testdir;
4670 }
4671
4672 sub pci_dev_group_bind_to_vfio {
4673     my ($pciid) = @_;
4674
4675     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4676
4677     if (!-d $vfio_basedir) {
4678         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4679     }
4680     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4681
4682     # get IOMMU group devices
4683     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4684       my @devs = grep /^0000:/, readdir($D);
4685     closedir($D);
4686
4687     foreach my $pciid (@devs) {
4688         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4689
4690         # pci bridges, switches or root ports are not supported
4691         # they have a pci_bus subdirectory so skip them
4692         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4693
4694         my $info = pci_device_info($1);
4695         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4696     }
4697
4698     return 1;
4699 }
4700
4701 sub print_pci_addr {
4702     my ($id, $bridges) = @_;
4703
4704     my $res = '';
4705     my $devices = {
4706         piix3 => { bus => 0, addr => 1 },
4707         #addr2 : first videocard
4708         balloon0 => { bus => 0, addr => 3 },
4709         watchdog => { bus => 0, addr => 4 },
4710         scsihw0 => { bus => 0, addr => 5 },
4711         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4712         scsihw1 => { bus => 0, addr => 6 },
4713         ahci0 => { bus => 0, addr => 7 },
4714         qga0 => { bus => 0, addr => 8 },
4715         spice => { bus => 0, addr => 9 },
4716         virtio0 => { bus => 0, addr => 10 },
4717         virtio1 => { bus => 0, addr => 11 },
4718         virtio2 => { bus => 0, addr => 12 },
4719         virtio3 => { bus => 0, addr => 13 },
4720         virtio4 => { bus => 0, addr => 14 },
4721         virtio5 => { bus => 0, addr => 15 },
4722         hostpci0 => { bus => 0, addr => 16 },
4723         hostpci1 => { bus => 0, addr => 17 },
4724         net0 => { bus => 0, addr => 18 },
4725         net1 => { bus => 0, addr => 19 },
4726         net2 => { bus => 0, addr => 20 },
4727         net3 => { bus => 0, addr => 21 },
4728         net4 => { bus => 0, addr => 22 },
4729         net5 => { bus => 0, addr => 23 },
4730         vga1 => { bus => 0, addr => 24 },
4731         vga2 => { bus => 0, addr => 25 },
4732         vga3 => { bus => 0, addr => 26 },
4733         hostpci2 => { bus => 0, addr => 27 },
4734         hostpci3 => { bus => 0, addr => 28 },
4735         #addr29 : usb-host (pve-usb.cfg)
4736         'pci.1' => { bus => 0, addr => 30 },
4737         'pci.2' => { bus => 0, addr => 31 },
4738         'net6' => { bus => 1, addr => 1 },
4739         'net7' => { bus => 1, addr => 2 },
4740         'net8' => { bus => 1, addr => 3 },
4741         'net9' => { bus => 1, addr => 4 },
4742         'net10' => { bus => 1, addr => 5 },
4743         'net11' => { bus => 1, addr => 6 },
4744         'net12' => { bus => 1, addr => 7 },
4745         'net13' => { bus => 1, addr => 8 },
4746         'net14' => { bus => 1, addr => 9 },
4747         'net15' => { bus => 1, addr => 10 },
4748         'net16' => { bus => 1, addr => 11 },
4749         'net17' => { bus => 1, addr => 12 },
4750         'net18' => { bus => 1, addr => 13 },
4751         'net19' => { bus => 1, addr => 14 },
4752         'net20' => { bus => 1, addr => 15 },
4753         'net21' => { bus => 1, addr => 16 },
4754         'net22' => { bus => 1, addr => 17 },
4755         'net23' => { bus => 1, addr => 18 },
4756         'net24' => { bus => 1, addr => 19 },
4757         'net25' => { bus => 1, addr => 20 },
4758         'net26' => { bus => 1, addr => 21 },
4759         'net27' => { bus => 1, addr => 22 },
4760         'net28' => { bus => 1, addr => 23 },
4761         'net29' => { bus => 1, addr => 24 },
4762         'net30' => { bus => 1, addr => 25 },
4763         'net31' => { bus => 1, addr => 26 },
4764         'virtio6' => { bus => 2, addr => 1 },
4765         'virtio7' => { bus => 2, addr => 2 },
4766         'virtio8' => { bus => 2, addr => 3 },
4767         'virtio9' => { bus => 2, addr => 4 },
4768         'virtio10' => { bus => 2, addr => 5 },
4769         'virtio11' => { bus => 2, addr => 6 },
4770         'virtio12' => { bus => 2, addr => 7 },
4771         'virtio13' => { bus => 2, addr => 8 },
4772         'virtio14' => { bus => 2, addr => 9 },
4773         'virtio15' => { bus => 2, addr => 10 },
4774         'virtioscsi0' => { bus => 3, addr => 1 },
4775         'virtioscsi1' => { bus => 3, addr => 2 },
4776         'virtioscsi2' => { bus => 3, addr => 3 },
4777         'virtioscsi3' => { bus => 3, addr => 4 },
4778         'virtioscsi4' => { bus => 3, addr => 5 },
4779         'virtioscsi5' => { bus => 3, addr => 6 },
4780         'virtioscsi6' => { bus => 3, addr => 7 },
4781         'virtioscsi7' => { bus => 3, addr => 8 },
4782         'virtioscsi8' => { bus => 3, addr => 9 },
4783         'virtioscsi9' => { bus => 3, addr => 10 },
4784         'virtioscsi10' => { bus => 3, addr => 11 },
4785         'virtioscsi11' => { bus => 3, addr => 12 },
4786         'virtioscsi12' => { bus => 3, addr => 13 },
4787         'virtioscsi13' => { bus => 3, addr => 14 },
4788         'virtioscsi14' => { bus => 3, addr => 15 },
4789         'virtioscsi15' => { bus => 3, addr => 16 },
4790         'virtioscsi16' => { bus => 3, addr => 17 },
4791         'virtioscsi17' => { bus => 3, addr => 18 },
4792         'virtioscsi18' => { bus => 3, addr => 19 },
4793         'virtioscsi19' => { bus => 3, addr => 20 },
4794         'virtioscsi20' => { bus => 3, addr => 21 },
4795         'virtioscsi21' => { bus => 3, addr => 22 },
4796         'virtioscsi22' => { bus => 3, addr => 23 },
4797         'virtioscsi23' => { bus => 3, addr => 24 },
4798         'virtioscsi24' => { bus => 3, addr => 25 },
4799         'virtioscsi25' => { bus => 3, addr => 26 },
4800         'virtioscsi26' => { bus => 3, addr => 27 },
4801         'virtioscsi27' => { bus => 3, addr => 28 },
4802         'virtioscsi28' => { bus => 3, addr => 29 },
4803         'virtioscsi29' => { bus => 3, addr => 30 },
4804         'virtioscsi30' => { bus => 3, addr => 31 },
4805
4806     };
4807
4808     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4809            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4810            my $bus = $devices->{$id}->{bus};
4811            $res = ",bus=pci.$bus,addr=$addr";
4812            $bridges->{$bus} = 1 if $bridges;
4813     }
4814     return $res;
4815
4816 }
4817
4818 sub print_pcie_addr {
4819     my ($id) = @_;
4820
4821     my $res = '';
4822     my $devices = {
4823         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4824         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4825         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4826         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4827     };
4828
4829     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4830            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4831            my $bus = $devices->{$id}->{bus};
4832            $res = ",bus=$bus,addr=$addr";
4833     }
4834     return $res;
4835
4836 }
4837
4838 # vzdump restore implementaion
4839
4840 sub tar_archive_read_firstfile {
4841     my $archive = shift;
4842
4843     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4844
4845     # try to detect archive type first
4846     my $pid = open (TMP, "tar tf '$archive'|") ||
4847         die "unable to open file '$archive'\n";
4848     my $firstfile = <TMP>;
4849     kill 15, $pid;
4850     close TMP;
4851
4852     die "ERROR: archive contaions no data\n" if !$firstfile;
4853     chomp $firstfile;
4854
4855     return $firstfile;
4856 }
4857
4858 sub tar_restore_cleanup {
4859     my ($storecfg, $statfile) = @_;
4860
4861     print STDERR "starting cleanup\n";
4862
4863     if (my $fd = IO::File->new($statfile, "r")) {
4864         while (defined(my $line = <$fd>)) {
4865             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4866                 my $volid = $2;
4867                 eval {
4868                     if ($volid =~ m|^/|) {
4869                         unlink $volid || die 'unlink failed\n';
4870                     } else {
4871                         PVE::Storage::vdisk_free($storecfg, $volid);
4872                     }
4873                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4874                 };
4875                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4876             } else {
4877                 print STDERR "unable to parse line in statfile - $line";
4878             }
4879         }
4880         $fd->close();
4881     }
4882 }
4883
4884 sub restore_archive {
4885     my ($archive, $vmid, $user, $opts) = @_;
4886
4887     my $format = $opts->{format};
4888     my $comp;
4889
4890     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4891         $format = 'tar' if !$format;
4892         $comp = 'gzip';
4893     } elsif ($archive =~ m/\.tar$/) {
4894         $format = 'tar' if !$format;
4895     } elsif ($archive =~ m/.tar.lzo$/) {
4896         $format = 'tar' if !$format;
4897         $comp = 'lzop';
4898     } elsif ($archive =~ m/\.vma$/) {
4899         $format = 'vma' if !$format;
4900     } elsif ($archive =~ m/\.vma\.gz$/) {
4901         $format = 'vma' if !$format;
4902         $comp = 'gzip';
4903     } elsif ($archive =~ m/\.vma\.lzo$/) {
4904         $format = 'vma' if !$format;
4905         $comp = 'lzop';
4906     } else {
4907         $format = 'vma' if !$format; # default
4908     }
4909
4910     # try to detect archive format
4911     if ($format eq 'tar') {
4912         return restore_tar_archive($archive, $vmid, $user, $opts);
4913     } else {
4914         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4915     }
4916 }
4917
4918 sub restore_update_config_line {
4919     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4920
4921     return if $line =~ m/^\#qmdump\#/;
4922     return if $line =~ m/^\#vzdump\#/;
4923     return if $line =~ m/^lock:/;
4924     return if $line =~ m/^unused\d+:/;
4925     return if $line =~ m/^parent:/;
4926     return if $line =~ m/^template:/; # restored VM is never a template
4927
4928     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4929         # try to convert old 1.X settings
4930         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4931         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4932             my ($model, $macaddr) = split(/\=/, $devconfig);
4933             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4934             my $net = {
4935                 model => $model,
4936                 bridge => "vmbr$ind",
4937                 macaddr => $macaddr,
4938             };
4939             my $netstr = print_net($net);
4940
4941             print $outfd "net$cookie->{netcount}: $netstr\n";
4942             $cookie->{netcount}++;
4943         }
4944     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4945         my ($id, $netstr) = ($1, $2);
4946         my $net = parse_net($netstr);
4947         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4948         $netstr = print_net($net);
4949         print $outfd "$id: $netstr\n";
4950     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4951         my $virtdev = $1;
4952         my $value = $3;
4953         if ($line =~ m/backup=no/) {
4954             print $outfd "#$line";
4955         } elsif ($virtdev && $map->{$virtdev}) {
4956             my $di = parse_drive($virtdev, $value);
4957             delete $di->{format}; # format can change on restore
4958             $di->{file} = $map->{$virtdev};
4959             $value = print_drive($vmid, $di);
4960             print $outfd "$virtdev: $value\n";
4961         } else {
4962             print $outfd $line;
4963         }
4964     } else {
4965         print $outfd $line;
4966     }
4967 }
4968
4969 sub scan_volids {
4970     my ($cfg, $vmid) = @_;
4971
4972     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4973
4974     my $volid_hash = {};
4975     foreach my $storeid (keys %$info) {
4976         foreach my $item (@{$info->{$storeid}}) {
4977             next if !($item->{volid} && $item->{size});
4978             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4979             $volid_hash->{$item->{volid}} = $item;
4980         }
4981     }
4982
4983     return $volid_hash;
4984 }
4985
4986 sub get_used_paths {
4987     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4988
4989     my $used_path = {};
4990
4991     my $scan_config = sub {
4992         my ($cref, $snapname) = @_;
4993
4994         foreach my $key (keys %$cref) {
4995             my $value = $cref->{$key};
4996             if (valid_drivename($key)) {
4997                 next if $skip_drive && $key eq $skip_drive;
4998                 my $drive = parse_drive($key, $value);
4999                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5000                 if ($drive->{file} =~ m!^/!) {
5001                     $used_path->{$drive->{file}}++; # = 1;
5002                 } else {
5003                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5004                     next if !$storeid;
5005                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5006                     next if !$scfg;
5007                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5008                     $used_path->{$path}++; # = 1;
5009                 }
5010             }
5011         }
5012     };
5013
5014     &$scan_config($conf);
5015
5016     undef $skip_drive;
5017
5018     if ($scan_snapshots) {
5019         foreach my $snapname (keys %{$conf->{snapshots}}) {
5020             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5021         }
5022     }
5023
5024     return $used_path;
5025 }
5026
5027 sub update_disksize {
5028     my ($vmid, $conf, $volid_hash) = @_;
5029
5030     my $changes;
5031
5032     my $used = {};
5033
5034     # Note: it is allowed to define multiple storages with same path (alias), so
5035     # we need to check both 'volid' and real 'path' (two different volid can point
5036     # to the same path).
5037
5038     my $usedpath = {};
5039
5040     # update size info
5041     foreach my $opt (keys %$conf) {
5042         if (valid_drivename($opt)) {
5043             my $drive = parse_drive($opt, $conf->{$opt});
5044             my $volid = $drive->{file};
5045             next if !$volid;
5046
5047             $used->{$volid} = 1;
5048             if ($volid_hash->{$volid} &&
5049                 (my $path = $volid_hash->{$volid}->{path})) {
5050                 $usedpath->{$path} = 1;
5051             }
5052
5053             next if drive_is_cdrom($drive);
5054             next if !$volid_hash->{$volid};
5055
5056             $drive->{size} = $volid_hash->{$volid}->{size};
5057             my $new = print_drive($vmid, $drive);
5058             if ($new ne $conf->{$opt}) {
5059                 $changes = 1;
5060                 $conf->{$opt} = $new;
5061             }
5062         }
5063     }
5064
5065     # remove 'unusedX' entry if volume is used
5066     foreach my $opt (keys %$conf) {
5067         next if $opt !~ m/^unused\d+$/;
5068         my $volid = $conf->{$opt};
5069         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5070         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5071             $changes = 1;
5072             delete $conf->{$opt};
5073         }
5074     }
5075
5076     foreach my $volid (sort keys %$volid_hash) {
5077         next if $volid =~ m/vm-$vmid-state-/;
5078         next if $used->{$volid};
5079         my $path = $volid_hash->{$volid}->{path};
5080         next if !$path; # just to be sure
5081         next if $usedpath->{$path};
5082         $changes = 1;
5083         add_unused_volume($conf, $volid);
5084         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5085     }
5086
5087     return $changes;
5088 }
5089
5090 sub rescan {
5091     my ($vmid, $nolock) = @_;
5092
5093     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5094
5095     my $volid_hash = scan_volids($cfg, $vmid);
5096
5097     my $updatefn =  sub {
5098         my ($vmid) = @_;
5099
5100         my $conf = load_config($vmid);
5101
5102         check_lock($conf);
5103
5104         my $vm_volids = {};
5105         foreach my $volid (keys %$volid_hash) {
5106             my $info = $volid_hash->{$volid};
5107             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5108         }
5109
5110         my $changes = update_disksize($vmid, $conf, $vm_volids);
5111
5112         update_config_nolock($vmid, $conf, 1) if $changes;
5113     };
5114
5115     if (defined($vmid)) {
5116         if ($nolock) {
5117             &$updatefn($vmid);
5118         } else {
5119             lock_config($vmid, $updatefn, $vmid);
5120         }
5121     } else {
5122         my $vmlist = config_list();
5123         foreach my $vmid (keys %$vmlist) {
5124             if ($nolock) {
5125                 &$updatefn($vmid);
5126             } else {
5127                 lock_config($vmid, $updatefn, $vmid);
5128             }
5129         }
5130     }
5131 }
5132
5133 sub restore_vma_archive {
5134     my ($archive, $vmid, $user, $opts, $comp) = @_;
5135
5136     my $input = $archive eq '-' ? "<&STDIN" : undef;
5137     my $readfrom = $archive;
5138
5139     my $uncomp = '';
5140     if ($comp) {
5141         $readfrom = '-';
5142         my $qarchive = PVE::Tools::shellquote($archive);
5143         if ($comp eq 'gzip') {
5144             $uncomp = "zcat $qarchive|";
5145         } elsif ($comp eq 'lzop') {
5146             $uncomp = "lzop -d -c $qarchive|";
5147         } else {
5148             die "unknown compression method '$comp'\n";
5149         }
5150
5151     }
5152
5153     my $tmpdir = "/var/tmp/vzdumptmp$$";
5154     rmtree $tmpdir;
5155
5156     # disable interrupts (always do cleanups)
5157     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5158         warn "got interrupt - ignored\n";
5159     };
5160
5161     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5162     POSIX::mkfifo($mapfifo, 0600);
5163     my $fifofh;
5164
5165     my $openfifo = sub {
5166         open($fifofh, '>', $mapfifo) || die $!;
5167     };
5168
5169     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5170
5171     my $oldtimeout;
5172     my $timeout = 5;
5173
5174     my $devinfo = {};
5175
5176     my $rpcenv = PVE::RPCEnvironment::get();
5177
5178     my $conffile = config_file($vmid);
5179     my $tmpfn = "$conffile.$$.tmp";
5180
5181     # Note: $oldconf is undef if VM does not exists
5182     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5183
5184     my $print_devmap = sub {
5185         my $virtdev_hash = {};
5186
5187         my $cfgfn = "$tmpdir/qemu-server.conf";
5188
5189         # we can read the config - that is already extracted
5190         my $fh = IO::File->new($cfgfn, "r") ||
5191             "unable to read qemu-server.conf - $!\n";
5192
5193         while (defined(my $line = <$fh>)) {
5194             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5195                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5196                 die "archive does not contain data for drive '$virtdev'\n"
5197                     if !$devinfo->{$devname};
5198                 if (defined($opts->{storage})) {
5199                     $storeid = $opts->{storage} || 'local';
5200                 } elsif (!$storeid) {
5201                     $storeid = 'local';
5202                 }
5203                 $format = 'raw' if !$format;
5204                 $devinfo->{$devname}->{devname} = $devname;
5205                 $devinfo->{$devname}->{virtdev} = $virtdev;
5206                 $devinfo->{$devname}->{format} = $format;
5207                 $devinfo->{$devname}->{storeid} = $storeid;
5208
5209                 # check permission on storage
5210                 my $pool = $opts->{pool}; # todo: do we need that?
5211                 if ($user ne 'root@pam') {
5212                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5213                 }
5214
5215                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5216             }
5217         }
5218
5219         foreach my $devname (keys %$devinfo) {
5220             die "found no device mapping information for device '$devname'\n"
5221                 if !$devinfo->{$devname}->{virtdev};
5222         }
5223
5224         my $cfg = cfs_read_file('storage.cfg');
5225
5226         # create empty/temp config
5227         if ($oldconf) {
5228             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5229             foreach_drive($oldconf, sub {
5230                 my ($ds, $drive) = @_;
5231
5232                 return if drive_is_cdrom($drive);
5233
5234                 my $volid = $drive->{file};
5235
5236                 return if !$volid || $volid =~ m|^/|;
5237
5238                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5239                 return if !$path || !$owner || ($owner != $vmid);
5240
5241                 # Note: only delete disk we want to restore
5242                 # other volumes will become unused
5243                 if ($virtdev_hash->{$ds}) {
5244                     PVE::Storage::vdisk_free($cfg, $volid);
5245                 }
5246             });
5247         }
5248
5249         my $map = {};
5250         foreach my $virtdev (sort keys %$virtdev_hash) {
5251             my $d = $virtdev_hash->{$virtdev};
5252             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5253             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5254
5255             # test if requested format is supported
5256             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5257             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5258             $d->{format} = $defFormat if !$supported;
5259
5260             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5261                                                   $d->{format}, undef, $alloc_size);
5262             print STDERR "new volume ID is '$volid'\n";
5263             $d->{volid} = $volid;
5264             my $path = PVE::Storage::path($cfg, $volid);
5265
5266             my $write_zeros = 1;
5267             # fixme: what other storages types initialize volumes with zero?
5268             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5269                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5270                 $write_zeros = 0;
5271             }
5272
5273             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5274
5275             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5276             $map->{$virtdev} = $volid;
5277         }
5278
5279         $fh->seek(0, 0) || die "seek failed - $!\n";
5280
5281         my $outfd = new IO::File ($tmpfn, "w") ||
5282             die "unable to write config for VM $vmid\n";
5283
5284         my $cookie = { netcount => 0 };
5285         while (defined(my $line = <$fh>)) {
5286             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5287         }
5288
5289         $fh->close();
5290         $outfd->close();
5291     };
5292
5293     eval {
5294         # enable interrupts
5295         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5296             die "interrupted by signal\n";
5297         };
5298         local $SIG{ALRM} = sub { die "got timeout\n"; };
5299
5300         $oldtimeout = alarm($timeout);
5301
5302         my $parser = sub {
5303             my $line = shift;
5304
5305             print "$line\n";
5306
5307             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5308                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5309                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5310             } elsif ($line =~ m/^CTIME: /) {
5311                 # we correctly received the vma config, so we can disable
5312                 # the timeout now for disk allocation (set to 10 minutes, so
5313                 # that we always timeout if something goes wrong)
5314                 alarm(600);
5315                 &$print_devmap();
5316                 print $fifofh "done\n";
5317                 my $tmp = $oldtimeout || 0;
5318                 $oldtimeout = undef;
5319                 alarm($tmp);
5320                 close($fifofh);
5321             }
5322         };
5323
5324         print "restore vma archive: $cmd\n";
5325         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5326     };
5327     my $err = $@;
5328
5329     alarm($oldtimeout) if $oldtimeout;
5330
5331     unlink $mapfifo;
5332
5333     if ($err) {
5334         rmtree $tmpdir;
5335         unlink $tmpfn;
5336
5337         my $cfg = cfs_read_file('storage.cfg');
5338         foreach my $devname (keys %$devinfo) {
5339             my $volid = $devinfo->{$devname}->{volid};
5340             next if !$volid;
5341             eval {
5342                 if ($volid =~ m|^/|) {
5343                     unlink $volid || die 'unlink failed\n';
5344                 } else {
5345                     PVE::Storage::vdisk_free($cfg, $volid);
5346                 }
5347                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5348             };
5349             print STDERR "unable to cleanup '$volid' - $@" if $@;
5350         }
5351         die $err;
5352     }
5353
5354     rmtree $tmpdir;
5355
5356     rename($tmpfn, $conffile) ||
5357         die "unable to commit configuration file '$conffile'\n";
5358
5359     PVE::Cluster::cfs_update(); # make sure we read new file
5360
5361     eval { rescan($vmid, 1); };
5362     warn $@ if $@;
5363 }
5364
5365 sub restore_tar_archive {
5366     my ($archive, $vmid, $user, $opts) = @_;
5367
5368     if ($archive ne '-') {
5369         my $firstfile = tar_archive_read_firstfile($archive);
5370         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5371             if $firstfile ne 'qemu-server.conf';
5372     }
5373
5374     my $storecfg = cfs_read_file('storage.cfg');
5375
5376     # destroy existing data - keep empty config
5377     my $vmcfgfn = config_file($vmid);
5378     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5379
5380     my $tocmd = "/usr/lib/qemu-server/qmextract";
5381
5382     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5383     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5384     $tocmd .= ' --prealloc' if $opts->{prealloc};
5385     $tocmd .= ' --info' if $opts->{info};
5386
5387     # tar option "xf" does not autodetect compression when read from STDIN,
5388     # so we pipe to zcat
5389     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5390         PVE::Tools::shellquote("--to-command=$tocmd");
5391
5392     my $tmpdir = "/var/tmp/vzdumptmp$$";
5393     mkpath $tmpdir;
5394
5395     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5396     local $ENV{VZDUMP_VMID} = $vmid;
5397     local $ENV{VZDUMP_USER} = $user;
5398
5399     my $conffile = config_file($vmid);
5400     my $tmpfn = "$conffile.$$.tmp";
5401
5402     # disable interrupts (always do cleanups)
5403     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5404         print STDERR "got interrupt - ignored\n";
5405     };
5406
5407     eval {
5408         # enable interrupts
5409         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5410             die "interrupted by signal\n";
5411         };
5412
5413         if ($archive eq '-') {
5414             print "extracting archive from STDIN\n";
5415             run_command($cmd, input => "<&STDIN");
5416         } else {
5417             print "extracting archive '$archive'\n";
5418             run_command($cmd);
5419         }
5420
5421         return if $opts->{info};
5422
5423         # read new mapping
5424         my $map = {};
5425         my $statfile = "$tmpdir/qmrestore.stat";
5426         if (my $fd = IO::File->new($statfile, "r")) {
5427             while (defined (my $line = <$fd>)) {
5428                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5429                     $map->{$1} = $2 if $1;
5430                 } else {
5431                     print STDERR "unable to parse line in statfile - $line\n";
5432                 }
5433             }
5434             $fd->close();
5435         }
5436
5437         my $confsrc = "$tmpdir/qemu-server.conf";
5438
5439         my $srcfd = new IO::File($confsrc, "r") ||
5440             die "unable to open file '$confsrc'\n";
5441
5442         my $outfd = new IO::File ($tmpfn, "w") ||
5443             die "unable to write config for VM $vmid\n";
5444
5445         my $cookie = { netcount => 0 };
5446         while (defined (my $line = <$srcfd>)) {
5447             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5448         }
5449
5450         $srcfd->close();
5451         $outfd->close();
5452     };
5453     my $err = $@;
5454
5455     if ($err) {
5456
5457         unlink $tmpfn;
5458
5459         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5460
5461         die $err;
5462     }
5463
5464     rmtree $tmpdir;
5465
5466     rename $tmpfn, $conffile ||
5467         die "unable to commit configuration file '$conffile'\n";
5468
5469     PVE::Cluster::cfs_update(); # make sure we read new file
5470
5471     eval { rescan($vmid, 1); };
5472     warn $@ if $@;
5473 };
5474
5475
5476 # Internal snapshots
5477
5478 # NOTE: Snapshot create/delete involves several non-atomic
5479 # action, and can take a long time.
5480 # So we try to avoid locking the file and use 'lock' variable
5481 # inside the config file instead.
5482
5483 my $snapshot_copy_config = sub {
5484     my ($source, $dest) = @_;
5485
5486     foreach my $k (keys %$source) {
5487         next if $k eq 'snapshots';
5488         next if $k eq 'snapstate';
5489         next if $k eq 'snaptime';
5490         next if $k eq 'vmstate';
5491         next if $k eq 'lock';
5492         next if $k eq 'digest';
5493         next if $k eq 'description';
5494         next if $k =~ m/^unused\d+$/;
5495
5496         $dest->{$k} = $source->{$k};
5497     }
5498 };
5499
5500 my $snapshot_apply_config = sub {
5501     my ($conf, $snap) = @_;
5502
5503     # copy snapshot list
5504     my $newconf = {
5505         snapshots => $conf->{snapshots},
5506     };
5507
5508     # keep description and list of unused disks
5509     foreach my $k (keys %$conf) {
5510         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5511         $newconf->{$k} = $conf->{$k};
5512     }
5513
5514     &$snapshot_copy_config($snap, $newconf);
5515
5516     return $newconf;
5517 };
5518
5519 sub foreach_writable_storage {
5520     my ($conf, $func) = @_;
5521
5522     my $sidhash = {};
5523
5524     foreach my $ds (keys %$conf) {
5525         next if !valid_drivename($ds);
5526
5527         my $drive = parse_drive($ds, $conf->{$ds});
5528         next if !$drive;
5529         next if drive_is_cdrom($drive);
5530
5531         my $volid = $drive->{file};
5532
5533         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5534         $sidhash->{$sid} = $sid if $sid;
5535     }
5536
5537     foreach my $sid (sort keys %$sidhash) {
5538         &$func($sid);
5539     }
5540 }
5541
5542 my $alloc_vmstate_volid = sub {
5543     my ($storecfg, $vmid, $conf, $snapname) = @_;
5544
5545     # Note: we try to be smart when selecting a $target storage
5546
5547     my $target;
5548
5549     # search shared storage first
5550     foreach_writable_storage($conf, sub {
5551         my ($sid) = @_;
5552         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5553         return if !$scfg->{shared};
5554
5555         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5556     });
5557
5558     if (!$target) {
5559         # now search local storage
5560         foreach_writable_storage($conf, sub {
5561             my ($sid) = @_;
5562             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5563             return if $scfg->{shared};
5564
5565             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5566         });
5567     }
5568
5569     $target = 'local' if !$target;
5570
5571     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5572     # we abort live save after $conf->{memory}, so we need at max twice that space
5573     my $size = $conf->{memory}*2 + $driver_state_size;
5574
5575     my $name = "vm-$vmid-state-$snapname";
5576     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5577     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5578     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5579
5580     return $volid;
5581 };
5582
5583 my $snapshot_prepare = sub {
5584     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5585
5586     my $snap;
5587
5588     my $updatefn =  sub {
5589
5590         my $conf = load_config($vmid);
5591
5592         die "you can't take a snapshot if it's a template\n"
5593             if is_template($conf);
5594
5595         check_lock($conf);
5596
5597         $conf->{lock} = 'snapshot';
5598
5599         die "snapshot name '$snapname' already used\n"
5600             if defined($conf->{snapshots}->{$snapname});
5601
5602         my $storecfg = PVE::Storage::config();
5603         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5604
5605         $snap = $conf->{snapshots}->{$snapname} = {};
5606
5607         if ($save_vmstate && check_running($vmid)) {
5608             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5609         }
5610
5611         &$snapshot_copy_config($conf, $snap);
5612
5613         $snap->{snapstate} = "prepare";
5614         $snap->{snaptime} = time();
5615         $snap->{description} = $comment if $comment;
5616
5617         # always overwrite machine if we save vmstate. This makes sure we
5618         # can restore it later using correct machine type
5619         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5620
5621         update_config_nolock($vmid, $conf, 1);
5622     };
5623
5624     lock_config($vmid, $updatefn);
5625
5626     return $snap;
5627 };
5628
5629 my $snapshot_commit = sub {
5630     my ($vmid, $snapname) = @_;
5631
5632     my $updatefn = sub {
5633
5634         my $conf = load_config($vmid);
5635
5636         die "missing snapshot lock\n"
5637             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5638
5639         my $has_machine_config = defined($conf->{machine});
5640
5641         my $snap = $conf->{snapshots}->{$snapname};
5642
5643         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5644
5645         die "wrong snapshot state\n"
5646             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5647
5648         delete $snap->{snapstate};
5649         delete $conf->{lock};
5650
5651         my $newconf = &$snapshot_apply_config($conf, $snap);
5652
5653         delete $newconf->{machine} if !$has_machine_config;
5654
5655         $newconf->{parent} = $snapname;
5656
5657         update_config_nolock($vmid, $newconf, 1);
5658     };
5659
5660     lock_config($vmid, $updatefn);
5661 };
5662
5663 sub snapshot_rollback {
5664     my ($vmid, $snapname) = @_;
5665
5666     my $prepare = 1;
5667
5668     my $storecfg = PVE::Storage::config();
5669
5670     my $conf = load_config($vmid);
5671
5672     my $get_snapshot_config = sub {
5673
5674         die "you can't rollback if vm is a template\n" if is_template($conf);
5675
5676         my $res = $conf->{snapshots}->{$snapname};
5677
5678         die "snapshot '$snapname' does not exist\n" if !defined($res);
5679
5680         return $res;
5681     };
5682
5683     my $snap = &$get_snapshot_config();
5684
5685     foreach_drive($snap, sub {
5686         my ($ds, $drive) = @_;
5687
5688         return if drive_is_cdrom($drive);
5689
5690         my $volid = $drive->{file};
5691
5692         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5693     });
5694
5695     my $updatefn = sub {
5696
5697         $conf = load_config($vmid);
5698
5699         $snap = &$get_snapshot_config();
5700
5701         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5702             if $snap->{snapstate};
5703
5704         if ($prepare) {
5705             check_lock($conf);
5706             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5707         }
5708
5709         die "unable to rollback vm $vmid: vm is running\n"
5710             if check_running($vmid);
5711
5712         if ($prepare) {
5713             $conf->{lock} = 'rollback';
5714         } else {
5715             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5716             delete $conf->{lock};
5717         }
5718
5719         my $forcemachine;
5720
5721         if (!$prepare) {
5722             my $has_machine_config = defined($conf->{machine});
5723
5724             # copy snapshot config to current config
5725             $conf = &$snapshot_apply_config($conf, $snap);
5726             $conf->{parent} = $snapname;
5727
5728             # Note: old code did not store 'machine', so we try to be smart
5729             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5730             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5731             # we remove the 'machine' configuration if not explicitly specified
5732             # in the original config.
5733             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5734         }
5735
5736         update_config_nolock($vmid, $conf, 1);
5737
5738         if (!$prepare && $snap->{vmstate}) {
5739             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5740             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5741         }
5742     };
5743
5744     lock_config($vmid, $updatefn);
5745
5746     foreach_drive($snap, sub {
5747         my ($ds, $drive) = @_;
5748
5749         return if drive_is_cdrom($drive);
5750
5751         my $volid = $drive->{file};
5752         my $device = "drive-$ds";
5753
5754         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5755     });
5756
5757     $prepare = 0;
5758     lock_config($vmid, $updatefn);
5759 }
5760
5761 my $savevm_wait = sub {
5762     my ($vmid) = @_;
5763
5764     for(;;) {
5765         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5766         if (!$stat->{status}) {
5767             die "savevm not active\n";
5768         } elsif ($stat->{status} eq 'active') {
5769             sleep(1);
5770             next;
5771         } elsif ($stat->{status} eq 'completed') {
5772             last;
5773         } else {
5774             die "query-savevm returned status '$stat->{status}'\n";
5775         }
5776     }
5777 };
5778
5779 sub do_snapshots_with_qemu {
5780     my ($storecfg, $volid) = @_;
5781
5782     my $storage_name = PVE::Storage::parse_volume_id($volid);
5783
5784     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5785         return 1;
5786     }
5787
5788     if ($volid =~ m/\.(qcow2|qed)$/){
5789         return 1;
5790     }
5791
5792     return undef;
5793 }
5794
5795 sub snapshot_create {
5796     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5797
5798     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5799
5800     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5801
5802     my $config = load_config($vmid);
5803
5804     my $running = check_running($vmid);
5805
5806     my $freezefs = $running && $config->{agent};
5807     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5808
5809     my $drivehash = {};
5810
5811     if ($freezefs) {
5812         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5813         warn "guest-fsfreeze-freeze problems - $@" if $@;
5814     }
5815
5816     eval {
5817         # create internal snapshots of all drives
5818
5819         my $storecfg = PVE::Storage::config();
5820
5821         if ($running) {
5822             if ($snap->{vmstate}) {
5823                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5824                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5825                 &$savevm_wait($vmid);
5826             } else {
5827                 vm_mon_cmd($vmid, "savevm-start");
5828             }
5829         };
5830
5831         foreach_drive($snap, sub {
5832             my ($ds, $drive) = @_;
5833
5834             return if drive_is_cdrom($drive);
5835
5836             my $volid = $drive->{file};
5837             my $device = "drive-$ds";
5838
5839             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5840             $drivehash->{$ds} = 1;
5841        });
5842     };
5843     my $err = $@;
5844
5845     if ($running) {
5846         eval { vm_mon_cmd($vmid, "savevm-end")  };
5847         warn $@ if $@;
5848
5849         if ($freezefs) {
5850             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5851             warn "guest-fsfreeze-thaw problems - $@" if $@;
5852         }
5853
5854         # savevm-end is async, we need to wait
5855         for (;;) {
5856             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5857             if (!$stat->{bytes}) {
5858                 last;
5859             } else {
5860                 print "savevm not yet finished\n";
5861                 sleep(1);
5862                 next;
5863             }
5864         }
5865     }
5866
5867     if ($err) {
5868         warn "snapshot create failed: starting cleanup\n";
5869         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5870         warn $@ if $@;
5871         die $err;
5872     }
5873
5874     &$snapshot_commit($vmid, $snapname);
5875 }
5876
5877 # Note: $drivehash is only set when called from snapshot_create.
5878 sub snapshot_delete {
5879     my ($vmid, $snapname, $force, $drivehash) = @_;
5880
5881     my $prepare = 1;
5882
5883     my $snap;
5884     my $unused = [];
5885
5886     my $unlink_parent = sub {
5887         my ($confref, $new_parent) = @_;
5888
5889         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5890             if ($new_parent) {
5891                 $confref->{parent} = $new_parent;
5892             } else {
5893                 delete $confref->{parent};
5894             }
5895         }
5896     };
5897
5898     my $updatefn =  sub {
5899         my ($remove_drive) = @_;
5900
5901         my $conf = load_config($vmid);
5902
5903         if (!$drivehash) {
5904             check_lock($conf);
5905             die "you can't delete a snapshot if vm is a template\n"
5906                 if is_template($conf);
5907         }
5908
5909         $snap = $conf->{snapshots}->{$snapname};
5910
5911         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5912
5913         # remove parent refs
5914         if (!$prepare) {
5915             &$unlink_parent($conf, $snap->{parent});
5916             foreach my $sn (keys %{$conf->{snapshots}}) {
5917                 next if $sn eq $snapname;
5918                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5919             }
5920         }
5921
5922         if ($remove_drive) {
5923             if ($remove_drive eq 'vmstate') {
5924                 delete $snap->{$remove_drive};
5925             } else {
5926                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5927                 my $volid = $drive->{file};
5928                 delete $snap->{$remove_drive};
5929                 add_unused_volume($conf, $volid);
5930             }
5931         }
5932
5933         if ($prepare) {
5934             $snap->{snapstate} = 'delete';
5935         } else {
5936             delete $conf->{snapshots}->{$snapname};
5937             delete $conf->{lock} if $drivehash;
5938             foreach my $volid (@$unused) {
5939                 add_unused_volume($conf, $volid);
5940             }
5941         }
5942
5943         update_config_nolock($vmid, $conf, 1);
5944     };
5945
5946     lock_config($vmid, $updatefn);
5947
5948     # now remove vmstate file
5949
5950     my $storecfg = PVE::Storage::config();
5951
5952     if ($snap->{vmstate}) {
5953         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5954         if (my $err = $@) {
5955             die $err if !$force;
5956             warn $err;
5957         }
5958         # save changes (remove vmstate from snapshot)
5959         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5960     };
5961
5962     # now remove all internal snapshots
5963     foreach_drive($snap, sub {
5964         my ($ds, $drive) = @_;
5965
5966         return if drive_is_cdrom($drive);
5967
5968         my $volid = $drive->{file};
5969         my $device = "drive-$ds";
5970
5971         if (!$drivehash || $drivehash->{$ds}) {
5972             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5973             if (my $err = $@) {
5974                 die $err if !$force;
5975                 warn $err;
5976             }
5977         }
5978
5979         # save changes (remove drive fron snapshot)
5980         lock_config($vmid, $updatefn, $ds) if !$force;
5981         push @$unused, $volid;
5982     });
5983
5984     # now cleanup config
5985     $prepare = 0;
5986     lock_config($vmid, $updatefn);
5987 }
5988
5989 sub has_feature {
5990     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5991
5992     my $err;
5993     foreach_drive($conf, sub {
5994         my ($ds, $drive) = @_;
5995
5996         return if drive_is_cdrom($drive);
5997         my $volid = $drive->{file};
5998         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5999     });
6000
6001     return $err ? 0 : 1;
6002 }
6003
6004 sub template_create {
6005     my ($vmid, $conf, $disk) = @_;
6006
6007     my $storecfg = PVE::Storage::config();
6008
6009     foreach_drive($conf, sub {
6010         my ($ds, $drive) = @_;
6011
6012         return if drive_is_cdrom($drive);
6013         return if $disk && $ds ne $disk;
6014
6015         my $volid = $drive->{file};
6016         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6017
6018         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6019         $drive->{file} = $voliddst;
6020         $conf->{$ds} = print_drive($vmid, $drive);
6021         update_config_nolock($vmid, $conf, 1);
6022     });
6023 }
6024
6025 sub is_template {
6026     my ($conf) = @_;
6027
6028     return 1 if defined $conf->{template} && $conf->{template} == 1;
6029 }
6030
6031 sub qemu_img_convert {
6032     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6033
6034     my $storecfg = PVE::Storage::config();
6035     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6036     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6037
6038     if ($src_storeid && $dst_storeid) {
6039         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6040         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6041
6042         my $src_format = qemu_img_format($src_scfg, $src_volname);
6043         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6044
6045         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6046         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6047
6048         my $cmd = [];
6049         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6050         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6051         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6052
6053         my $parser = sub {
6054             my $line = shift;
6055             if($line =~ m/\((\S+)\/100\%\)/){
6056                 my $percent = $1;
6057                 my $transferred = int($size * $percent / 100);
6058                 my $remaining = $size - $transferred;
6059
6060                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6061             }
6062
6063         };
6064
6065         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6066         my $err = $@;
6067         die "copy failed: $err" if $err;
6068     }
6069 }
6070
6071 sub qemu_img_format {
6072     my ($scfg, $volname) = @_;
6073
6074     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6075         return $1;
6076     } elsif ($scfg->{type} eq 'iscsi') {
6077         return "host_device";
6078     } else {
6079         return "raw";
6080     }
6081 }
6082
6083 sub qemu_drive_mirror {
6084     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6085
6086     my $storecfg = PVE::Storage::config();
6087     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6088
6089     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6090
6091     my $format;
6092     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6093         $format = $1;
6094     }
6095
6096     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6097
6098     #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
6099     #during this time, qmp socket is hanging
6100     #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
6101     #so we need to setup a big timeout
6102     my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6103     $opts->{format} = $format if $format;
6104
6105     print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
6106
6107     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6108     eval {
6109         while (1) {
6110             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6111             my $stat = @$stats[0];
6112             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6113             die "error job is not mirroring" if $stat->{type} ne "mirror";
6114
6115             my $busy = $stat->{busy};
6116             my $ready = $stat->{ready};
6117
6118             if (my $total = $stat->{len}) {
6119                 my $transferred = $stat->{offset} || 0;
6120                 my $remaining = $total - $transferred;
6121                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6122
6123                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6124             }
6125
6126
6127             if ($stat->{ready} eq 'true') {
6128
6129                 last if $vmiddst != $vmid;
6130
6131                 # try to switch the disk if source and destination are on the same guest
6132                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6133                 last if !$@;
6134                 die $@ if $@ !~ m/cannot be completed/;
6135             }
6136             sleep 1;
6137         }
6138
6139
6140     };
6141     my $err = $@;
6142
6143     my $cancel_job = sub {
6144         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6145         while (1) {
6146             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6147             my $stat = @$stats[0];
6148             last if !$stat;
6149             sleep 1;
6150         }
6151     };
6152
6153     if ($err) {
6154         eval { &$cancel_job(); };
6155         die "mirroring error: $err";
6156     }
6157
6158     if ($vmiddst != $vmid) {
6159         # if we clone a disk for a new target vm, we don't switch the disk
6160         &$cancel_job(); # so we call block-job-cancel
6161     }
6162 }
6163
6164 sub clone_disk {
6165     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6166         $newvmid, $storage, $format, $full, $newvollist) = @_;
6167
6168     my $newvolid;
6169
6170     if (!$full) {
6171         print "create linked clone of drive $drivename ($drive->{file})\n";
6172         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6173         push @$newvollist, $newvolid;
6174     } else {
6175         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6176         $storeid = $storage if $storage;
6177
6178         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6179         if (!$format) {
6180             $format = $drive->{format} || $defFormat;
6181         }
6182
6183         # test if requested format is supported - else use default
6184         my $supported = grep { $_ eq $format } @$validFormats;
6185         $format = $defFormat if !$supported;
6186
6187         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6188
6189         print "create full clone of drive $drivename ($drive->{file})\n";
6190         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6191         push @$newvollist, $newvolid;
6192
6193         if (!$running || $snapname) {
6194             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6195         } else {
6196             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6197         }
6198     }
6199
6200     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6201
6202     my $disk = $drive;
6203     $disk->{format} = undef;
6204     $disk->{file} = $newvolid;
6205     $disk->{size} = $size;
6206
6207     return $disk;
6208 }
6209
6210 # this only works if VM is running
6211 sub get_current_qemu_machine {
6212     my ($vmid) = @_;
6213
6214     my $cmd = { execute => 'query-machines', arguments => {} };
6215     my $res = vm_qmp_command($vmid, $cmd);
6216
6217     my ($current, $default);
6218     foreach my $e (@$res) {
6219         $default = $e->{name} if $e->{'is-default'};
6220         $current = $e->{name} if $e->{'is-current'};
6221     }
6222
6223     # fallback to the default machine if current is not supported by qemu
6224     return $current || $default || 'pc';
6225 }
6226
6227 sub qemu_machine_feature_enabled {
6228     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6229
6230     my $current_major;
6231     my $current_minor;
6232
6233     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6234
6235         $current_major = $3;
6236         $current_minor = $4;
6237
6238     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6239
6240         $current_major = $1;
6241         $current_minor = $2;
6242     }
6243
6244     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6245
6246
6247 }
6248
6249 sub lspci {
6250
6251     my $devices = {};
6252
6253     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6254             my (undef, $id, $function) = @_;
6255             my $res = { id => $id, function => $function};
6256             push @{$devices->{$id}}, $res;
6257     });
6258
6259     return $devices;
6260 }
6261
6262 sub vm_iothreads_list {
6263     my ($vmid) = @_;
6264
6265     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6266
6267     my $iothreads = {};
6268     foreach my $iothread (@$res) {
6269         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6270     }
6271
6272     return $iothreads;
6273 }
6274
6275 sub scsihw_infos {
6276     my ($conf, $drive) = @_;
6277
6278     my $maxdev = 0;
6279
6280     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6281         $maxdev = 7;
6282     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6283         $maxdev = 1;
6284     } else {
6285         $maxdev = 256;
6286     }
6287
6288     my $controller = int($drive->{index} / $maxdev);
6289     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6290
6291     return ($maxdev, $controller, $controller_prefix);
6292 }
6293
6294 1;