PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     onboot => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Specifies whether a VM will be started during system bootup.",
 159         default => 0,
 160     },
 161     autostart => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Automatic restart after crash (currently ignored).",
 165         default => 0,
 166     },
 167     hotplug => {
 168         optional => 1,
 169         type => 'string', format => 'pve-hotplug-features',
 170         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 171         default => 'network,disk,usb',
 172     },
 173     reboot => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 177         default => 1,
 178     },
 179     lock => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Lock/unlock the VM.",
 183         enum => [qw(migrate backup snapshot rollback)],
 184     },
 185     cpulimit => {
 186         optional => 1,
 187         type => 'integer',
 188         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 189         minimum => 0,
 190         default => 0,
 191     },
 192     cpuunits => {
 193         optional => 1,
 194         type => 'integer',
 195         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 196         minimum => 0,
 197         maximum => 500000,
 198         default => 1000,
 199     },
 200     memory => {
 201         optional => 1,
 202         type => 'integer',
 203         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 204         minimum => 16,
 205         default => 512,
 206     },
 207     balloon => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 211         minimum => 0,
 212     },
 213     shares => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 217         minimum => 0,
 218         maximum => 50000,
 219         default => 1000,
 220     },
 221     keyboard => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 225         enum => PVE::Tools::kvmkeymaplist(),
 226         default => 'en-us',
 227     },
 228     name => {
 229         optional => 1,
 230         type => 'string', format => 'dns-name',
 231         description => "Set a name for the VM. Only used on the configuration web interface.",
 232     },
 233     scsihw => {
 234         optional => 1,
 235         type => 'string',
 236         description => "scsi controller model",
 237         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 238         default => 'lsi',
 239     },
 240     description => {
 241         optional => 1,
 242         type => 'string',
 243         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 244     },
 245     ostype => {
 246         optional => 1,
 247         type => 'string',
 248         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 249         description => <<EODESC,
 250 Used to enable special optimization/features for specific
 251 operating systems:
 252
 253 other  => unspecified OS
 254 wxp    => Microsoft Windows XP
 255 w2k    => Microsoft Windows 2000
 256 w2k3   => Microsoft Windows 2003
 257 w2k8   => Microsoft Windows 2008
 258 wvista => Microsoft Windows Vista
 259 win7   => Microsoft Windows 7
 260 win8   => Microsoft Windows 8/2012
 261 l24    => Linux 2.4 Kernel
 262 l26    => Linux 2.6/3.X Kernel
 263 solaris => solaris/opensolaris/openindiania kernel
 264
 265 other|l24|l26|solaris                       ... no special behaviour
 266 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 267 EODESC
 268     },
 269     boot => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 273         pattern => '[acdn]{1,4}',
 274         default => 'cdn',
 275     },
 276     bootdisk => {
 277         optional => 1,
 278         type => 'string', format => 'pve-qm-bootdisk',
 279         description => "Enable booting from specified disk.",
 280         pattern => '(ide|sata|scsi|virtio)\d+',
 281     },
 282     smp => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "The number of CPUs. Please use option -sockets instead.",
 286         minimum => 1,
 287         default => 1,
 288     },
 289     sockets => {
 290         optional => 1,
 291         type => 'integer',
 292         description => "The number of CPU sockets.",
 293         minimum => 1,
 294         default => 1,
 295     },
 296     cores => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of cores per socket.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     numa => {
 304         optional => 1,
 305         type => 'boolean',
 306         description => "Enable/disable Numa.",
 307         default => 0,
 308     },
 309     vcpus => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "Number of hotplugged vcpus.",
 313         minimum => 1,
 314         default => 0,
 315     },
 316     acpi => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable ACPI.",
 320         default => 1,
 321     },
 322     agent => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable Qemu GuestAgent.",
 326         default => 0,
 327     },
 328     kvm => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable KVM hardware virtualization.",
 332         default => 1,
 333     },
 334     tdf => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Enable/disable time drift fix.",
 338         default => 0,
 339     },
 340     localtime => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 344     },
 345     freeze => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 349     },
 350     vga => {
 351         optional => 1,
 352         type => 'string',
 353         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 354         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 355     },
 356     watchdog => {
 357         optional => 1,
 358         type => 'string', format => 'pve-qm-watchdog',
 359         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 360         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 361     },
 362     startdate => {
 363         optional => 1,
 364         type => 'string',
 365         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 366         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 367         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 368         default => 'now',
 369     },
 370     startup => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-startup',
 373         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 374         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 375     },
 376     template => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable Template.",
 380         default => 0,
 381     },
 382     args => {
 383         optional => 1,
 384         type => 'string',
 385         description => <<EODESCR,
 386 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 387
 388 args: -no-reboot -no-hpet
 389 EODESCR
 390     },
 391     tablet => {
 392         optional => 1,
 393         type => 'boolean',
 394         default => 1,
 395         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 396     },
 397     migrate_speed => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 401         minimum => 0,
 402         default => 0,
 403     },
 404     migrate_downtime => {
 405         optional => 1,
 406         type => 'number',
 407         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 408         minimum => 0,
 409         default => 0.1,
 410     },
 411     cdrom => {
 412         optional => 1,
 413         type => 'string', format => 'pve-qm-drive',
 414         typetext => 'volume',
 415         description => "This is an alias for option -ide2",
 416     },
 417     cpu => {
 418         optional => 1,
 419         description => "Emulated CPU type.",
 420         type => 'string',
 421         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 422         default => 'kvm64',
 423     },
 424     parent => get_standard_option('pve-snapshot-name', {
 425         optional => 1,
 426         description => "Parent snapshot name. This is used internally, and should not be modified.",
 427     }),
 428     snaptime => {
 429         optional => 1,
 430         description => "Timestamp for snapshots.",
 431         type => 'integer',
 432         minimum => 0,
 433     },
 434     vmstate => {
 435         optional => 1,
 436         type => 'string', format => 'pve-volume-id',
 437         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 438     },
 439     machine => {
 440         description => "Specific the Qemu machine type.",
 441         type => 'string',
 442         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 443         maxLength => 40,
 444         optional => 1,
 445     },
 446     smbios1 => {
 447         description => "Specify SMBIOS type 1 fields.",
 448         type => 'string', format => 'pve-qm-smbios1',
 449         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 450         maxLength => 256,
 451         optional => 1,
 452     },
 453 };
 454
 455 # what about other qemu settings ?
 456 #cpu => 'string',
 457 #machine => 'string',
 458 #fda => 'file',
 459 #fdb => 'file',
 460 #mtdblock => 'file',
 461 #sd => 'file',
 462 #pflash => 'file',
 463 #snapshot => 'bool',
 464 #bootp => 'file',
 465 ##tftp => 'dir',
 466 ##smb => 'dir',
 467 #kernel => 'file',
 468 #append => 'string',
 469 #initrd => 'file',
 470 ##soundhw => 'string',
 471
 472 while (my ($k, $v) = each %$confdesc) {
 473     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 474 }
 475
 476 my $MAX_IDE_DISKS = 4;
 477 my $MAX_SCSI_DISKS = 14;
 478 my $MAX_VIRTIO_DISKS = 16;
 479 my $MAX_SATA_DISKS = 6;
 480 my $MAX_USB_DEVICES = 5;
 481 my $MAX_NETS = 32;
 482 my $MAX_UNUSED_DISKS = 8;
 483 my $MAX_HOSTPCI_DEVICES = 4;
 484 my $MAX_SERIAL_PORTS = 4;
 485 my $MAX_PARALLEL_PORTS = 3;
 486 my $MAX_NUMA = 8;
 487 my $MAX_MEM = 4194304;
 488 my $STATICMEM = 1024;
 489
 490 my $numadesc = {
 491     optional => 1,
 492     type => 'string', format => 'pve-qm-numanode',
 493     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 494     description => "numa topology",
 495 };
 496 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 497
 498 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 499     $confdesc->{"numa$i"} = $numadesc;
 500 }
 501
 502 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 503                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 504                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 505 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 506
 507 my $netdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-net',
 510     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 511     description => <<EODESCR,
 512 Specify network devices.
 513
 514 MODEL is one of: $nic_model_list_txt
 515
 516 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 517 automatically generated if not specified.
 518
 519 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 520
 521 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 522
 523 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 524
 525 10.0.2.2   Gateway
 526 10.0.2.3   DNS Server
 527 10.0.2.4   SMB Server
 528
 529 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 530
 531 EODESCR
 532 };
 533 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 534
 535 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 536     $confdesc->{"net$i"} = $netdesc;
 537 }
 538
 539 my $drivename_hash;
 540
 541 my $idedesc = {
 542     optional => 1,
 543     type => 'string', format => 'pve-qm-drive',
 544     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 545     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 546 };
 547 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 548
 549 my $scsidesc = {
 550     optional => 1,
 551     type => 'string', format => 'pve-qm-drive',
 552     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on]',
 553     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 554 };
 555 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 556
 557 my $satadesc = {
 558     optional => 1,
 559     type => 'string', format => 'pve-qm-drive',
 560     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 561     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 562 };
 563 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 564
 565 my $virtiodesc = {
 566     optional => 1,
 567     type => 'string', format => 'pve-qm-drive',
 568     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 569     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 570 };
 571 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 572
 573 my $usbdesc = {
 574     optional => 1,
 575     type => 'string', format => 'pve-qm-usb-device',
 576     typetext => 'host=HOSTUSBDEVICE|spice',
 577     description => <<EODESCR,
 578 Configure an USB device (n is 0 to 4). This can be used to
 579 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 580
 581 'bus-port(.port)*' (decimal numbers) or
 582 'vendor_id:product_id' (hexadeciaml numbers)
 583
 584 You can use the 'lsusb -t' command to list existing usb devices.
 585
 586 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 587
 588 The value 'spice' can be used to add a usb redirection devices for spice.
 589
 590 EODESCR
 591 };
 592 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 593
 594 my $hostpcidesc = {
 595         optional => 1,
 596         type => 'string', format => 'pve-qm-hostpci',
 597         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 598         description => <<EODESCR,
 599 Map host pci devices. HOSTPCIDEVICE syntax is:
 600
 601 'bus:dev.func' (hexadecimal numbers)
 602
 603 You can us the 'lspci' command to list existing pci devices.
 604
 605 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 606
 607 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 608
 609 Experimental: user reported problems with this option.
 610 EODESCR
 611 };
 612 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 613
 614 my $serialdesc = {
 615         optional => 1,
 616         type => 'string',
 617         pattern => '(/dev/.+|socket)',
 618         description =>  <<EODESCR,
 619 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 620
 621 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 622
 623 Experimental: user reported problems with this option.
 624 EODESCR
 625 };
 626
 627 my $paralleldesc= {
 628         optional => 1,
 629         type => 'string',
 630         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 631         description =>  <<EODESCR,
 632 Map host parallel devices (n is 0 to 2).
 633
 634 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 635
 636 Experimental: user reported problems with this option.
 637 EODESCR
 638 };
 639
 640 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 641     $confdesc->{"parallel$i"} = $paralleldesc;
 642 }
 643
 644 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 645     $confdesc->{"serial$i"} = $serialdesc;
 646 }
 647
 648 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 649     $confdesc->{"hostpci$i"} = $hostpcidesc;
 650 }
 651
 652 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 653     $drivename_hash->{"ide$i"} = 1;
 654     $confdesc->{"ide$i"} = $idedesc;
 655 }
 656
 657 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 658     $drivename_hash->{"sata$i"} = 1;
 659     $confdesc->{"sata$i"} = $satadesc;
 660 }
 661
 662 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 663     $drivename_hash->{"scsi$i"} = 1;
 664     $confdesc->{"scsi$i"} = $scsidesc ;
 665 }
 666
 667 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 668     $drivename_hash->{"virtio$i"} = 1;
 669     $confdesc->{"virtio$i"} = $virtiodesc;
 670 }
 671
 672 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 673     $confdesc->{"usb$i"} = $usbdesc;
 674 }
 675
 676 my $unuseddesc = {
 677     optional => 1,
 678     type => 'string', format => 'pve-volume-id',
 679     description => "Reference to unused volumes.",
 680 };
 681
 682 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 683     $confdesc->{"unused$i"} = $unuseddesc;
 684 }
 685
 686 my $kvm_api_version = 0;
 687
 688 sub kvm_version {
 689
 690     return $kvm_api_version if $kvm_api_version;
 691
 692     my $fh = IO::File->new("</dev/kvm") ||
 693         return 0;
 694
 695     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 696         $kvm_api_version = $v;
 697     }
 698
 699     $fh->close();
 700
 701     return  $kvm_api_version;
 702 }
 703
 704 my $kvm_user_version;
 705
 706 sub kvm_user_version {
 707
 708     return $kvm_user_version if $kvm_user_version;
 709
 710     $kvm_user_version = 'unknown';
 711
 712     my $tmp = `kvm -help 2>/dev/null`;
 713
 714     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 715         $kvm_user_version = $2;
 716     }
 717
 718     return $kvm_user_version;
 719
 720 }
 721
 722 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 723
 724 sub disknames {
 725     # order is important - used to autoselect boot disk
 726     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 727             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 728             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 729             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 730 }
 731
 732 sub valid_drivename {
 733     my $dev = shift;
 734
 735     return defined($drivename_hash->{$dev});
 736 }
 737
 738 sub option_exists {
 739     my $key = shift;
 740     return defined($confdesc->{$key});
 741 }
 742
 743 sub nic_models {
 744     return $nic_model_list;
 745 }
 746
 747 sub os_list_description {
 748
 749     return {
 750         other => 'Other',
 751         wxp => 'Windows XP',
 752         w2k => 'Windows 2000',
 753         w2k3 =>, 'Windows 2003',
 754         w2k8 => 'Windows 2008',
 755         wvista => 'Windows Vista',
 756         win7 => 'Windows 7',
 757         win8 => 'Windows 8/2012',
 758         l24 => 'Linux 2.4',
 759         l26 => 'Linux 2.6',
 760     };
 761 }
 762
 763 my $cdrom_path;
 764
 765 sub get_cdrom_path {
 766
 767     return  $cdrom_path if $cdrom_path;
 768
 769     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 770     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 771     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 772 }
 773
 774 sub get_iso_path {
 775     my ($storecfg, $vmid, $cdrom) = @_;
 776
 777     if ($cdrom eq 'cdrom') {
 778         return get_cdrom_path();
 779     } elsif ($cdrom eq 'none') {
 780         return '';
 781     } elsif ($cdrom =~ m|^/|) {
 782         return $cdrom;
 783     } else {
 784         return PVE::Storage::path($storecfg, $cdrom);
 785     }
 786 }
 787
 788 # try to convert old style file names to volume IDs
 789 sub filename_to_volume_id {
 790     my ($vmid, $file, $media) = @_;
 791
 792     if (!($file eq 'none' || $file eq 'cdrom' ||
 793           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 794
 795         return undef if $file =~ m|/|;
 796
 797         if ($media && $media eq 'cdrom') {
 798             $file = "local:iso/$file";
 799         } else {
 800             $file = "local:$vmid/$file";
 801         }
 802     }
 803
 804     return $file;
 805 }
 806
 807 sub verify_media_type {
 808     my ($opt, $vtype, $media) = @_;
 809
 810     return if !$media;
 811
 812     my $etype;
 813     if ($media eq 'disk') {
 814         $etype = 'images';
 815     } elsif ($media eq 'cdrom') {
 816         $etype = 'iso';
 817     } else {
 818         die "internal error";
 819     }
 820
 821     return if ($vtype eq $etype);
 822
 823     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 824 }
 825
 826 sub cleanup_drive_path {
 827     my ($opt, $storecfg, $drive) = @_;
 828
 829     # try to convert filesystem paths to volume IDs
 830
 831     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 832         ($drive->{file} !~ m|^/dev/.+|) &&
 833         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 834         ($drive->{file} !~ m/^\d+$/)) {
 835         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 836         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 837         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 838         verify_media_type($opt, $vtype, $drive->{media});
 839         $drive->{file} = $volid;
 840     }
 841
 842     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 843 }
 844
 845 sub create_conf_nolock {
 846     my ($vmid, $settings) = @_;
 847
 848     my $filename = config_file($vmid);
 849
 850     die "configuration file '$filename' already exists\n" if -f $filename;
 851
 852     my $defaults = load_defaults();
 853
 854     $settings->{name} = "vm$vmid" if !$settings->{name};
 855     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 856
 857     my $data = '';
 858     foreach my $opt (keys %$settings) {
 859         next if !$confdesc->{$opt};
 860
 861         my $value = $settings->{$opt};
 862         next if !$value;
 863
 864         $data .= "$opt: $value\n";
 865     }
 866
 867     PVE::Tools::file_set_contents($filename, $data);
 868 }
 869
 870 sub parse_hotplug_features {
 871     my ($data) = @_;
 872
 873     my $res = {};
 874
 875     return $res if $data eq '0';
 876
 877     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 878
 879     foreach my $feature (PVE::Tools::split_list($data)) {
 880         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 881             $res->{$1} = 1;
 882         } else {
 883             warn "ignoring unknown hotplug feature '$feature'\n";
 884         }
 885     }
 886     return $res;
 887 }
 888
 889 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 890 sub pve_verify_hotplug_features {
 891     my ($value, $noerr) = @_;
 892
 893     return $value if parse_hotplug_features($value);
 894
 895     return undef if $noerr;
 896
 897     die "unable to parse hotplug option\n";
 898 }
 899
 900 my $parse_size = sub {
 901     my ($value) = @_;
 902
 903     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 904     my ($size, $unit) = ($1, $3);
 905     if ($unit) {
 906         if ($unit eq 'K') {
 907             $size = $size * 1024;
 908         } elsif ($unit eq 'M') {
 909             $size = $size * 1024 * 1024;
 910         } elsif ($unit eq 'G') {
 911             $size = $size * 1024 * 1024 * 1024;
 912         }
 913     }
 914     return int($size);
 915 };
 916
 917 my $format_size = sub {
 918     my ($size) = @_;
 919
 920     $size = int($size);
 921
 922     my $kb = int($size/1024);
 923     return $size if $kb*1024 != $size;
 924
 925     my $mb = int($kb/1024);
 926     return "${kb}K" if $mb*1024 != $kb;
 927
 928     my $gb = int($mb/1024);
 929     return "${mb}M" if $gb*1024 != $mb;
 930
 931     return "${gb}G";
 932 };
 933
 934 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 935 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 936 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 937 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 938
 939 sub parse_drive {
 940     my ($key, $data) = @_;
 941
 942     my $res = {};
 943
 944     # $key may be undefined - used to verify JSON parameters
 945     if (!defined($key)) {
 946         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 947         $res->{index} = 0;
 948     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 949         $res->{interface} = $1;
 950         $res->{index} = $2;
 951     } else {
 952         return undef;
 953     }
 954
 955     foreach my $p (split (/,/, $data)) {
 956         next if $p =~ m/^\s*$/;
 957
 958         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread)=(.+)$/) {
 959             my ($k, $v) = ($1, $2);
 960
 961             $k = 'file' if $k eq 'volume';
 962
 963             return undef if defined $res->{$k};
 964
 965             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 966                 return undef if !$v || $v !~ m/^\d+/;
 967                 $k = "m$k";
 968                 $v = sprintf("%.3f", $v / (1024*1024));
 969             }
 970             $res->{$k} = $v;
 971         } else {
 972             if (!$res->{file} && $p !~ m/=/) {
 973                 $res->{file} = $p;
 974             } else {
 975                 return undef;
 976             }
 977         }
 978     }
 979
 980     return undef if !$res->{file};
 981
 982     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 983         $res->{format} = $1;
 984     }
 985
 986     return undef if $res->{cache} &&
 987         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 988     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 989     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 990     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 991     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 992     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 993     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 994     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 995     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 996     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 997     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 998     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 999     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
1000     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
1001
1002     return undef if $res->{mbps_rd} && $res->{mbps};
1003     return undef if $res->{mbps_wr} && $res->{mbps};
1004
1005     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1006     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1007     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1008     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1009     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1010     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1011
1012     return undef if $res->{iops_rd} && $res->{iops};
1013     return undef if $res->{iops_wr} && $res->{iops};
1014
1015
1016     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1017     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1018     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1019     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1020     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1021     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1022
1023
1024     if ($res->{size}) {
1025         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1026     }
1027
1028     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1029         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1030         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1031         return undef if $res->{interface} eq 'virtio';
1032     }
1033
1034     # rerror does not work with scsi drives
1035     if ($res->{rerror}) {
1036         return undef if $res->{interface} eq 'scsi';
1037     }
1038
1039     return $res;
1040 }
1041
1042 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1043
1044 sub print_drive {
1045     my ($vmid, $drive) = @_;
1046
1047     my $opts = '';
1048     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread') {
1049         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1050     }
1051
1052     if ($drive->{size}) {
1053         $opts .= ",size=" . &$format_size($drive->{size});
1054     }
1055
1056     return "$drive->{file}$opts";
1057 }
1058
1059 sub scsi_inquiry {
1060     my($fh, $noerr) = @_;
1061
1062     my $SG_IO = 0x2285;
1063     my $SG_GET_VERSION_NUM = 0x2282;
1064
1065     my $versionbuf = "\x00" x 8;
1066     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1067     if (!$ret) {
1068         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1069         return undef;
1070     }
1071     my $version = unpack("I", $versionbuf);
1072     if ($version < 30000) {
1073         die "scsi generic interface too old\n"  if !$noerr;
1074         return undef;
1075     }
1076
1077     my $buf = "\x00" x 36;
1078     my $sensebuf = "\x00" x 8;
1079     my $cmd = pack("C x3 C x1", 0x12, 36);
1080
1081     # see /usr/include/scsi/sg.h
1082     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1083
1084     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1085                       length($sensebuf), 0, length($buf), $buf,
1086                       $cmd, $sensebuf, 6000);
1087
1088     $ret = ioctl($fh, $SG_IO, $packet);
1089     if (!$ret) {
1090         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1091         return undef;
1092     }
1093
1094     my @res = unpack($sg_io_hdr_t, $packet);
1095     if ($res[17] || $res[18]) {
1096         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1097         return undef;
1098     }
1099
1100     my $res = {};
1101     (my $byte0, my $byte1, $res->{vendor},
1102      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1103
1104     $res->{removable} = $byte1 & 128 ? 1 : 0;
1105     $res->{type} = $byte0 & 31;
1106
1107     return $res;
1108 }
1109
1110 sub path_is_scsi {
1111     my ($path) = @_;
1112
1113     my $fh = IO::File->new("+<$path") || return undef;
1114     my $res = scsi_inquiry($fh, 1);
1115     close($fh);
1116
1117     return $res;
1118 }
1119
1120 sub machine_type_is_q35 {
1121     my ($conf) = @_;
1122
1123     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1124 }
1125
1126 sub print_tabletdevice_full {
1127     my ($conf) = @_;
1128
1129     my $q35 = machine_type_is_q35($conf);
1130
1131     # we use uhci for old VMs because tablet driver was buggy in older qemu
1132     my $usbbus = $q35 ? "ehci" : "uhci";
1133
1134     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1135 }
1136
1137 sub print_drivedevice_full {
1138     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1139
1140     my $device = '';
1141     my $maxdev = 0;
1142
1143     if ($drive->{interface} eq 'virtio') {
1144         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1145         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1146         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1147     } elsif ($drive->{interface} eq 'scsi') {
1148
1149         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1150         my $unit = $drive->{index} % $maxdev;
1151         my $devicetype = 'hd';
1152         my $path = '';
1153         if (drive_is_cdrom($drive)) {
1154               $devicetype = 'cd';
1155         } else {
1156               if ($drive->{file} =~ m|^/|) {
1157                   $path = $drive->{file};
1158               } else {
1159                   $path = PVE::Storage::path($storecfg, $drive->{file});
1160               }
1161
1162               if($path =~ m/^iscsi\:\/\//){
1163                   $devicetype = 'generic';
1164               } else {
1165                   if (my $info = path_is_scsi($path)) {
1166                       if ($info->{type} == 0) {
1167                           $devicetype = 'block';
1168                       } elsif ($info->{type} == 1) { # tape
1169                           $devicetype = 'generic';
1170                       }
1171                   }
1172               }
1173          }
1174
1175         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1176             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1177         } else {
1178             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1179         }
1180
1181     } elsif ($drive->{interface} eq 'ide'){
1182         $maxdev = 2;
1183         my $controller = int($drive->{index} / $maxdev);
1184         my $unit = $drive->{index} % $maxdev;
1185         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1186
1187         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1188     } elsif ($drive->{interface} eq 'sata'){
1189         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1190         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1191         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1192     } elsif ($drive->{interface} eq 'usb') {
1193         die "implement me";
1194         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1195     } else {
1196         die "unsupported interface type";
1197     }
1198
1199     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1200
1201     return $device;
1202 }
1203
1204 sub get_initiator_name {
1205     my $initiator;
1206
1207     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1208     while (defined(my $line = <$fh>)) {
1209         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1210         $initiator = $1;
1211         last;
1212     }
1213     $fh->close();
1214
1215     return $initiator;
1216 }
1217
1218 sub print_drive_full {
1219     my ($storecfg, $vmid, $drive) = @_;
1220
1221     my $opts = '';
1222     foreach my $o (@qemu_drive_options) {
1223         next if $o eq 'bootindex';
1224         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1225     }
1226
1227     foreach my $o (qw(bps bps_rd bps_wr)) {
1228         my $v = $drive->{"m$o"};
1229         $opts .= ",$o=" . int($v*1024*1024) if $v;
1230     }
1231
1232     # use linux-aio by default (qemu default is threads)
1233     $opts .= ",aio=native" if !$drive->{aio};
1234
1235     my $path;
1236     my $volid = $drive->{file};
1237     if (drive_is_cdrom($drive)) {
1238         $path = get_iso_path($storecfg, $vmid, $volid);
1239     } else {
1240         if ($volid =~ m|^/|) {
1241             $path = $volid;
1242         } else {
1243             $path = PVE::Storage::path($storecfg, $volid);
1244         }
1245     }
1246
1247     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1248
1249     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1250     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1251
1252     my $pathinfo = $path ? "file=$path," : '';
1253
1254     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1255 }
1256
1257 sub print_netdevice_full {
1258     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1259
1260     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1261
1262     my $device = $net->{model};
1263     if ($net->{model} eq 'virtio') {
1264          $device = 'virtio-net-pci';
1265      };
1266
1267     my $pciaddr = print_pci_addr("$netid", $bridges);
1268     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1269     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1270         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1271         my $vectors = $net->{queues} * 2 + 2;
1272         $tmpstr .= ",vectors=$vectors,mq=on";
1273     }
1274     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1275     return $tmpstr;
1276 }
1277
1278 sub print_netdev_full {
1279     my ($vmid, $conf, $net, $netid) = @_;
1280
1281     my $i = '';
1282     if ($netid =~ m/^net(\d+)$/) {
1283         $i = int($1);
1284     }
1285
1286     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1287
1288     my $ifname = "tap${vmid}i$i";
1289
1290     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1291     die "interface name '$ifname' is too long (max 15 character)\n"
1292         if length($ifname) >= 16;
1293
1294     my $vhostparam = '';
1295     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1296
1297     my $vmname = $conf->{name} || "vm$vmid";
1298
1299     my $netdev = "";
1300
1301     if ($net->{bridge}) {
1302         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1303     } else {
1304         $netdev = "type=user,id=$netid,hostname=$vmname";
1305     }
1306
1307     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1308
1309     return $netdev;
1310 }
1311
1312 sub drive_is_cdrom {
1313     my ($drive) = @_;
1314
1315     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1316
1317 }
1318
1319 sub parse_numa {
1320     my ($data) = @_;
1321
1322     my $res = {};
1323
1324     foreach my $kvp (split(/,/, $data)) {
1325
1326         if ($kvp =~ m/^memory=(\S+)$/) {
1327             $res->{memory} = $1;
1328         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1329             $res->{policy} = $1;
1330         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1331             $res->{cpus}->{start} = $1;
1332             $res->{cpus}->{end} = $3;
1333         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1334             $res->{hostnodes}->{start} = $1;
1335             $res->{hostnodes}->{end} = $3;
1336         } else {
1337             return undef;
1338         }
1339     }
1340
1341     return $res;
1342 }
1343
1344 sub parse_hostpci {
1345     my ($value) = @_;
1346
1347     return undef if !$value;
1348
1349
1350     my @list = split(/,/, $value);
1351     my $found;
1352
1353     my $res = {};
1354     foreach my $kv (@list) {
1355
1356         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1357             $found = 1;
1358             if(defined($4)){
1359                 push @{$res->{pciid}}, { id => $2 , function => $4};
1360
1361             }else{
1362                 my $pcidevices = lspci($2);
1363                 $res->{pciid} = $pcidevices->{$2};
1364             }
1365         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1366             $res->{driver} = $1;
1367         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1368             $res->{rombar} = $1;
1369         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1370             $res->{'x-vga'} = $1;
1371         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1372             $res->{pcie} = 1 if $1 == 1;
1373         } else {
1374             warn "unknown hostpci setting '$kv'\n";
1375         }
1376     }
1377
1378     return undef if !$found;
1379
1380     return $res;
1381 }
1382
1383 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1384 sub parse_net {
1385     my ($data) = @_;
1386
1387     my $res = {};
1388
1389     foreach my $kvp (split(/,/, $data)) {
1390
1391         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1392             my $model = lc($1);
1393             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1394             $res->{model} = $model;
1395             $res->{macaddr} = $mac;
1396         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1397             $res->{bridge} = $1;
1398         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1399             $res->{queues} = $1;
1400         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1401             $res->{rate} = $1;
1402         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1403             $res->{tag} = $1;
1404         } elsif ($kvp =~ m/^firewall=([01])$/) {
1405             $res->{firewall} = $1;
1406         } elsif ($kvp =~ m/^link_down=([01])$/) {
1407             $res->{link_down} = $1;
1408         } else {
1409             return undef;
1410         }
1411
1412     }
1413
1414     return undef if !$res->{model};
1415
1416     return $res;
1417 }
1418
1419 sub print_net {
1420     my $net = shift;
1421
1422     my $res = "$net->{model}";
1423     $res .= "=$net->{macaddr}" if $net->{macaddr};
1424     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1425     $res .= ",rate=$net->{rate}" if $net->{rate};
1426     $res .= ",tag=$net->{tag}" if $net->{tag};
1427     $res .= ",firewall=1" if $net->{firewall};
1428     $res .= ",link_down=1" if $net->{link_down};
1429     $res .= ",queues=$net->{queues}" if $net->{queues};
1430
1431     return $res;
1432 }
1433
1434 sub add_random_macs {
1435     my ($settings) = @_;
1436
1437     foreach my $opt (keys %$settings) {
1438         next if $opt !~ m/^net(\d+)$/;
1439         my $net = parse_net($settings->{$opt});
1440         next if !$net;
1441         $settings->{$opt} = print_net($net);
1442     }
1443 }
1444
1445 sub add_unused_volume {
1446     my ($config, $volid) = @_;
1447
1448     my $key;
1449     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1450         my $test = "unused$ind";
1451         if (my $vid = $config->{$test}) {
1452             return if $vid eq $volid; # do not add duplicates
1453         } else {
1454             $key = $test;
1455         }
1456     }
1457
1458     die "To many unused volume - please delete them first.\n" if !$key;
1459
1460     $config->{$key} = $volid;
1461
1462     return $key;
1463 }
1464
1465 sub vm_is_volid_owner {
1466     my ($storecfg, $vmid, $volid) = @_;
1467
1468     if ($volid !~  m|^/|) {
1469         my ($path, $owner);
1470         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1471         if ($owner && ($owner == $vmid)) {
1472             return 1;
1473         }
1474     }
1475
1476     return undef;
1477 }
1478
1479 sub vmconfig_delete_pending_option {
1480     my ($conf, $key) = @_;
1481
1482     delete $conf->{pending}->{$key};
1483     my $pending_delete_hash = { $key => 1 };
1484     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1485         $pending_delete_hash->{$opt} = 1;
1486     }
1487     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1488 }
1489
1490 sub vmconfig_undelete_pending_option {
1491     my ($conf, $key) = @_;
1492
1493     my $pending_delete_hash = {};
1494     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1495         $pending_delete_hash->{$opt} = 1;
1496     }
1497     delete $pending_delete_hash->{$key};
1498
1499     my @keylist = keys %$pending_delete_hash;
1500     if (scalar(@keylist)) {
1501         $conf->{pending}->{delete} = join(',', @keylist);
1502     } else {
1503         delete $conf->{pending}->{delete};
1504     }
1505 }
1506
1507 sub vmconfig_register_unused_drive {
1508     my ($storecfg, $vmid, $conf, $drive) = @_;
1509
1510     if (!drive_is_cdrom($drive)) {
1511         my $volid = $drive->{file};
1512         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1513             add_unused_volume($conf, $volid, $vmid);
1514         }
1515     }
1516 }
1517
1518 sub vmconfig_cleanup_pending {
1519     my ($conf) = @_;
1520
1521     # remove pending changes when nothing changed
1522     my $changes;
1523     foreach my $opt (keys %{$conf->{pending}}) {
1524         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1525             $changes = 1;
1526             delete $conf->{pending}->{$opt};
1527         }
1528     }
1529
1530     # remove delete if option is not set
1531     my $pending_delete_hash = {};
1532     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1533         if (defined($conf->{$opt})) {
1534             $pending_delete_hash->{$opt} = 1;
1535         } else {
1536             $changes = 1;
1537         }
1538     }
1539
1540     my @keylist = keys %$pending_delete_hash;
1541     if (scalar(@keylist)) {
1542         $conf->{pending}->{delete} = join(',', @keylist);
1543     } else {
1544         delete $conf->{pending}->{delete};
1545     }
1546
1547     return $changes;
1548 }
1549
1550 my $valid_smbios1_options = {
1551     manufacturer => '\S+',
1552     product => '\S+',
1553     version => '\S+',
1554     serial => '\S+',
1555     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1556     sku => '\S+',
1557     family => '\S+',
1558 };
1559
1560 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1561 sub parse_smbios1 {
1562     my ($data) = @_;
1563
1564     my $res = {};
1565
1566     foreach my $kvp (split(/,/, $data)) {
1567         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1568         my ($k, $v) = split(/=/, $kvp);
1569         return undef if !defined($k) || !defined($v);
1570         return undef if !$valid_smbios1_options->{$k};
1571         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1572         $res->{$k} = $v;
1573     }
1574
1575     return $res;
1576 }
1577
1578 sub print_smbios1 {
1579     my ($smbios1) = @_;
1580
1581     my $data = '';
1582     foreach my $k (keys %$smbios1) {
1583         next if !defined($smbios1->{$k});
1584         next if !$valid_smbios1_options->{$k};
1585         $data .= ',' if $data;
1586         $data .= "$k=$smbios1->{$k}";
1587     }
1588     return $data;
1589 }
1590
1591 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1592 sub verify_smbios1 {
1593     my ($value, $noerr) = @_;
1594
1595     return $value if parse_smbios1($value);
1596
1597     return undef if $noerr;
1598
1599     die "unable to parse smbios (type 1) options\n";
1600 }
1601
1602 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1603 sub verify_bootdisk {
1604     my ($value, $noerr) = @_;
1605
1606     return $value if valid_drivename($value);
1607
1608     return undef if $noerr;
1609
1610     die "invalid boot disk '$value'\n";
1611 }
1612
1613 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1614 sub verify_numa {
1615     my ($value, $noerr) = @_;
1616
1617     return $value if parse_numa($value);
1618
1619     return undef if $noerr;
1620
1621     die "unable to parse numa options\n";
1622 }
1623
1624 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1625 sub verify_net {
1626     my ($value, $noerr) = @_;
1627
1628     return $value if parse_net($value);
1629
1630     return undef if $noerr;
1631
1632     die "unable to parse network options\n";
1633 }
1634
1635 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1636 sub verify_drive {
1637     my ($value, $noerr) = @_;
1638
1639     return $value if parse_drive(undef, $value);
1640
1641     return undef if $noerr;
1642
1643     die "unable to parse drive options\n";
1644 }
1645
1646 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1647 sub verify_hostpci {
1648     my ($value, $noerr) = @_;
1649
1650     return $value if parse_hostpci($value);
1651
1652     return undef if $noerr;
1653
1654     die "unable to parse pci id\n";
1655 }
1656
1657 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1658 sub verify_watchdog {
1659     my ($value, $noerr) = @_;
1660
1661     return $value if parse_watchdog($value);
1662
1663     return undef if $noerr;
1664
1665     die "unable to parse watchdog options\n";
1666 }
1667
1668 sub parse_watchdog {
1669     my ($value) = @_;
1670
1671     return undef if !$value;
1672
1673     my $res = {};
1674
1675     foreach my $p (split(/,/, $value)) {
1676         next if $p =~ m/^\s*$/;
1677
1678         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1679             $res->{model} = $2;
1680         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1681             $res->{action} = $2;
1682         } else {
1683             return undef;
1684         }
1685     }
1686
1687     return $res;
1688 }
1689
1690 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1691 sub verify_startup {
1692     my ($value, $noerr) = @_;
1693
1694     return $value if parse_startup($value);
1695
1696     return undef if $noerr;
1697
1698     die "unable to parse startup options\n";
1699 }
1700
1701 sub parse_startup {
1702     my ($value) = @_;
1703
1704     return undef if !$value;
1705
1706     my $res = {};
1707
1708     foreach my $p (split(/,/, $value)) {
1709         next if $p =~ m/^\s*$/;
1710
1711         if ($p =~ m/^(order=)?(\d+)$/) {
1712             $res->{order} = $2;
1713         } elsif ($p =~ m/^up=(\d+)$/) {
1714             $res->{up} = $1;
1715         } elsif ($p =~ m/^down=(\d+)$/) {
1716             $res->{down} = $1;
1717         } else {
1718             return undef;
1719         }
1720     }
1721
1722     return $res;
1723 }
1724
1725 sub parse_usb_device {
1726     my ($value) = @_;
1727
1728     return undef if !$value;
1729
1730     my @dl = split(/,/, $value);
1731     my $found;
1732
1733     my $res = {};
1734     foreach my $v (@dl) {
1735         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1736             $found = 1;
1737             $res->{vendorid} = $2;
1738             $res->{productid} = $4;
1739         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1740             $found = 1;
1741             $res->{hostbus} = $1;
1742             $res->{hostport} = $2;
1743         } elsif ($v =~ m/^spice$/) {
1744             $found = 1;
1745             $res->{spice} = 1;
1746         } else {
1747             return undef;
1748         }
1749     }
1750     return undef if !$found;
1751
1752     return $res;
1753 }
1754
1755 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1756 sub verify_usb_device {
1757     my ($value, $noerr) = @_;
1758
1759     return $value if parse_usb_device($value);
1760
1761     return undef if $noerr;
1762
1763     die "unable to parse usb device\n";
1764 }
1765
1766 # add JSON properties for create and set function
1767 sub json_config_properties {
1768     my $prop = shift;
1769
1770     foreach my $opt (keys %$confdesc) {
1771         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1772         $prop->{$opt} = $confdesc->{$opt};
1773     }
1774
1775     return $prop;
1776 }
1777
1778 sub check_type {
1779     my ($key, $value) = @_;
1780
1781     die "unknown setting '$key'\n" if !$confdesc->{$key};
1782
1783     my $type = $confdesc->{$key}->{type};
1784
1785     if (!defined($value)) {
1786         die "got undefined value\n";
1787     }
1788
1789     if ($value =~ m/[\n\r]/) {
1790         die "property contains a line feed\n";
1791     }
1792
1793     if ($type eq 'boolean') {
1794         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1795         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1796         die "type check ('boolean') failed - got '$value'\n";
1797     } elsif ($type eq 'integer') {
1798         return int($1) if $value =~ m/^(\d+)$/;
1799         die "type check ('integer') failed - got '$value'\n";
1800     } elsif ($type eq 'number') {
1801         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1802         die "type check ('number') failed - got '$value'\n";
1803     } elsif ($type eq 'string') {
1804         if (my $fmt = $confdesc->{$key}->{format}) {
1805             if ($fmt eq 'pve-qm-drive') {
1806                 # special case - we need to pass $key to parse_drive()
1807                 my $drive = parse_drive($key, $value);
1808                 return $value if $drive;
1809                 die "unable to parse drive options\n";
1810             }
1811             PVE::JSONSchema::check_format($fmt, $value);
1812             return $value;
1813         }
1814         $value =~ s/^\"(.*)\"$/$1/;
1815         return $value;
1816     } else {
1817         die "internal error"
1818     }
1819 }
1820
1821 sub lock_config_full {
1822     my ($vmid, $timeout, $code, @param) = @_;
1823
1824     my $filename = config_file_lock($vmid);
1825
1826     my $res = lock_file($filename, $timeout, $code, @param);
1827
1828     die $@ if $@;
1829
1830     return $res;
1831 }
1832
1833 sub lock_config_mode {
1834     my ($vmid, $timeout, $shared, $code, @param) = @_;
1835
1836     my $filename = config_file_lock($vmid);
1837
1838     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1839
1840     die $@ if $@;
1841
1842     return $res;
1843 }
1844
1845 sub lock_config {
1846     my ($vmid, $code, @param) = @_;
1847
1848     return lock_config_full($vmid, 10, $code, @param);
1849 }
1850
1851 sub cfs_config_path {
1852     my ($vmid, $node) = @_;
1853
1854     $node = $nodename if !$node;
1855     return "nodes/$node/qemu-server/$vmid.conf";
1856 }
1857
1858 sub check_iommu_support{
1859     #fixme : need to check IOMMU support
1860     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1861
1862     my $iommu=1;
1863     return $iommu;
1864
1865 }
1866
1867 sub config_file {
1868     my ($vmid, $node) = @_;
1869
1870     my $cfspath = cfs_config_path($vmid, $node);
1871     return "/etc/pve/$cfspath";
1872 }
1873
1874 sub config_file_lock {
1875     my ($vmid) = @_;
1876
1877     return "$lock_dir/lock-$vmid.conf";
1878 }
1879
1880 sub touch_config {
1881     my ($vmid) = @_;
1882
1883     my $conf = config_file($vmid);
1884     utime undef, undef, $conf;
1885 }
1886
1887 sub destroy_vm {
1888     my ($storecfg, $vmid, $keep_empty_config) = @_;
1889
1890     my $conffile = config_file($vmid);
1891
1892     my $conf = load_config($vmid);
1893
1894     check_lock($conf);
1895
1896     # only remove disks owned by this VM
1897     foreach_drive($conf, sub {
1898         my ($ds, $drive) = @_;
1899
1900         return if drive_is_cdrom($drive);
1901
1902         my $volid = $drive->{file};
1903
1904         return if !$volid || $volid =~ m|^/|;
1905
1906         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1907         return if !$path || !$owner || ($owner != $vmid);
1908
1909         PVE::Storage::vdisk_free($storecfg, $volid);
1910     });
1911
1912     if ($keep_empty_config) {
1913         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1914     } else {
1915         unlink $conffile;
1916     }
1917
1918     # also remove unused disk
1919     eval {
1920         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1921
1922         eval {
1923             PVE::Storage::foreach_volid($dl, sub {
1924                 my ($volid, $sid, $volname, $d) = @_;
1925                 PVE::Storage::vdisk_free($storecfg, $volid);
1926             });
1927         };
1928         warn $@ if $@;
1929
1930     };
1931     warn $@ if $@;
1932 }
1933
1934 sub load_config {
1935     my ($vmid, $node) = @_;
1936
1937     my $cfspath = cfs_config_path($vmid, $node);
1938
1939     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1940
1941     die "no such VM ('$vmid')\n" if !defined($conf);
1942
1943     return $conf;
1944 }
1945
1946 sub parse_vm_config {
1947     my ($filename, $raw) = @_;
1948
1949     return undef if !defined($raw);
1950
1951     my $res = {
1952         digest => Digest::SHA::sha1_hex($raw),
1953         snapshots => {},
1954         pending => {},
1955     };
1956
1957     $filename =~ m|/qemu-server/(\d+)\.conf$|
1958         || die "got strange filename '$filename'";
1959
1960     my $vmid = $1;
1961
1962     my $conf = $res;
1963     my $descr = '';
1964     my $section = '';
1965
1966     my @lines = split(/\n/, $raw);
1967     foreach my $line (@lines) {
1968         next if $line =~ m/^\s*$/;
1969
1970         if ($line =~ m/^\[PENDING\]\s*$/i) {
1971             $section = 'pending';
1972             $conf->{description} = $descr if $descr;
1973             $descr = '';
1974             $conf = $res->{$section} = {};
1975             next;
1976
1977         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1978             $section = $1;
1979             $conf->{description} = $descr if $descr;
1980             $descr = '';
1981             $conf = $res->{snapshots}->{$section} = {};
1982             next;
1983         }
1984
1985         if ($line =~ m/^\#(.*)\s*$/) {
1986             $descr .= PVE::Tools::decode_text($1) . "\n";
1987             next;
1988         }
1989
1990         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1991             $descr .= PVE::Tools::decode_text($2);
1992         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1993             $conf->{snapstate} = $1;
1994         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1995             my $key = $1;
1996             my $value = $2;
1997             $conf->{$key} = $value;
1998         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1999             my $value = $1;
2000             if ($section eq 'pending') {
2001                 $conf->{delete} = $value; # we parse this later
2002             } else {
2003                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2004             }
2005         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2006             my $key = $1;
2007             my $value = $2;
2008             eval { $value = check_type($key, $value); };
2009             if ($@) {
2010                 warn "vm $vmid - unable to parse value of '$key' - $@";
2011             } else {
2012                 my $fmt = $confdesc->{$key}->{format};
2013                 if ($fmt && $fmt eq 'pve-qm-drive') {
2014                     my $v = parse_drive($key, $value);
2015                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2016                         $v->{file} = $volid;
2017                         $value = print_drive($vmid, $v);
2018                     } else {
2019                         warn "vm $vmid - unable to parse value of '$key'\n";
2020                         next;
2021                     }
2022                 }
2023
2024                 if ($key eq 'cdrom') {
2025                     $conf->{ide2} = $value;
2026                 } else {
2027                     $conf->{$key} = $value;
2028                 }
2029             }
2030         }
2031     }
2032
2033     $conf->{description} = $descr if $descr;
2034
2035     delete $res->{snapstate}; # just to be sure
2036
2037     return $res;
2038 }
2039
2040 sub write_vm_config {
2041     my ($filename, $conf) = @_;
2042
2043     delete $conf->{snapstate}; # just to be sure
2044
2045     if ($conf->{cdrom}) {
2046         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2047         $conf->{ide2} = $conf->{cdrom};
2048         delete $conf->{cdrom};
2049     }
2050
2051     # we do not use 'smp' any longer
2052     if ($conf->{sockets}) {
2053         delete $conf->{smp};
2054     } elsif ($conf->{smp}) {
2055         $conf->{sockets} = $conf->{smp};
2056         delete $conf->{cores};
2057         delete $conf->{smp};
2058     }
2059
2060     my $used_volids = {};
2061
2062     my $cleanup_config = sub {
2063         my ($cref, $pending, $snapname) = @_;
2064
2065         foreach my $key (keys %$cref) {
2066             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2067                 $key eq 'snapstate' || $key eq 'pending';
2068             my $value = $cref->{$key};
2069             if ($key eq 'delete') {
2070                 die "propertry 'delete' is only allowed in [PENDING]\n"
2071                     if !$pending;
2072                 # fixme: check syntax?
2073                 next;
2074             }
2075             eval { $value = check_type($key, $value); };
2076             die "unable to parse value of '$key' - $@" if $@;
2077
2078             $cref->{$key} = $value;
2079
2080             if (!$snapname && valid_drivename($key)) {
2081                 my $drive = parse_drive($key, $value);
2082                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2083             }
2084         }
2085     };
2086
2087     &$cleanup_config($conf);
2088
2089     &$cleanup_config($conf->{pending}, 1);
2090
2091     foreach my $snapname (keys %{$conf->{snapshots}}) {
2092         die "internal error" if $snapname eq 'pending';
2093         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2094     }
2095
2096     # remove 'unusedX' settings if we re-add a volume
2097     foreach my $key (keys %$conf) {
2098         my $value = $conf->{$key};
2099         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2100             delete $conf->{$key};
2101         }
2102     }
2103
2104     my $generate_raw_config = sub {
2105         my ($conf) = @_;
2106
2107         my $raw = '';
2108
2109         # add description as comment to top of file
2110         my $descr = $conf->{description} || '';
2111         foreach my $cl (split(/\n/, $descr)) {
2112             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2113         }
2114
2115         foreach my $key (sort keys %$conf) {
2116             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2117             $raw .= "$key: $conf->{$key}\n";
2118         }
2119         return $raw;
2120     };
2121
2122     my $raw = &$generate_raw_config($conf);
2123
2124     if (scalar(keys %{$conf->{pending}})){
2125         $raw .= "\n[PENDING]\n";
2126         $raw .= &$generate_raw_config($conf->{pending});
2127     }
2128
2129     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2130         $raw .= "\n[$snapname]\n";
2131         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2132     }
2133
2134     return $raw;
2135 }
2136
2137 sub update_config_nolock {
2138     my ($vmid, $conf, $skiplock) = @_;
2139
2140     check_lock($conf) if !$skiplock;
2141
2142     my $cfspath = cfs_config_path($vmid);
2143
2144     PVE::Cluster::cfs_write_file($cfspath, $conf);
2145 }
2146
2147 sub update_config {
2148     my ($vmid, $conf, $skiplock) = @_;
2149
2150     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2151 }
2152
2153 sub load_defaults {
2154
2155     my $res = {};
2156
2157     # we use static defaults from our JSON schema configuration
2158     foreach my $key (keys %$confdesc) {
2159         if (defined(my $default = $confdesc->{$key}->{default})) {
2160             $res->{$key} = $default;
2161         }
2162     }
2163
2164     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2165     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2166
2167     return $res;
2168 }
2169
2170 sub config_list {
2171     my $vmlist = PVE::Cluster::get_vmlist();
2172     my $res = {};
2173     return $res if !$vmlist || !$vmlist->{ids};
2174     my $ids = $vmlist->{ids};
2175
2176     foreach my $vmid (keys %$ids) {
2177         my $d = $ids->{$vmid};
2178         next if !$d->{node} || $d->{node} ne $nodename;
2179         next if !$d->{type} || $d->{type} ne 'qemu';
2180         $res->{$vmid}->{exists} = 1;
2181     }
2182     return $res;
2183 }
2184
2185 # test if VM uses local resources (to prevent migration)
2186 sub check_local_resources {
2187     my ($conf, $noerr) = @_;
2188
2189     my $loc_res = 0;
2190
2191     $loc_res = 1 if $conf->{hostusb}; # old syntax
2192     $loc_res = 1 if $conf->{hostpci}; # old syntax
2193
2194     foreach my $k (keys %$conf) {
2195         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2196         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2197     }
2198
2199     die "VM uses local resources\n" if $loc_res && !$noerr;
2200
2201     return $loc_res;
2202 }
2203
2204 # check if used storages are available on all nodes (use by migrate)
2205 sub check_storage_availability {
2206     my ($storecfg, $conf, $node) = @_;
2207
2208     foreach_drive($conf, sub {
2209         my ($ds, $drive) = @_;
2210
2211         my $volid = $drive->{file};
2212         return if !$volid;
2213
2214         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2215         return if !$sid;
2216
2217         # check if storage is available on both nodes
2218         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2219         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2220    });
2221 }
2222
2223 # list nodes where all VM images are available (used by has_feature API)
2224 sub shared_nodes {
2225     my ($conf, $storecfg) = @_;
2226
2227     my $nodelist = PVE::Cluster::get_nodelist();
2228     my $nodehash = { map { $_ => 1 } @$nodelist };
2229     my $nodename = PVE::INotify::nodename();
2230
2231     foreach_drive($conf, sub {
2232         my ($ds, $drive) = @_;
2233
2234         my $volid = $drive->{file};
2235         return if !$volid;
2236
2237         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2238         if ($storeid) {
2239             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2240             if ($scfg->{disable}) {
2241                 $nodehash = {};
2242             } elsif (my $avail = $scfg->{nodes}) {
2243                 foreach my $node (keys %$nodehash) {
2244                     delete $nodehash->{$node} if !$avail->{$node};
2245                 }
2246             } elsif (!$scfg->{shared}) {
2247                 foreach my $node (keys %$nodehash) {
2248                     delete $nodehash->{$node} if $node ne $nodename
2249                 }
2250             }
2251         }
2252     });
2253
2254     return $nodehash
2255 }
2256
2257 sub check_lock {
2258     my ($conf) = @_;
2259
2260     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2261 }
2262
2263 sub check_cmdline {
2264     my ($pidfile, $pid) = @_;
2265
2266     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2267     if (defined($fh)) {
2268         my $line = <$fh>;
2269         $fh->close;
2270         return undef if !$line;
2271         my @param = split(/\0/, $line);
2272
2273         my $cmd = $param[0];
2274         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2275
2276         for (my $i = 0; $i < scalar (@param); $i++) {
2277             my $p = $param[$i];
2278             next if !$p;
2279             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2280                 my $p = $param[$i+1];
2281                 return 1 if $p && ($p eq $pidfile);
2282                 return undef;
2283             }
2284         }
2285     }
2286     return undef;
2287 }
2288
2289 sub check_running {
2290     my ($vmid, $nocheck, $node) = @_;
2291
2292     my $filename = config_file($vmid, $node);
2293
2294     die "unable to find configuration file for VM $vmid - no such machine\n"
2295         if !$nocheck && ! -f $filename;
2296
2297     my $pidfile = pidfile_name($vmid);
2298
2299     if (my $fd = IO::File->new("<$pidfile")) {
2300         my $st = stat($fd);
2301         my $line = <$fd>;
2302         close($fd);
2303
2304         my $mtime = $st->mtime;
2305         if ($mtime > time()) {
2306             warn "file '$filename' modified in future\n";
2307         }
2308
2309         if ($line =~ m/^(\d+)$/) {
2310             my $pid = $1;
2311             if (check_cmdline($pidfile, $pid)) {
2312                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2313                     return $pid;
2314                 }
2315             }
2316         }
2317     }
2318
2319     return undef;
2320 }
2321
2322 sub vzlist {
2323
2324     my $vzlist = config_list();
2325
2326     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2327
2328     while (defined(my $de = $fd->read)) {
2329         next if $de !~ m/^(\d+)\.pid$/;
2330         my $vmid = $1;
2331         next if !defined($vzlist->{$vmid});
2332         if (my $pid = check_running($vmid)) {
2333             $vzlist->{$vmid}->{pid} = $pid;
2334         }
2335     }
2336
2337     return $vzlist;
2338 }
2339
2340 sub disksize {
2341     my ($storecfg, $conf) = @_;
2342
2343     my $bootdisk = $conf->{bootdisk};
2344     return undef if !$bootdisk;
2345     return undef if !valid_drivename($bootdisk);
2346
2347     return undef if !$conf->{$bootdisk};
2348
2349     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2350     return undef if !defined($drive);
2351
2352     return undef if drive_is_cdrom($drive);
2353
2354     my $volid = $drive->{file};
2355     return undef if !$volid;
2356
2357     return $drive->{size};
2358 }
2359
2360 my $last_proc_pid_stat;
2361
2362 # get VM status information
2363 # This must be fast and should not block ($full == false)
2364 # We only query KVM using QMP if $full == true (this can be slow)
2365 sub vmstatus {
2366     my ($opt_vmid, $full) = @_;
2367
2368     my $res = {};
2369
2370     my $storecfg = PVE::Storage::config();
2371
2372     my $list = vzlist();
2373     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2374
2375     my $cpucount = $cpuinfo->{cpus} || 1;
2376
2377     foreach my $vmid (keys %$list) {
2378         next if $opt_vmid && ($vmid ne $opt_vmid);
2379
2380         my $cfspath = cfs_config_path($vmid);
2381         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2382
2383         my $d = {};
2384         $d->{pid} = $list->{$vmid}->{pid};
2385
2386         # fixme: better status?
2387         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2388
2389         my $size = disksize($storecfg, $conf);
2390         if (defined($size)) {
2391             $d->{disk} = 0; # no info available
2392             $d->{maxdisk} = $size;
2393         } else {
2394             $d->{disk} = 0;
2395             $d->{maxdisk} = 0;
2396         }
2397
2398         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2399         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2400         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2401
2402         $d->{name} = $conf->{name} || "VM $vmid";
2403         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2404
2405         if ($conf->{balloon}) {
2406             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2407             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2408         }
2409
2410         $d->{uptime} = 0;
2411         $d->{cpu} = 0;
2412         $d->{mem} = 0;
2413
2414         $d->{netout} = 0;
2415         $d->{netin} = 0;
2416
2417         $d->{diskread} = 0;
2418         $d->{diskwrite} = 0;
2419
2420         $d->{template} = is_template($conf);
2421
2422         $res->{$vmid} = $d;
2423     }
2424
2425     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2426     foreach my $dev (keys %$netdev) {
2427         next if $dev !~ m/^tap([1-9]\d*)i/;
2428         my $vmid = $1;
2429         my $d = $res->{$vmid};
2430         next if !$d;
2431
2432         $d->{netout} += $netdev->{$dev}->{receive};
2433         $d->{netin} += $netdev->{$dev}->{transmit};
2434     }
2435
2436     my $ctime = gettimeofday;
2437
2438     foreach my $vmid (keys %$list) {
2439
2440         my $d = $res->{$vmid};
2441         my $pid = $d->{pid};
2442         next if !$pid;
2443
2444         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2445         next if !$pstat; # not running
2446
2447         my $used = $pstat->{utime} + $pstat->{stime};
2448
2449         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2450
2451         if ($pstat->{vsize}) {
2452             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2453         }
2454
2455         my $old = $last_proc_pid_stat->{$pid};
2456         if (!$old) {
2457             $last_proc_pid_stat->{$pid} = {
2458                 time => $ctime,
2459                 used => $used,
2460                 cpu => 0,
2461             };
2462             next;
2463         }
2464
2465         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2466
2467         if ($dtime > 1000) {
2468             my $dutime = $used -  $old->{used};
2469
2470             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2471             $last_proc_pid_stat->{$pid} = {
2472                 time => $ctime,
2473                 used => $used,
2474                 cpu => $d->{cpu},
2475             };
2476         } else {
2477             $d->{cpu} = $old->{cpu};
2478         }
2479     }
2480
2481     return $res if !$full;
2482
2483     my $qmpclient = PVE::QMPClient->new();
2484
2485     my $ballooncb = sub {
2486         my ($vmid, $resp) = @_;
2487
2488         my $info = $resp->{'return'};
2489         return if !$info->{max_mem};
2490
2491         my $d = $res->{$vmid};
2492
2493         # use memory assigned to VM
2494         $d->{maxmem} = $info->{max_mem};
2495         $d->{balloon} = $info->{actual};
2496
2497         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2498             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2499             $d->{freemem} = $info->{free_mem};
2500         }
2501
2502     };
2503
2504     my $blockstatscb = sub {
2505         my ($vmid, $resp) = @_;
2506         my $data = $resp->{'return'} || [];
2507         my $totalrdbytes = 0;
2508         my $totalwrbytes = 0;
2509         for my $blockstat (@$data) {
2510             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2511             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2512         }
2513         $res->{$vmid}->{diskread} = $totalrdbytes;
2514         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2515     };
2516
2517     my $statuscb = sub {
2518         my ($vmid, $resp) = @_;
2519
2520         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2521         # this fails if ballon driver is not loaded, so this must be
2522         # the last commnand (following command are aborted if this fails).
2523         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2524
2525         my $status = 'unknown';
2526         if (!defined($status = $resp->{'return'}->{status})) {
2527             warn "unable to get VM status\n";
2528             return;
2529         }
2530
2531         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2532     };
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536         next if !$res->{$vmid}->{pid}; # not running
2537         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2538     }
2539
2540     $qmpclient->queue_execute(undef, 1);
2541
2542     foreach my $vmid (keys %$list) {
2543         next if $opt_vmid && ($vmid ne $opt_vmid);
2544         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2545     }
2546
2547     return $res;
2548 }
2549
2550 sub foreach_dimm {
2551     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2552
2553     my $dimm_id = 0;
2554     my $current_size = 1024;
2555     my $dimm_size = 512;
2556     return if $current_size == $memory;
2557
2558     for (my $j = 0; $j < 8; $j++) {
2559         for (my $i = 0; $i < 32; $i++) {
2560             my $name = "dimm${dimm_id}";
2561             $dimm_id++;
2562             my $numanode = $i % $sockets;
2563             $current_size += $dimm_size;
2564             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2565             return  $current_size if $current_size >= $memory;
2566         }
2567         $dimm_size *= 2;
2568     }
2569 }
2570
2571 sub foreach_drive {
2572     my ($conf, $func) = @_;
2573
2574     foreach my $ds (keys %$conf) {
2575         next if !valid_drivename($ds);
2576
2577         my $drive = parse_drive($ds, $conf->{$ds});
2578         next if !$drive;
2579
2580         &$func($ds, $drive);
2581     }
2582 }
2583
2584 sub foreach_volid {
2585     my ($conf, $func) = @_;
2586
2587     my $volhash = {};
2588
2589     my $test_volid = sub {
2590         my ($volid, $is_cdrom) = @_;
2591
2592         return if !$volid;
2593
2594         $volhash->{$volid} = $is_cdrom || 0;
2595     };
2596
2597     foreach_drive($conf, sub {
2598         my ($ds, $drive) = @_;
2599         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2600     });
2601
2602     foreach my $snapname (keys %{$conf->{snapshots}}) {
2603         my $snap = $conf->{snapshots}->{$snapname};
2604         &$test_volid($snap->{vmstate}, 0);
2605         foreach_drive($snap, sub {
2606             my ($ds, $drive) = @_;
2607             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2608         });
2609     }
2610
2611     foreach my $volid (keys %$volhash) {
2612         &$func($volid, $volhash->{$volid});
2613     }
2614 }
2615
2616 sub vga_conf_has_spice {
2617     my ($vga) = @_;
2618
2619     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2620
2621     return $1 || 1;
2622 }
2623
2624 sub config_to_command {
2625     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2626
2627     my $cmd = [];
2628     my $globalFlags = [];
2629     my $machineFlags = [];
2630     my $rtcFlags = [];
2631     my $cpuFlags = [];
2632     my $devices = [];
2633     my $pciaddr = '';
2634     my $bridges = {};
2635     my $kvmver = kvm_user_version();
2636     my $vernum = 0; # unknown
2637     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2638         $vernum = $1*1000000+$2*1000;
2639     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2640         $vernum = $1*1000000+$2*1000+$3;
2641     }
2642
2643     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2644
2645     my $have_ovz = -f '/proc/vz/vestat';
2646
2647     my $q35 = machine_type_is_q35($conf);
2648     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2649     my $machine_type = $forcemachine || $conf->{machine};
2650
2651     push @$cmd, '/usr/bin/kvm';
2652
2653     push @$cmd, '-id', $vmid;
2654
2655     my $use_virtio = 0;
2656
2657     my $qmpsocket = qmp_socket($vmid);
2658     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2659     push @$cmd, '-mon', "chardev=qmp,mode=control";
2660
2661     my $socket = vnc_socket($vmid);
2662     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2663
2664     push @$cmd, '-pidfile' , pidfile_name($vmid);
2665
2666     push @$cmd, '-daemonize';
2667
2668     if ($conf->{smbios1}) {
2669         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2670     }
2671
2672     if ($q35) {
2673         # the q35 chipset support native usb2, so we enable usb controller
2674         # by default for this machine type
2675         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2676     } else {
2677         $pciaddr = print_pci_addr("piix3", $bridges);
2678         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2679
2680         my $use_usb2 = 0;
2681         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2682             next if !$conf->{"usb$i"};
2683             $use_usb2 = 1;
2684         }
2685         # include usb device config
2686         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2687     }
2688
2689     my $vga = $conf->{vga};
2690
2691     my $qxlnum = vga_conf_has_spice($vga);
2692     $vga = 'qxl' if $qxlnum;
2693
2694     if (!$vga) {
2695         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2696                                 $conf->{ostype} eq 'win7' ||
2697                                 $conf->{ostype} eq 'w2k8')) {
2698             $vga = 'std';
2699         } else {
2700             $vga = 'cirrus';
2701         }
2702     }
2703
2704     # enable absolute mouse coordinates (needed by vnc)
2705     my $tablet;
2706     if (defined($conf->{tablet})) {
2707         $tablet = $conf->{tablet};
2708     } else {
2709         $tablet = $defaults->{tablet};
2710         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2711         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2712     }
2713
2714     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2715
2716     # host pci devices
2717     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2718         my $d = parse_hostpci($conf->{"hostpci$i"});
2719         next if !$d;
2720
2721         my $pcie = $d->{pcie};
2722         if($pcie){
2723             die "q35 machine model is not enabled" if !$q35;
2724             $pciaddr = print_pcie_addr("hostpci$i");
2725         }else{
2726             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2727         }
2728
2729         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2730         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2731         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2732         if ($xvga && $xvga ne '') {
2733             push @$cpuFlags, 'kvm=off';
2734             $vga = 'none';
2735         }
2736         $driver = "vfio-pci" if $xvga ne '';
2737         my $pcidevices = $d->{pciid};
2738         my $multifunction = 1 if @$pcidevices > 1;
2739
2740         my $j=0;
2741         foreach my $pcidevice (@$pcidevices) {
2742
2743             my $id = "hostpci$i";
2744             $id .= ".$j" if $multifunction;
2745             my $addr = $pciaddr;
2746             $addr .= ".$j" if $multifunction;
2747             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2748
2749             if($j == 0){
2750                 $devicestr .= "$rombar$xvga";
2751                 $devicestr .= ",multifunction=on" if $multifunction;
2752             }
2753
2754             push @$devices, '-device', $devicestr;
2755             $j++;
2756         }
2757     }
2758
2759     # usb devices
2760     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2761         my $d = parse_usb_device($conf->{"usb$i"});
2762         next if !$d;
2763         if ($d->{vendorid} && $d->{productid}) {
2764             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2765         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2766             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2767         } elsif ($d->{spice}) {
2768             # usb redir support for spice
2769             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2770             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2771         }
2772     }
2773
2774     # serial devices
2775     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2776         if (my $path = $conf->{"serial$i"}) {
2777             if ($path eq 'socket') {
2778                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2779                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2780                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2781             } else {
2782                 die "no such serial device\n" if ! -c $path;
2783                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2784                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2785             }
2786         }
2787     }
2788
2789     # parallel devices
2790     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2791         if (my $path = $conf->{"parallel$i"}) {
2792             die "no such parallel device\n" if ! -c $path;
2793             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2794             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2795             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2796         }
2797     }
2798
2799     my $vmname = $conf->{name} || "vm$vmid";
2800
2801     push @$cmd, '-name', $vmname;
2802
2803     my $sockets = 1;
2804     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2805     $sockets = $conf->{sockets} if  $conf->{sockets};
2806
2807     my $cores = $conf->{cores} || 1;
2808
2809     my $maxcpus = $sockets * $cores;
2810
2811     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2812
2813     my $allowed_vcpus = $cpuinfo->{cpus};
2814
2815     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2816         if ($allowed_vcpus < $maxcpus);
2817
2818     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2819
2820     push @$cmd, '-nodefaults';
2821
2822     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2823
2824     my $bootindex_hash = {};
2825     my $i = 1;
2826     foreach my $o (split(//, $bootorder)) {
2827         $bootindex_hash->{$o} = $i*100;
2828         $i++;
2829     }
2830
2831     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2832
2833     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2834
2835     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2836
2837     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2838
2839     # time drift fix
2840     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2841
2842     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2843     my $useLocaltime = $conf->{localtime};
2844
2845     if (my $ost = $conf->{ostype}) {
2846         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2847
2848         if ($ost =~ m/^w/) { # windows
2849             $useLocaltime = 1 if !defined($conf->{localtime});
2850
2851             # use time drift fix when acpi is enabled
2852             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2853                 $tdf = 1 if !defined($conf->{tdf});
2854             }
2855         }
2856
2857         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2858             $ost eq 'wvista') {
2859             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2860             push @$cmd, '-no-hpet';
2861             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2862                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2863                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2864                 push @$cpuFlags , 'hv_time' if !$nokvm;
2865
2866             } else {
2867                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2868             }
2869         }
2870
2871         if ($ost eq 'win7' || $ost eq 'win8') {
2872             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2873         }
2874     }
2875
2876     push @$rtcFlags, 'driftfix=slew' if $tdf;
2877
2878     if ($nokvm) {
2879         push @$machineFlags, 'accel=tcg';
2880     } else {
2881         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2882     }
2883
2884     if ($machine_type) {
2885         push @$machineFlags, "type=${machine_type}";
2886     }
2887
2888     if ($conf->{startdate}) {
2889         push @$rtcFlags, "base=$conf->{startdate}";
2890     } elsif ($useLocaltime) {
2891         push @$rtcFlags, 'base=localtime';
2892     }
2893
2894     my $cpu = $nokvm ? "qemu64" : "kvm64";
2895     $cpu = $conf->{cpu} if $conf->{cpu};
2896
2897     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2898
2899     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2900
2901     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2902
2903     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2904
2905     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2906
2907         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2908         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2909     }
2910
2911     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2912
2913     push @$cmd, '-cpu', "$cpu,enforce";
2914
2915     my $memory = $conf->{memory} || $defaults->{memory};
2916     my $static_memory = 0;
2917     my $dimm_memory = 0;
2918
2919     if ($hotplug_features->{memory}) {
2920         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2921         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2922         $static_memory = $STATICMEM;
2923         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2924         $dimm_memory = $memory - $static_memory;
2925         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2926
2927     } else {
2928
2929         $static_memory = $memory;
2930         push @$cmd, '-m', $static_memory;
2931     }
2932
2933     if ($conf->{numa}) {
2934
2935         my $numa_totalmemory = undef;
2936         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2937             next if !$conf->{"numa$i"};
2938             my $numa = parse_numa($conf->{"numa$i"});
2939             next if !$numa;
2940             # memory
2941             die "missing numa node$i memory value\n" if !$numa->{memory};
2942             my $numa_memory = $numa->{memory};
2943             $numa_totalmemory += $numa_memory;
2944             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2945
2946             # cpus
2947             my $cpus_start = $numa->{cpus}->{start};
2948             die "missing numa node$i cpus\n" if !defined($cpus_start);
2949             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2950             my $cpus = $cpus_start;
2951             if (defined($cpus_end)) {
2952                 $cpus .= "-$cpus_end";
2953                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2954             }
2955
2956             # hostnodes
2957             my $hostnodes_start = $numa->{hostnodes}->{start};
2958             if (defined($hostnodes_start)) {
2959                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2960                 my $hostnodes = $hostnodes_start;
2961                 if (defined($hostnodes_end)) {
2962                     $hostnodes .= "-$hostnodes_end";
2963                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2964                 }
2965
2966                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2967                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2968                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2969                 }
2970
2971                 # policy
2972                 my $policy = $numa->{policy};
2973                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2974                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2975             }
2976
2977             push @$cmd, '-object', $numa_object;
2978             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2979         }
2980
2981         die "total memory for NUMA nodes must be equal to vm static memory\n"
2982             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2983
2984         #if no custom tology, we split memory and cores across numa nodes
2985         if(!$numa_totalmemory) {
2986
2987             my $numa_memory = ($static_memory / $sockets) . "M";
2988
2989             for (my $i = 0; $i < $sockets; $i++)  {
2990
2991                 my $cpustart = ($cores * $i);
2992                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2993                 my $cpus = $cpustart;
2994                 $cpus .= "-$cpuend" if $cpuend;
2995
2996                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2997                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2998             }
2999         }
3000     }
3001
3002     if ($hotplug_features->{memory}) {
3003         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3004             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3005             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3006             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3007
3008             #if dimm_memory is not aligned to dimm map
3009             if($current_size > $memory) {
3010                  $conf->{memory} = $current_size;
3011                  update_config_nolock($vmid, $conf, 1);
3012             }
3013         });
3014     }
3015
3016     push @$cmd, '-S' if $conf->{freeze};
3017
3018     # set keyboard layout
3019     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3020     push @$cmd, '-k', $kb if $kb;
3021
3022     # enable sound
3023     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3024     #push @$cmd, '-soundhw', 'es1370';
3025     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3026
3027     if($conf->{agent}) {
3028         my $qgasocket = qmp_socket($vmid, 1);
3029         my $pciaddr = print_pci_addr("qga0", $bridges);
3030         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3031         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3032         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3033     }
3034
3035     my $spice_port;
3036
3037     if ($qxlnum) {
3038         if ($qxlnum > 1) {
3039             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3040                 for(my $i = 1; $i < $qxlnum; $i++){
3041                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3042                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3043                 }
3044             } else {
3045                 # assume other OS works like Linux
3046                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3047                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3048             }
3049         }
3050
3051         my $pciaddr = print_pci_addr("spice", $bridges);
3052
3053         $spice_port = PVE::Tools::next_spice_port();
3054
3055         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3056
3057         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3058         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3059         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3060     }
3061
3062     # enable balloon by default, unless explicitly disabled
3063     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3064         $pciaddr = print_pci_addr("balloon0", $bridges);
3065         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3066     }
3067
3068     if ($conf->{watchdog}) {
3069         my $wdopts = parse_watchdog($conf->{watchdog});
3070         $pciaddr = print_pci_addr("watchdog", $bridges);
3071         my $watchdog = $wdopts->{model} || 'i6300esb';
3072         push @$devices, '-device', "$watchdog$pciaddr";
3073         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3074     }
3075
3076     my $vollist = [];
3077     my $scsicontroller = {};
3078     my $ahcicontroller = {};
3079     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3080
3081     # Add iscsi initiator name if available
3082     if (my $initiator = get_initiator_name()) {
3083         push @$devices, '-iscsi', "initiator-name=$initiator";
3084     }
3085
3086     foreach_drive($conf, sub {
3087         my ($ds, $drive) = @_;
3088
3089         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3090             push @$vollist, $drive->{file};
3091         }
3092
3093         $use_virtio = 1 if $ds =~ m/^virtio/;
3094
3095         if (drive_is_cdrom ($drive)) {
3096             if ($bootindex_hash->{d}) {
3097                 $drive->{bootindex} = $bootindex_hash->{d};
3098                 $bootindex_hash->{d} += 1;
3099             }
3100         } else {
3101             if ($bootindex_hash->{c}) {
3102                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3103                 $bootindex_hash->{c} += 1;
3104             }
3105         }
3106
3107         if($drive->{interface} eq 'virtio'){
3108            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3109         }
3110
3111         if ($drive->{interface} eq 'scsi') {
3112
3113             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3114
3115             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3116             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3117
3118             my $iothread = '';
3119             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3120                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3121                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3122             }
3123
3124             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread" if !$scsicontroller->{$controller};
3125             $scsicontroller->{$controller}=1;
3126         }
3127
3128         if ($drive->{interface} eq 'sata') {
3129            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3130            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3131            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3132            $ahcicontroller->{$controller}=1;
3133         }
3134
3135         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3136         push @$devices, '-drive',$drive_cmd;
3137         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3138     });
3139
3140     for (my $i = 0; $i < $MAX_NETS; $i++) {
3141          next if !$conf->{"net$i"};
3142          my $d = parse_net($conf->{"net$i"});
3143          next if !$d;
3144
3145          $use_virtio = 1 if $d->{model} eq 'virtio';
3146
3147          if ($bootindex_hash->{n}) {
3148             $d->{bootindex} = $bootindex_hash->{n};
3149             $bootindex_hash->{n} += 1;
3150          }
3151
3152          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3153          push @$devices, '-netdev', $netdevfull;
3154
3155          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3156          push @$devices, '-device', $netdevicefull;
3157     }
3158
3159     if (!$q35) {
3160         # add pci bridges
3161         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3162            $bridges->{1} = 1;
3163            $bridges->{2} = 1;
3164         }
3165
3166         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3167
3168         while (my ($k, $v) = each %$bridges) {
3169             $pciaddr = print_pci_addr("pci.$k");
3170             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3171         }
3172     }
3173
3174     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3175     # when the VM uses virtio devices.
3176     if (!$use_virtio && $have_ovz) {
3177
3178         my $cpuunits = defined($conf->{cpuunits}) ?
3179             $conf->{cpuunits} : $defaults->{cpuunits};
3180
3181         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3182
3183         # fixme: cpulimit is currently ignored
3184         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3185     }
3186
3187     # add custom args
3188     if ($conf->{args}) {
3189         my $aa = PVE::Tools::split_args($conf->{args});
3190         push @$cmd, @$aa;
3191     }
3192
3193     push @$cmd, @$devices;
3194     push @$cmd, '-rtc', join(',', @$rtcFlags)
3195         if scalar(@$rtcFlags);
3196     push @$cmd, '-machine', join(',', @$machineFlags)
3197         if scalar(@$machineFlags);
3198     push @$cmd, '-global', join(',', @$globalFlags)
3199         if scalar(@$globalFlags);
3200
3201     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3202 }
3203
3204 sub vnc_socket {
3205     my ($vmid) = @_;
3206     return "${var_run_tmpdir}/$vmid.vnc";
3207 }
3208
3209 sub spice_port {
3210     my ($vmid) = @_;
3211
3212     my $res = vm_mon_cmd($vmid, 'query-spice');
3213
3214     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3215 }
3216
3217 sub qmp_socket {
3218     my ($vmid, $qga) = @_;
3219     my $sockettype = $qga ? 'qga' : 'qmp';
3220     return "${var_run_tmpdir}/$vmid.$sockettype";
3221 }
3222
3223 sub pidfile_name {
3224     my ($vmid) = @_;
3225     return "${var_run_tmpdir}/$vmid.pid";
3226 }
3227
3228 sub vm_devices_list {
3229     my ($vmid) = @_;
3230
3231     my $res = vm_mon_cmd($vmid, 'query-pci');
3232     my $devices = {};
3233     foreach my $pcibus (@$res) {
3234         foreach my $device (@{$pcibus->{devices}}) {
3235             next if !$device->{'qdev_id'};
3236             if ($device->{'pci_bridge'}) {
3237                 $devices->{$device->{'qdev_id'}} = 1;
3238                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3239                     next if !$bridge_device->{'qdev_id'};
3240                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3241                     $devices->{$device->{'qdev_id'}}++;
3242                 }
3243             } else {
3244                 $devices->{$device->{'qdev_id'}} = 1;
3245             }
3246         }
3247     }
3248
3249     my $resblock = vm_mon_cmd($vmid, 'query-block');
3250     foreach my $block (@$resblock) {
3251         if($block->{device} =~ m/^drive-(\S+)/){
3252                 $devices->{$1} = 1;
3253         }
3254     }
3255
3256     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3257     foreach my $mice (@$resmice) {
3258         if ($mice->{name} eq 'QEMU HID Tablet') {
3259             $devices->{tablet} = 1;
3260             last;
3261         }
3262     }
3263
3264     return $devices;
3265 }
3266
3267 sub vm_deviceplug {
3268     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3269
3270     my $q35 = machine_type_is_q35($conf);
3271
3272     my $devices_list = vm_devices_list($vmid);
3273     return 1 if defined($devices_list->{$deviceid});
3274
3275     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3276
3277     if ($deviceid eq 'tablet') {
3278
3279         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3280
3281     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3282
3283         qemu_iothread_add($vmid, $deviceid, $device);
3284
3285         qemu_driveadd($storecfg, $vmid, $device);
3286         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3287
3288         qemu_deviceadd($vmid, $devicefull);
3289         eval { qemu_deviceaddverify($vmid, $deviceid); };
3290         if (my $err = $@) {
3291             eval { qemu_drivedel($vmid, $deviceid); };
3292             warn $@ if $@;
3293             die $err;
3294         }
3295
3296     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3297
3298
3299         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3300         my $pciaddr = print_pci_addr($deviceid);
3301         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3302
3303         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3304
3305         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3306             qemu_iothread_add($vmid, $deviceid, $device);
3307             $devicefull .= ",iothread=iothread-$deviceid";
3308         }
3309
3310         qemu_deviceadd($vmid, $devicefull);
3311         qemu_deviceaddverify($vmid, $deviceid);
3312
3313     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3314
3315         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3316         qemu_driveadd($storecfg, $vmid, $device);
3317
3318         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3319         eval { qemu_deviceadd($vmid, $devicefull); };
3320         if (my $err = $@) {
3321             eval { qemu_drivedel($vmid, $deviceid); };
3322             warn $@ if $@;
3323             die $err;
3324         }
3325
3326     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3327
3328         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3329         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3330         qemu_deviceadd($vmid, $netdevicefull);
3331         eval { qemu_deviceaddverify($vmid, $deviceid); };
3332         if (my $err = $@) {
3333             eval { qemu_netdevdel($vmid, $deviceid); };
3334             warn $@ if $@;
3335             die $err;
3336         }
3337
3338     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3339
3340         my $bridgeid = $2;
3341         my $pciaddr = print_pci_addr($deviceid);
3342         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3343
3344         qemu_deviceadd($vmid, $devicefull);
3345         qemu_deviceaddverify($vmid, $deviceid);
3346
3347     } else {
3348         die "can't hotplug device '$deviceid'\n";
3349     }
3350
3351     return 1;
3352 }
3353
3354 # fixme: this should raise exceptions on error!
3355 sub vm_deviceunplug {
3356     my ($vmid, $conf, $deviceid) = @_;
3357
3358     my $devices_list = vm_devices_list($vmid);
3359     return 1 if !defined($devices_list->{$deviceid});
3360
3361     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3362
3363     if ($deviceid eq 'tablet') {
3364
3365         qemu_devicedel($vmid, $deviceid);
3366
3367     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3368
3369         qemu_devicedel($vmid, $deviceid);
3370         qemu_devicedelverify($vmid, $deviceid);
3371         qemu_drivedel($vmid, $deviceid);
3372         qemu_iothread_del($conf, $vmid, $deviceid);
3373
3374     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3375
3376         qemu_devicedel($vmid, $deviceid);
3377         qemu_devicedelverify($vmid, $deviceid);
3378         qemu_iothread_del($conf, $vmid, $deviceid);
3379
3380     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3381
3382         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3383         my $device = parse_drive($deviceid, $conf->{$deviceid});
3384         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3385
3386         qemu_devicedel($vmid, $deviceid);
3387         qemu_drivedel($vmid, $deviceid);
3388         qemu_deletescsihw($conf, $vmid, $deviceid);
3389
3390     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3391
3392         qemu_devicedel($vmid, $deviceid);
3393         qemu_devicedelverify($vmid, $deviceid);
3394         qemu_netdevdel($vmid, $deviceid);
3395
3396     } else {
3397         die "can't unplug device '$deviceid'\n";
3398     }
3399
3400     return 1;
3401 }
3402
3403 sub qemu_deviceadd {
3404     my ($vmid, $devicefull) = @_;
3405
3406     $devicefull = "driver=".$devicefull;
3407     my %options =  split(/[=,]/, $devicefull);
3408
3409     vm_mon_cmd($vmid, "device_add" , %options);
3410 }
3411
3412 sub qemu_devicedel {
3413     my ($vmid, $deviceid) = @_;
3414
3415     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3416 }
3417
3418 sub qemu_iothread_add {
3419     my($vmid, $deviceid, $device) = @_;
3420
3421     if ($device->{iothread}) {
3422         my $iothreads = vm_iothreads_list($vmid);
3423         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3424     }
3425 }
3426
3427 sub qemu_iothread_del {
3428     my($conf, $vmid, $deviceid) = @_;
3429
3430     my $device = parse_drive($deviceid, $conf->{$deviceid});
3431     if ($device->{iothread}) {
3432         my $iothreads = vm_iothreads_list($vmid);
3433         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3434     }
3435 }
3436
3437 sub qemu_objectadd {
3438     my($vmid, $objectid, $qomtype) = @_;
3439
3440     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3441
3442     return 1;
3443 }
3444
3445 sub qemu_objectdel {
3446     my($vmid, $objectid) = @_;
3447
3448     vm_mon_cmd($vmid, "object-del", id => $objectid);
3449
3450     return 1;
3451 }
3452
3453 sub qemu_driveadd {
3454     my ($storecfg, $vmid, $device) = @_;
3455
3456     my $drive = print_drive_full($storecfg, $vmid, $device);
3457     $drive =~ s/\\/\\\\/g;
3458     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3459
3460     # If the command succeeds qemu prints: "OK"
3461     return 1 if $ret =~ m/OK/s;
3462
3463     die "adding drive failed: $ret\n";
3464 }
3465
3466 sub qemu_drivedel {
3467     my($vmid, $deviceid) = @_;
3468
3469     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3470     $ret =~ s/^\s+//;
3471
3472     return 1 if $ret eq "";
3473
3474     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3475     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3476
3477     die "deleting drive $deviceid failed : $ret\n";
3478 }
3479
3480 sub qemu_deviceaddverify {
3481     my ($vmid, $deviceid) = @_;
3482
3483     for (my $i = 0; $i <= 5; $i++) {
3484          my $devices_list = vm_devices_list($vmid);
3485          return 1 if defined($devices_list->{$deviceid});
3486          sleep 1;
3487     }
3488
3489     die "error on hotplug device '$deviceid'\n";
3490 }
3491
3492
3493 sub qemu_devicedelverify {
3494     my ($vmid, $deviceid) = @_;
3495
3496     # need to verify that the device is correctly removed as device_del
3497     # is async and empty return is not reliable
3498
3499     for (my $i = 0; $i <= 5; $i++) {
3500          my $devices_list = vm_devices_list($vmid);
3501          return 1 if !defined($devices_list->{$deviceid});
3502          sleep 1;
3503     }
3504
3505     die "error on hot-unplugging device '$deviceid'\n";
3506 }
3507
3508 sub qemu_findorcreatescsihw {
3509     my ($storecfg, $conf, $vmid, $device) = @_;
3510
3511     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3512
3513     my $scsihwid="$controller_prefix$controller";
3514     my $devices_list = vm_devices_list($vmid);
3515
3516     if(!defined($devices_list->{$scsihwid})) {
3517         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3518     }
3519
3520     return 1;
3521 }
3522
3523 sub qemu_deletescsihw {
3524     my ($conf, $vmid, $opt) = @_;
3525
3526     my $device = parse_drive($opt, $conf->{$opt});
3527
3528     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3529         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3530         return 1;
3531     }
3532
3533     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3534
3535     my $devices_list = vm_devices_list($vmid);
3536     foreach my $opt (keys %{$devices_list}) {
3537         if (PVE::QemuServer::valid_drivename($opt)) {
3538             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3539             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3540                 return 1;
3541             }
3542         }
3543     }
3544
3545     my $scsihwid="scsihw$controller";
3546
3547     vm_deviceunplug($vmid, $conf, $scsihwid);
3548
3549     return 1;
3550 }
3551
3552 sub qemu_add_pci_bridge {
3553     my ($storecfg, $conf, $vmid, $device) = @_;
3554
3555     my $bridges = {};
3556
3557     my $bridgeid;
3558
3559     print_pci_addr($device, $bridges);
3560
3561     while (my ($k, $v) = each %$bridges) {
3562         $bridgeid = $k;
3563     }
3564     return 1 if !defined($bridgeid) || $bridgeid < 1;
3565
3566     my $bridge = "pci.$bridgeid";
3567     my $devices_list = vm_devices_list($vmid);
3568
3569     if (!defined($devices_list->{$bridge})) {
3570         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3571     }
3572
3573     return 1;
3574 }
3575
3576 sub qemu_set_link_status {
3577     my ($vmid, $device, $up) = @_;
3578
3579     vm_mon_cmd($vmid, "set_link", name => $device,
3580                up => $up ? JSON::true : JSON::false);
3581 }
3582
3583 sub qemu_netdevadd {
3584     my ($vmid, $conf, $device, $deviceid) = @_;
3585
3586     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3587     my %options =  split(/[=,]/, $netdev);
3588
3589     vm_mon_cmd($vmid, "netdev_add",  %options);
3590     return 1;
3591 }
3592
3593 sub qemu_netdevdel {
3594     my ($vmid, $deviceid) = @_;
3595
3596     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3597 }
3598
3599 sub qemu_cpu_hotplug {
3600     my ($vmid, $conf, $vcpus) = @_;
3601
3602     my $sockets = 1;
3603     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3604     $sockets = $conf->{sockets} if  $conf->{sockets};
3605     my $cores = $conf->{cores} || 1;
3606     my $maxcpus = $sockets * $cores;
3607
3608     $vcpus = $maxcpus if !$vcpus;
3609
3610     die "you can't add more vcpus than maxcpus\n"
3611         if $vcpus > $maxcpus;
3612
3613     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3614     die "online cpu unplug is not yet possible\n"
3615         if $vcpus < $currentvcpus;
3616
3617     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3618     die "vcpus in running vm is different than configuration\n"
3619         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3620
3621     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3622         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3623     }
3624 }
3625
3626 sub qemu_memory_hotplug {
3627     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3628
3629     return $value if !check_running($vmid);
3630
3631     my $memory = $conf->{memory} || $defaults->{memory};
3632     $value = $defaults->{memory} if !$value;
3633     return $value if $value == $memory;
3634
3635     my $static_memory = $STATICMEM;
3636     my $dimm_memory = $memory - $static_memory;
3637
3638     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3639     die "memory unplug is not yet available" if $value < $memory;
3640     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3641
3642
3643     my $sockets = 1;
3644     $sockets = $conf->{sockets} if $conf->{sockets};
3645
3646     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3647         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3648
3649             return if $current_size <= $conf->{memory};
3650
3651             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3652             if (my $err = $@) {
3653                 eval { qemu_objectdel($vmid, "mem-$name"); };
3654                 die $err;
3655             }
3656
3657             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3658             if (my $err = $@) {
3659                 eval { qemu_objectdel($vmid, "mem-$name"); };
3660                 die $err;
3661             }
3662             #update conf after each succesful module hotplug
3663             $conf->{memory} = $current_size;
3664             update_config_nolock($vmid, $conf, 1);
3665     });
3666 }
3667
3668 sub qemu_block_set_io_throttle {
3669     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3670
3671     return if !check_running($vmid) ;
3672
3673     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3674
3675 }
3676
3677 # old code, only used to shutdown old VM after update
3678 sub __read_avail {
3679     my ($fh, $timeout) = @_;
3680
3681     my $sel = new IO::Select;
3682     $sel->add($fh);
3683
3684     my $res = '';
3685     my $buf;
3686
3687     my @ready;
3688     while (scalar (@ready = $sel->can_read($timeout))) {
3689         my $count;
3690         if ($count = $fh->sysread($buf, 8192)) {
3691             if ($buf =~ /^(.*)\(qemu\) $/s) {
3692                 $res .= $1;
3693                 last;
3694             } else {
3695                 $res .= $buf;
3696             }
3697         } else {
3698             if (!defined($count)) {
3699                 die "$!\n";
3700             }
3701             last;
3702         }
3703     }
3704
3705     die "monitor read timeout\n" if !scalar(@ready);
3706
3707     return $res;
3708 }
3709
3710 # old code, only used to shutdown old VM after update
3711 sub vm_monitor_command {
3712     my ($vmid, $cmdstr, $nocheck) = @_;
3713
3714     my $res;
3715
3716     eval {
3717         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3718
3719         my $sname = "${var_run_tmpdir}/$vmid.mon";
3720
3721         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3722             die "unable to connect to VM $vmid socket - $!\n";
3723
3724         my $timeout = 3;
3725
3726         # hack: migrate sometime blocks the monitor (when migrate_downtime
3727         # is set)
3728         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3729             $timeout = 60*60; # 1 hour
3730         }
3731
3732         # read banner;
3733         my $data = __read_avail($sock, $timeout);
3734
3735         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3736             die "got unexpected qemu monitor banner\n";
3737         }
3738
3739         my $sel = new IO::Select;
3740         $sel->add($sock);
3741
3742         if (!scalar(my @ready = $sel->can_write($timeout))) {
3743             die "monitor write error - timeout";
3744         }
3745
3746         my $fullcmd = "$cmdstr\r";
3747
3748         # syslog('info', "VM $vmid monitor command: $cmdstr");
3749
3750         my $b;
3751         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3752             die "monitor write error - $!";
3753         }
3754
3755         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3756
3757         $timeout = 20;
3758
3759         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3760             $timeout = 60*60; # 1 hour
3761         } elsif ($cmdstr =~ m/^(eject|change)/) {
3762             $timeout = 60; # note: cdrom mount command is slow
3763         }
3764         if ($res = __read_avail($sock, $timeout)) {
3765
3766             my @lines = split("\r?\n", $res);
3767
3768             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3769
3770             $res = join("\n", @lines);
3771             $res .= "\n";
3772         }
3773     };
3774
3775     my $err = $@;
3776
3777     if ($err) {
3778         syslog("err", "VM $vmid monitor command failed - $err");
3779         die $err;
3780     }
3781
3782     return $res;
3783 }
3784
3785 sub qemu_block_resize {
3786     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3787
3788     my $running = check_running($vmid);
3789
3790     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3791
3792     return if !$running;
3793
3794     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3795
3796 }
3797
3798 sub qemu_volume_snapshot {
3799     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3800
3801     my $running = check_running($vmid);
3802
3803     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3804
3805     return if !$running;
3806
3807     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3808
3809 }
3810
3811 sub qemu_volume_snapshot_delete {
3812     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3813
3814     my $running = check_running($vmid);
3815
3816     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3817
3818     return if !$running;
3819
3820     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3821 }
3822
3823 sub set_migration_caps {
3824     my ($vmid) = @_;
3825
3826     my $cap_ref = [];
3827
3828     my $enabled_cap = {
3829         "auto-converge" => 1,
3830         "xbzrle" => 0,
3831         "x-rdma-pin-all" => 0,
3832         "zero-blocks" => 0,
3833     };
3834
3835     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3836
3837     for my $supported_capability (@$supported_capabilities) {
3838         push @$cap_ref, {
3839             capability => $supported_capability->{capability},
3840             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3841         };
3842     }
3843
3844     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3845 }
3846
3847 my $fast_plug_option = {
3848     'lock' => 1,
3849     'name' => 1,
3850     'onboot' => 1,
3851     'shares' => 1,
3852     'startup' => 1,
3853 };
3854
3855 # hotplug changes in [PENDING]
3856 # $selection hash can be used to only apply specified options, for
3857 # example: { cores => 1 } (only apply changed 'cores')
3858 # $errors ref is used to return error messages
3859 sub vmconfig_hotplug_pending {
3860     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3861
3862     my $defaults = load_defaults();
3863
3864     # commit values which do not have any impact on running VM first
3865     # Note: those option cannot raise errors, we we do not care about
3866     # $selection and always apply them.
3867
3868     my $add_error = sub {
3869         my ($opt, $msg) = @_;
3870         $errors->{$opt} = "hotplug problem - $msg";
3871     };
3872
3873     my $changes = 0;
3874     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3875         if ($fast_plug_option->{$opt}) {
3876             $conf->{$opt} = $conf->{pending}->{$opt};
3877             delete $conf->{pending}->{$opt};
3878             $changes = 1;
3879         }
3880     }
3881
3882     if ($changes) {
3883         update_config_nolock($vmid, $conf, 1);
3884         $conf = load_config($vmid); # update/reload
3885     }
3886
3887     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3888
3889     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3890     foreach my $opt (@delete) {
3891         next if $selection && !$selection->{$opt};
3892         eval {
3893             if ($opt eq 'hotplug') {
3894                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3895             } elsif ($opt eq 'tablet') {
3896                 die "skip\n" if !$hotplug_features->{usb};
3897                 if ($defaults->{tablet}) {
3898                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3899                 } else {
3900                     vm_deviceunplug($vmid, $conf, $opt);
3901                 }
3902             } elsif ($opt eq 'vcpus') {
3903                 die "skip\n" if !$hotplug_features->{cpu};
3904                 qemu_cpu_hotplug($vmid, $conf, undef);
3905             } elsif ($opt eq 'balloon') {
3906                 # enable balloon device is not hotpluggable
3907                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3908             } elsif ($fast_plug_option->{$opt}) {
3909                 # do nothing
3910             } elsif ($opt =~ m/^net(\d+)$/) {
3911                 die "skip\n" if !$hotplug_features->{network};
3912                 vm_deviceunplug($vmid, $conf, $opt);
3913             } elsif (valid_drivename($opt)) {
3914                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3915                 vm_deviceunplug($vmid, $conf, $opt);
3916                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3917             } elsif ($opt =~ m/^memory$/) {
3918                 die "skip\n" if !$hotplug_features->{memory};
3919                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3920             } else {
3921                 die "skip\n";
3922             }
3923         };
3924         if (my $err = $@) {
3925             &$add_error($opt, $err) if $err ne "skip\n";
3926         } else {
3927             # save new config if hotplug was successful
3928             delete $conf->{$opt};
3929             vmconfig_undelete_pending_option($conf, $opt);
3930             update_config_nolock($vmid, $conf, 1);
3931             $conf = load_config($vmid); # update/reload
3932         }
3933     }
3934
3935     foreach my $opt (keys %{$conf->{pending}}) {
3936         next if $selection && !$selection->{$opt};
3937         my $value = $conf->{pending}->{$opt};
3938         eval {
3939             if ($opt eq 'hotplug') {
3940                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3941             } elsif ($opt eq 'tablet') {
3942                 die "skip\n" if !$hotplug_features->{usb};
3943                 if ($value == 1) {
3944                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3945                 } elsif ($value == 0) {
3946                     vm_deviceunplug($vmid, $conf, $opt);
3947                 }
3948             } elsif ($opt eq 'vcpus') {
3949                 die "skip\n" if !$hotplug_features->{cpu};
3950                 qemu_cpu_hotplug($vmid, $conf, $value);
3951             } elsif ($opt eq 'balloon') {
3952                 # enable/disable balloning device is not hotpluggable
3953                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3954                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3955                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3956
3957                 # allow manual ballooning if shares is set to zero
3958                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3959                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3960                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3961                 }
3962             } elsif ($opt =~ m/^net(\d+)$/) {
3963                 # some changes can be done without hotplug
3964                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3965                                     $vmid, $opt, $value);
3966             } elsif (valid_drivename($opt)) {
3967                 # some changes can be done without hotplug
3968                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3969                                      $vmid, $opt, $value, 1);
3970             } elsif ($opt =~ m/^memory$/) { #dimms
3971                 die "skip\n" if !$hotplug_features->{memory};
3972                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3973             } else {
3974                 die "skip\n";  # skip non-hot-pluggable options
3975             }
3976         };
3977         if (my $err = $@) {
3978             &$add_error($opt, $err) if $err ne "skip\n";
3979         } else {
3980             # save new config if hotplug was successful
3981             $conf->{$opt} = $value;
3982             delete $conf->{pending}->{$opt};
3983             update_config_nolock($vmid, $conf, 1);
3984             $conf = load_config($vmid); # update/reload
3985         }
3986     }
3987 }
3988
3989 sub vmconfig_apply_pending {
3990     my ($vmid, $conf, $storecfg) = @_;
3991
3992     # cold plug
3993
3994     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3995     foreach my $opt (@delete) { # delete
3996         die "internal error" if $opt =~ m/^unused/;
3997         $conf = load_config($vmid); # update/reload
3998         if (!defined($conf->{$opt})) {
3999             vmconfig_undelete_pending_option($conf, $opt);
4000             update_config_nolock($vmid, $conf, 1);
4001         } elsif (valid_drivename($opt)) {
4002             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
4003             vmconfig_undelete_pending_option($conf, $opt);
4004             delete $conf->{$opt};
4005             update_config_nolock($vmid, $conf, 1);
4006         } else {
4007             vmconfig_undelete_pending_option($conf, $opt);
4008             delete $conf->{$opt};
4009             update_config_nolock($vmid, $conf, 1);
4010         }
4011     }
4012
4013     $conf = load_config($vmid); # update/reload
4014
4015     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4016         $conf = load_config($vmid); # update/reload
4017
4018         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4019             # skip if nothing changed
4020         } elsif (valid_drivename($opt)) {
4021             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4022                 if defined($conf->{$opt});
4023             $conf->{$opt} = $conf->{pending}->{$opt};
4024         } else {
4025             $conf->{$opt} = $conf->{pending}->{$opt};
4026         }
4027
4028         delete $conf->{pending}->{$opt};
4029         update_config_nolock($vmid, $conf, 1);
4030     }
4031 }
4032
4033 my $safe_num_ne = sub {
4034     my ($a, $b) = @_;
4035
4036     return 0 if !defined($a) && !defined($b);
4037     return 1 if !defined($a);
4038     return 1 if !defined($b);
4039
4040     return $a != $b;
4041 };
4042
4043 my $safe_string_ne = sub {
4044     my ($a, $b) = @_;
4045
4046     return 0 if !defined($a) && !defined($b);
4047     return 1 if !defined($a);
4048     return 1 if !defined($b);
4049
4050     return $a ne $b;
4051 };
4052
4053 sub vmconfig_update_net {
4054     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4055
4056     my $newnet = parse_net($value);
4057
4058     if ($conf->{$opt}) {
4059         my $oldnet = parse_net($conf->{$opt});
4060
4061         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4062             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4063             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4064             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4065
4066             # for non online change, we try to hot-unplug
4067             die "skip\n" if !$hotplug;
4068             vm_deviceunplug($vmid, $conf, $opt);
4069         } else {
4070
4071             die "internal error" if $opt !~ m/net(\d+)/;
4072             my $iface = "tap${vmid}i$1";
4073
4074             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4075                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4076             }
4077
4078             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4079                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4080                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4081                 PVE::Network::tap_unplug($iface);
4082                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4083             }
4084
4085             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4086                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4087             }
4088
4089             return 1;
4090         }
4091     }
4092
4093     if ($hotplug) {
4094         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4095     } else {
4096         die "skip\n";
4097     }
4098 }
4099
4100 sub vmconfig_update_disk {
4101     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4102
4103     # fixme: do we need force?
4104
4105     my $drive = parse_drive($opt, $value);
4106
4107     if ($conf->{$opt}) {
4108
4109         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4110
4111             my $media = $drive->{media} || 'disk';
4112             my $oldmedia = $old_drive->{media} || 'disk';
4113             die "unable to change media type\n" if $media ne $oldmedia;
4114
4115             if (!drive_is_cdrom($old_drive)) {
4116
4117                 if ($drive->{file} ne $old_drive->{file}) {
4118
4119                     die "skip\n" if !$hotplug;
4120
4121                     # unplug and register as unused
4122                     vm_deviceunplug($vmid, $conf, $opt);
4123                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4124
4125                 } else {
4126                     # update existing disk
4127
4128                     # skip non hotpluggable value
4129                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4130                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4131                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4132                         die "skip\n";
4133                     }
4134
4135                     # apply throttle
4136                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4137                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4138                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4139                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4140                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4141                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4142                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4143                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4144                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4145                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4146                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4147                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4148
4149                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4150                                                    ($drive->{mbps} || 0)*1024*1024,
4151                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4152                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4153                                                    $drive->{iops} || 0,
4154                                                    $drive->{iops_rd} || 0,
4155                                                    $drive->{iops_wr} || 0,
4156                                                    ($drive->{mbps_max} || 0)*1024*1024,
4157                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4158                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4159                                                    $drive->{iops_max} || 0,
4160                                                    $drive->{iops_rd_max} || 0,
4161                                                    $drive->{iops_wr_max} || 0);
4162
4163                     }
4164
4165                     return 1;
4166                 }
4167
4168             } else { # cdrom
4169
4170                 if ($drive->{file} eq 'none') {
4171                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4172                 } else {
4173                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4174                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4175                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4176                 }
4177
4178                 return 1;
4179             }
4180         }
4181     }
4182
4183     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4184     # hotplug new disks
4185     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4186 }
4187
4188 sub vm_start {
4189     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4190
4191     lock_config($vmid, sub {
4192         my $conf = load_config($vmid, $migratedfrom);
4193
4194         die "you can't start a vm if it's a template\n" if is_template($conf);
4195
4196         check_lock($conf) if !$skiplock;
4197
4198         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4199
4200         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4201             vmconfig_apply_pending($vmid, $conf, $storecfg);
4202             $conf = load_config($vmid); # update/reload
4203         }
4204
4205         my $defaults = load_defaults();
4206
4207         # set environment variable useful inside network script
4208         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4209
4210         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4211
4212         my $migrate_port = 0;
4213         my $migrate_uri;
4214         if ($statefile) {
4215             if ($statefile eq 'tcp') {
4216                 my $localip = "localhost";
4217                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4218                 if ($datacenterconf->{migration_unsecure}) {
4219                         my $nodename = PVE::INotify::nodename();
4220                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4221                 }
4222                 $migrate_port = PVE::Tools::next_migrate_port();
4223                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4224                 push @$cmd, '-incoming', $migrate_uri;
4225                 push @$cmd, '-S';
4226             } else {
4227                 push @$cmd, '-loadstate', $statefile;
4228             }
4229         } elsif ($paused) {
4230             push @$cmd, '-S';
4231         }
4232
4233         # host pci devices
4234         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4235           my $d = parse_hostpci($conf->{"hostpci$i"});
4236           next if !$d;
4237           my $pcidevices = $d->{pciid};
4238           foreach my $pcidevice (@$pcidevices) {
4239                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4240
4241                 my $info = pci_device_info("0000:$pciid");
4242                 die "IOMMU not present\n" if !check_iommu_support();
4243                 die "no pci device info for device '$pciid'\n" if !$info;
4244
4245                 if ($d->{driver} && $d->{driver} eq "vfio") {
4246                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4247                 } else {
4248                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4249                 }
4250
4251                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4252           }
4253         }
4254
4255         PVE::Storage::activate_volumes($storecfg, $vollist);
4256
4257         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4258                     umask => 0077); };
4259         my $err = $@;
4260         die "start failed: $err" if $err;
4261
4262         print "migration listens on $migrate_uri\n" if $migrate_uri;
4263
4264         if ($statefile && $statefile ne 'tcp')  {
4265             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4266             warn $@ if $@;
4267         }
4268
4269         if ($migratedfrom) {
4270
4271             eval {
4272                 set_migration_caps($vmid);
4273             };
4274             warn $@ if $@;
4275
4276             if ($spice_port) {
4277                 print "spice listens on port $spice_port\n";
4278                 if ($spice_ticket) {
4279                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4280                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4281                 }
4282             }
4283
4284         } else {
4285
4286             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4287                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4288                     if $conf->{balloon};
4289             }
4290
4291             foreach my $opt (keys %$conf) {
4292                 next if $opt !~  m/^net\d+$/;
4293                 my $nicconf = parse_net($conf->{$opt});
4294                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4295             }
4296         }
4297
4298         vm_mon_cmd_nocheck($vmid, 'qom-set',
4299                     path => "machine/peripheral/balloon0",
4300                     property => "guest-stats-polling-interval",
4301                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4302
4303     });
4304 }
4305
4306 sub vm_mon_cmd {
4307     my ($vmid, $execute, %params) = @_;
4308
4309     my $cmd = { execute => $execute, arguments => \%params };
4310     vm_qmp_command($vmid, $cmd);
4311 }
4312
4313 sub vm_mon_cmd_nocheck {
4314     my ($vmid, $execute, %params) = @_;
4315
4316     my $cmd = { execute => $execute, arguments => \%params };
4317     vm_qmp_command($vmid, $cmd, 1);
4318 }
4319
4320 sub vm_qmp_command {
4321     my ($vmid, $cmd, $nocheck) = @_;
4322
4323     my $res;
4324
4325     my $timeout;
4326     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4327         $timeout = $cmd->{arguments}->{timeout};
4328         delete $cmd->{arguments}->{timeout};
4329     }
4330
4331     eval {
4332         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4333         my $sname = qmp_socket($vmid);
4334         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4335             my $qmpclient = PVE::QMPClient->new();
4336
4337             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4338         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4339             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4340                 if scalar(%{$cmd->{arguments}});
4341             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4342         } else {
4343             die "unable to open monitor socket\n";
4344         }
4345     };
4346     if (my $err = $@) {
4347         syslog("err", "VM $vmid qmp command failed - $err");
4348         die $err;
4349     }
4350
4351     return $res;
4352 }
4353
4354 sub vm_human_monitor_command {
4355     my ($vmid, $cmdline) = @_;
4356
4357     my $res;
4358
4359     my $cmd = {
4360         execute => 'human-monitor-command',
4361         arguments => { 'command-line' => $cmdline},
4362     };
4363
4364     return vm_qmp_command($vmid, $cmd);
4365 }
4366
4367 sub vm_commandline {
4368     my ($storecfg, $vmid) = @_;
4369
4370     my $conf = load_config($vmid);
4371
4372     my $defaults = load_defaults();
4373
4374     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4375
4376     return join(' ', @$cmd);
4377 }
4378
4379 sub vm_reset {
4380     my ($vmid, $skiplock) = @_;
4381
4382     lock_config($vmid, sub {
4383
4384         my $conf = load_config($vmid);
4385
4386         check_lock($conf) if !$skiplock;
4387
4388         vm_mon_cmd($vmid, "system_reset");
4389     });
4390 }
4391
4392 sub get_vm_volumes {
4393     my ($conf) = @_;
4394
4395     my $vollist = [];
4396     foreach_volid($conf, sub {
4397         my ($volid, $is_cdrom) = @_;
4398
4399         return if $volid =~ m|^/|;
4400
4401         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4402         return if !$sid;
4403
4404         push @$vollist, $volid;
4405     });
4406
4407     return $vollist;
4408 }
4409
4410 sub vm_stop_cleanup {
4411     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4412
4413     eval {
4414         fairsched_rmnod($vmid); # try to destroy group
4415
4416         if (!$keepActive) {
4417             my $vollist = get_vm_volumes($conf);
4418             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4419         }
4420
4421         foreach my $ext (qw(mon qmp pid vnc qga)) {
4422             unlink "/var/run/qemu-server/${vmid}.$ext";
4423         }
4424
4425         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4426     };
4427     warn $@ if $@; # avoid errors - just warn
4428 }
4429
4430 # Note: use $nockeck to skip tests if VM configuration file exists.
4431 # We need that when migration VMs to other nodes (files already moved)
4432 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4433 sub vm_stop {
4434     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4435
4436     $force = 1 if !defined($force) && !$shutdown;
4437
4438     if ($migratedfrom){
4439         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4440         kill 15, $pid if $pid;
4441         my $conf = load_config($vmid, $migratedfrom);
4442         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4443         return;
4444     }
4445
4446     lock_config($vmid, sub {
4447
4448         my $pid = check_running($vmid, $nocheck);
4449         return if !$pid;
4450
4451         my $conf;
4452         if (!$nocheck) {
4453             $conf = load_config($vmid);
4454             check_lock($conf) if !$skiplock;
4455             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4456                 my $opts = parse_startup($conf->{startup});
4457                 $timeout = $opts->{down} if $opts->{down};
4458             }
4459         }
4460
4461         $timeout = 60 if !defined($timeout);
4462
4463         eval {
4464             if ($shutdown) {
4465                 if (defined($conf) && $conf->{agent}) {
4466                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4467                 } else {
4468                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4469                 }
4470             } else {
4471                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4472             }
4473         };
4474         my $err = $@;
4475
4476         if (!$err) {
4477             my $count = 0;
4478             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4479                 $count++;
4480                 sleep 1;
4481             }
4482
4483             if ($count >= $timeout) {
4484                 if ($force) {
4485                     warn "VM still running - terminating now with SIGTERM\n";
4486                     kill 15, $pid;
4487                 } else {
4488                     die "VM quit/powerdown failed - got timeout\n";
4489                 }
4490             } else {
4491                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4492                 return;
4493             }
4494         } else {
4495             if ($force) {
4496                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4497                 kill 15, $pid;
4498             } else {
4499                 die "VM quit/powerdown failed\n";
4500             }
4501         }
4502
4503         # wait again
4504         $timeout = 10;
4505
4506         my $count = 0;
4507         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4508             $count++;
4509             sleep 1;
4510         }
4511
4512         if ($count >= $timeout) {
4513             warn "VM still running - terminating now with SIGKILL\n";
4514             kill 9, $pid;
4515             sleep 1;
4516         }
4517
4518         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4519    });
4520 }
4521
4522 sub vm_suspend {
4523     my ($vmid, $skiplock) = @_;
4524
4525     lock_config($vmid, sub {
4526
4527         my $conf = load_config($vmid);
4528
4529         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4530
4531         vm_mon_cmd($vmid, "stop");
4532     });
4533 }
4534
4535 sub vm_resume {
4536     my ($vmid, $skiplock) = @_;
4537
4538     lock_config($vmid, sub {
4539
4540         my $conf = load_config($vmid);
4541
4542         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4543
4544         vm_mon_cmd($vmid, "cont");
4545     });
4546 }
4547
4548 sub vm_sendkey {
4549     my ($vmid, $skiplock, $key) = @_;
4550
4551     lock_config($vmid, sub {
4552
4553         my $conf = load_config($vmid);
4554
4555         # there is no qmp command, so we use the human monitor command
4556         vm_human_monitor_command($vmid, "sendkey $key");
4557     });
4558 }
4559
4560 sub vm_destroy {
4561     my ($storecfg, $vmid, $skiplock) = @_;
4562
4563     lock_config($vmid, sub {
4564
4565         my $conf = load_config($vmid);
4566
4567         check_lock($conf) if !$skiplock;
4568
4569         if (!check_running($vmid)) {
4570             fairsched_rmnod($vmid); # try to destroy group
4571             destroy_vm($storecfg, $vmid);
4572         } else {
4573             die "VM $vmid is running - destroy failed\n";
4574         }
4575     });
4576 }
4577
4578 # pci helpers
4579
4580 sub file_write {
4581     my ($filename, $buf) = @_;
4582
4583     my $fh = IO::File->new($filename, "w");
4584     return undef if !$fh;
4585
4586     my $res = print $fh $buf;
4587
4588     $fh->close();
4589
4590     return $res;
4591 }
4592
4593 sub pci_device_info {
4594     my ($name) = @_;
4595
4596     my $res;
4597
4598     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4599     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4600
4601     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4602     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4603
4604     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4605     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4606
4607     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4608     return undef if !defined($product) || $product !~ s/^0x//;
4609
4610     $res = {
4611         name => $name,
4612         vendor => $vendor,
4613         product => $product,
4614         domain => $domain,
4615         bus => $bus,
4616         slot => $slot,
4617         func => $func,
4618         irq => $irq,
4619         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4620     };
4621
4622     return $res;
4623 }
4624
4625 sub pci_dev_reset {
4626     my ($dev) = @_;
4627
4628     my $name = $dev->{name};
4629
4630     my $fn = "$pcisysfs/devices/$name/reset";
4631
4632     return file_write($fn, "1");
4633 }
4634
4635 sub pci_dev_bind_to_stub {
4636     my ($dev) = @_;
4637
4638     my $name = $dev->{name};
4639
4640     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4641     return 1 if -d $testdir;
4642
4643     my $data = "$dev->{vendor} $dev->{product}";
4644     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4645
4646     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4647     if (!file_write($fn, $name)) {
4648         return undef if -f $fn;
4649     }
4650
4651     $fn = "$pcisysfs/drivers/pci-stub/bind";
4652     if (! -d $testdir) {
4653         return undef if !file_write($fn, $name);
4654     }
4655
4656     return -d $testdir;
4657 }
4658
4659 sub pci_dev_bind_to_vfio {
4660     my ($dev) = @_;
4661
4662     my $name = $dev->{name};
4663
4664     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4665
4666     if (!-d $vfio_basedir) {
4667         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4668     }
4669     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4670
4671     my $testdir = "$vfio_basedir/$name";
4672     return 1 if -d $testdir;
4673
4674     my $data = "$dev->{vendor} $dev->{product}";
4675     return undef if !file_write("$vfio_basedir/new_id", $data);
4676
4677     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4678     if (!file_write($fn, $name)) {
4679         return undef if -f $fn;
4680     }
4681
4682     $fn = "$vfio_basedir/bind";
4683     if (! -d $testdir) {
4684         return undef if !file_write($fn, $name);
4685     }
4686
4687     return -d $testdir;
4688 }
4689
4690 sub pci_dev_group_bind_to_vfio {
4691     my ($pciid) = @_;
4692
4693     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4694
4695     if (!-d $vfio_basedir) {
4696         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4697     }
4698     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4699
4700     # get IOMMU group devices
4701     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4702       my @devs = grep /^0000:/, readdir($D);
4703     closedir($D);
4704
4705     foreach my $pciid (@devs) {
4706         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4707
4708         # pci bridges, switches or root ports are not supported
4709         # they have a pci_bus subdirectory so skip them
4710         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4711
4712         my $info = pci_device_info($1);
4713         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4714     }
4715
4716     return 1;
4717 }
4718
4719 sub print_pci_addr {
4720     my ($id, $bridges) = @_;
4721
4722     my $res = '';
4723     my $devices = {
4724         piix3 => { bus => 0, addr => 1 },
4725         #addr2 : first videocard
4726         balloon0 => { bus => 0, addr => 3 },
4727         watchdog => { bus => 0, addr => 4 },
4728         scsihw0 => { bus => 0, addr => 5 },
4729         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4730         scsihw1 => { bus => 0, addr => 6 },
4731         ahci0 => { bus => 0, addr => 7 },
4732         qga0 => { bus => 0, addr => 8 },
4733         spice => { bus => 0, addr => 9 },
4734         virtio0 => { bus => 0, addr => 10 },
4735         virtio1 => { bus => 0, addr => 11 },
4736         virtio2 => { bus => 0, addr => 12 },
4737         virtio3 => { bus => 0, addr => 13 },
4738         virtio4 => { bus => 0, addr => 14 },
4739         virtio5 => { bus => 0, addr => 15 },
4740         hostpci0 => { bus => 0, addr => 16 },
4741         hostpci1 => { bus => 0, addr => 17 },
4742         net0 => { bus => 0, addr => 18 },
4743         net1 => { bus => 0, addr => 19 },
4744         net2 => { bus => 0, addr => 20 },
4745         net3 => { bus => 0, addr => 21 },
4746         net4 => { bus => 0, addr => 22 },
4747         net5 => { bus => 0, addr => 23 },
4748         vga1 => { bus => 0, addr => 24 },
4749         vga2 => { bus => 0, addr => 25 },
4750         vga3 => { bus => 0, addr => 26 },
4751         hostpci2 => { bus => 0, addr => 27 },
4752         hostpci3 => { bus => 0, addr => 28 },
4753         #addr29 : usb-host (pve-usb.cfg)
4754         'pci.1' => { bus => 0, addr => 30 },
4755         'pci.2' => { bus => 0, addr => 31 },
4756         'net6' => { bus => 1, addr => 1 },
4757         'net7' => { bus => 1, addr => 2 },
4758         'net8' => { bus => 1, addr => 3 },
4759         'net9' => { bus => 1, addr => 4 },
4760         'net10' => { bus => 1, addr => 5 },
4761         'net11' => { bus => 1, addr => 6 },
4762         'net12' => { bus => 1, addr => 7 },
4763         'net13' => { bus => 1, addr => 8 },
4764         'net14' => { bus => 1, addr => 9 },
4765         'net15' => { bus => 1, addr => 10 },
4766         'net16' => { bus => 1, addr => 11 },
4767         'net17' => { bus => 1, addr => 12 },
4768         'net18' => { bus => 1, addr => 13 },
4769         'net19' => { bus => 1, addr => 14 },
4770         'net20' => { bus => 1, addr => 15 },
4771         'net21' => { bus => 1, addr => 16 },
4772         'net22' => { bus => 1, addr => 17 },
4773         'net23' => { bus => 1, addr => 18 },
4774         'net24' => { bus => 1, addr => 19 },
4775         'net25' => { bus => 1, addr => 20 },
4776         'net26' => { bus => 1, addr => 21 },
4777         'net27' => { bus => 1, addr => 22 },
4778         'net28' => { bus => 1, addr => 23 },
4779         'net29' => { bus => 1, addr => 24 },
4780         'net30' => { bus => 1, addr => 25 },
4781         'net31' => { bus => 1, addr => 26 },
4782         'virtio6' => { bus => 2, addr => 1 },
4783         'virtio7' => { bus => 2, addr => 2 },
4784         'virtio8' => { bus => 2, addr => 3 },
4785         'virtio9' => { bus => 2, addr => 4 },
4786         'virtio10' => { bus => 2, addr => 5 },
4787         'virtio11' => { bus => 2, addr => 6 },
4788         'virtio12' => { bus => 2, addr => 7 },
4789         'virtio13' => { bus => 2, addr => 8 },
4790         'virtio14' => { bus => 2, addr => 9 },
4791         'virtio15' => { bus => 2, addr => 10 },
4792         'virtioscsi0' => { bus => 3, addr => 1 },
4793         'virtioscsi1' => { bus => 3, addr => 2 },
4794         'virtioscsi2' => { bus => 3, addr => 3 },
4795         'virtioscsi3' => { bus => 3, addr => 4 },
4796         'virtioscsi4' => { bus => 3, addr => 5 },
4797         'virtioscsi5' => { bus => 3, addr => 6 },
4798         'virtioscsi6' => { bus => 3, addr => 7 },
4799         'virtioscsi7' => { bus => 3, addr => 8 },
4800         'virtioscsi8' => { bus => 3, addr => 9 },
4801         'virtioscsi9' => { bus => 3, addr => 10 },
4802         'virtioscsi10' => { bus => 3, addr => 11 },
4803         'virtioscsi11' => { bus => 3, addr => 12 },
4804         'virtioscsi12' => { bus => 3, addr => 13 },
4805         'virtioscsi13' => { bus => 3, addr => 14 },
4806         'virtioscsi14' => { bus => 3, addr => 15 },
4807         'virtioscsi15' => { bus => 3, addr => 16 },
4808         'virtioscsi16' => { bus => 3, addr => 17 },
4809         'virtioscsi17' => { bus => 3, addr => 18 },
4810         'virtioscsi18' => { bus => 3, addr => 19 },
4811         'virtioscsi19' => { bus => 3, addr => 20 },
4812         'virtioscsi20' => { bus => 3, addr => 21 },
4813         'virtioscsi21' => { bus => 3, addr => 22 },
4814         'virtioscsi22' => { bus => 3, addr => 23 },
4815         'virtioscsi23' => { bus => 3, addr => 24 },
4816         'virtioscsi24' => { bus => 3, addr => 25 },
4817         'virtioscsi25' => { bus => 3, addr => 26 },
4818         'virtioscsi26' => { bus => 3, addr => 27 },
4819         'virtioscsi27' => { bus => 3, addr => 28 },
4820         'virtioscsi28' => { bus => 3, addr => 29 },
4821         'virtioscsi29' => { bus => 3, addr => 30 },
4822         'virtioscsi30' => { bus => 3, addr => 31 },
4823
4824     };
4825
4826     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4827            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4828            my $bus = $devices->{$id}->{bus};
4829            $res = ",bus=pci.$bus,addr=$addr";
4830            $bridges->{$bus} = 1 if $bridges;
4831     }
4832     return $res;
4833
4834 }
4835
4836 sub print_pcie_addr {
4837     my ($id) = @_;
4838
4839     my $res = '';
4840     my $devices = {
4841         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4842         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4843         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4844         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4845     };
4846
4847     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4848            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4849            my $bus = $devices->{$id}->{bus};
4850            $res = ",bus=$bus,addr=$addr";
4851     }
4852     return $res;
4853
4854 }
4855
4856 # vzdump restore implementaion
4857
4858 sub tar_archive_read_firstfile {
4859     my $archive = shift;
4860
4861     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4862
4863     # try to detect archive type first
4864     my $pid = open (TMP, "tar tf '$archive'|") ||
4865         die "unable to open file '$archive'\n";
4866     my $firstfile = <TMP>;
4867     kill 15, $pid;
4868     close TMP;
4869
4870     die "ERROR: archive contaions no data\n" if !$firstfile;
4871     chomp $firstfile;
4872
4873     return $firstfile;
4874 }
4875
4876 sub tar_restore_cleanup {
4877     my ($storecfg, $statfile) = @_;
4878
4879     print STDERR "starting cleanup\n";
4880
4881     if (my $fd = IO::File->new($statfile, "r")) {
4882         while (defined(my $line = <$fd>)) {
4883             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4884                 my $volid = $2;
4885                 eval {
4886                     if ($volid =~ m|^/|) {
4887                         unlink $volid || die 'unlink failed\n';
4888                     } else {
4889                         PVE::Storage::vdisk_free($storecfg, $volid);
4890                     }
4891                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4892                 };
4893                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4894             } else {
4895                 print STDERR "unable to parse line in statfile - $line";
4896             }
4897         }
4898         $fd->close();
4899     }
4900 }
4901
4902 sub restore_archive {
4903     my ($archive, $vmid, $user, $opts) = @_;
4904
4905     my $format = $opts->{format};
4906     my $comp;
4907
4908     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4909         $format = 'tar' if !$format;
4910         $comp = 'gzip';
4911     } elsif ($archive =~ m/\.tar$/) {
4912         $format = 'tar' if !$format;
4913     } elsif ($archive =~ m/.tar.lzo$/) {
4914         $format = 'tar' if !$format;
4915         $comp = 'lzop';
4916     } elsif ($archive =~ m/\.vma$/) {
4917         $format = 'vma' if !$format;
4918     } elsif ($archive =~ m/\.vma\.gz$/) {
4919         $format = 'vma' if !$format;
4920         $comp = 'gzip';
4921     } elsif ($archive =~ m/\.vma\.lzo$/) {
4922         $format = 'vma' if !$format;
4923         $comp = 'lzop';
4924     } else {
4925         $format = 'vma' if !$format; # default
4926     }
4927
4928     # try to detect archive format
4929     if ($format eq 'tar') {
4930         return restore_tar_archive($archive, $vmid, $user, $opts);
4931     } else {
4932         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4933     }
4934 }
4935
4936 sub restore_update_config_line {
4937     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4938
4939     return if $line =~ m/^\#qmdump\#/;
4940     return if $line =~ m/^\#vzdump\#/;
4941     return if $line =~ m/^lock:/;
4942     return if $line =~ m/^unused\d+:/;
4943     return if $line =~ m/^parent:/;
4944     return if $line =~ m/^template:/; # restored VM is never a template
4945
4946     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4947         # try to convert old 1.X settings
4948         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4949         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4950             my ($model, $macaddr) = split(/\=/, $devconfig);
4951             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4952             my $net = {
4953                 model => $model,
4954                 bridge => "vmbr$ind",
4955                 macaddr => $macaddr,
4956             };
4957             my $netstr = print_net($net);
4958
4959             print $outfd "net$cookie->{netcount}: $netstr\n";
4960             $cookie->{netcount}++;
4961         }
4962     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4963         my ($id, $netstr) = ($1, $2);
4964         my $net = parse_net($netstr);
4965         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4966         $netstr = print_net($net);
4967         print $outfd "$id: $netstr\n";
4968     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4969         my $virtdev = $1;
4970         my $value = $3;
4971         if ($line =~ m/backup=no/) {
4972             print $outfd "#$line";
4973         } elsif ($virtdev && $map->{$virtdev}) {
4974             my $di = parse_drive($virtdev, $value);
4975             delete $di->{format}; # format can change on restore
4976             $di->{file} = $map->{$virtdev};
4977             $value = print_drive($vmid, $di);
4978             print $outfd "$virtdev: $value\n";
4979         } else {
4980             print $outfd $line;
4981         }
4982     } else {
4983         print $outfd $line;
4984     }
4985 }
4986
4987 sub scan_volids {
4988     my ($cfg, $vmid) = @_;
4989
4990     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4991
4992     my $volid_hash = {};
4993     foreach my $storeid (keys %$info) {
4994         foreach my $item (@{$info->{$storeid}}) {
4995             next if !($item->{volid} && $item->{size});
4996             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4997             $volid_hash->{$item->{volid}} = $item;
4998         }
4999     }
5000
5001     return $volid_hash;
5002 }
5003
5004 sub get_used_paths {
5005     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5006
5007     my $used_path = {};
5008
5009     my $scan_config = sub {
5010         my ($cref, $snapname) = @_;
5011
5012         foreach my $key (keys %$cref) {
5013             my $value = $cref->{$key};
5014             if (valid_drivename($key)) {
5015                 next if $skip_drive && $key eq $skip_drive;
5016                 my $drive = parse_drive($key, $value);
5017                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5018                 if ($drive->{file} =~ m!^/!) {
5019                     $used_path->{$drive->{file}}++; # = 1;
5020                 } else {
5021                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5022                     next if !$storeid;
5023                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5024                     next if !$scfg;
5025                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5026                     $used_path->{$path}++; # = 1;
5027                 }
5028             }
5029         }
5030     };
5031
5032     &$scan_config($conf);
5033
5034     undef $skip_drive;
5035
5036     if ($scan_snapshots) {
5037         foreach my $snapname (keys %{$conf->{snapshots}}) {
5038             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5039         }
5040     }
5041
5042     return $used_path;
5043 }
5044
5045 sub update_disksize {
5046     my ($vmid, $conf, $volid_hash) = @_;
5047
5048     my $changes;
5049
5050     my $used = {};
5051
5052     # Note: it is allowed to define multiple storages with same path (alias), so
5053     # we need to check both 'volid' and real 'path' (two different volid can point
5054     # to the same path).
5055
5056     my $usedpath = {};
5057
5058     # update size info
5059     foreach my $opt (keys %$conf) {
5060         if (valid_drivename($opt)) {
5061             my $drive = parse_drive($opt, $conf->{$opt});
5062             my $volid = $drive->{file};
5063             next if !$volid;
5064
5065             $used->{$volid} = 1;
5066             if ($volid_hash->{$volid} &&
5067                 (my $path = $volid_hash->{$volid}->{path})) {
5068                 $usedpath->{$path} = 1;
5069             }
5070
5071             next if drive_is_cdrom($drive);
5072             next if !$volid_hash->{$volid};
5073
5074             $drive->{size} = $volid_hash->{$volid}->{size};
5075             my $new = print_drive($vmid, $drive);
5076             if ($new ne $conf->{$opt}) {
5077                 $changes = 1;
5078                 $conf->{$opt} = $new;
5079             }
5080         }
5081     }
5082
5083     # remove 'unusedX' entry if volume is used
5084     foreach my $opt (keys %$conf) {
5085         next if $opt !~ m/^unused\d+$/;
5086         my $volid = $conf->{$opt};
5087         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5088         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5089             $changes = 1;
5090             delete $conf->{$opt};
5091         }
5092     }
5093
5094     foreach my $volid (sort keys %$volid_hash) {
5095         next if $volid =~ m/vm-$vmid-state-/;
5096         next if $used->{$volid};
5097         my $path = $volid_hash->{$volid}->{path};
5098         next if !$path; # just to be sure
5099         next if $usedpath->{$path};
5100         $changes = 1;
5101         add_unused_volume($conf, $volid);
5102         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5103     }
5104
5105     return $changes;
5106 }
5107
5108 sub rescan {
5109     my ($vmid, $nolock) = @_;
5110
5111     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5112
5113     my $volid_hash = scan_volids($cfg, $vmid);
5114
5115     my $updatefn =  sub {
5116         my ($vmid) = @_;
5117
5118         my $conf = load_config($vmid);
5119
5120         check_lock($conf);
5121
5122         my $vm_volids = {};
5123         foreach my $volid (keys %$volid_hash) {
5124             my $info = $volid_hash->{$volid};
5125             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5126         }
5127
5128         my $changes = update_disksize($vmid, $conf, $vm_volids);
5129
5130         update_config_nolock($vmid, $conf, 1) if $changes;
5131     };
5132
5133     if (defined($vmid)) {
5134         if ($nolock) {
5135             &$updatefn($vmid);
5136         } else {
5137             lock_config($vmid, $updatefn, $vmid);
5138         }
5139     } else {
5140         my $vmlist = config_list();
5141         foreach my $vmid (keys %$vmlist) {
5142             if ($nolock) {
5143                 &$updatefn($vmid);
5144             } else {
5145                 lock_config($vmid, $updatefn, $vmid);
5146             }
5147         }
5148     }
5149 }
5150
5151 sub restore_vma_archive {
5152     my ($archive, $vmid, $user, $opts, $comp) = @_;
5153
5154     my $input = $archive eq '-' ? "<&STDIN" : undef;
5155     my $readfrom = $archive;
5156
5157     my $uncomp = '';
5158     if ($comp) {
5159         $readfrom = '-';
5160         my $qarchive = PVE::Tools::shellquote($archive);
5161         if ($comp eq 'gzip') {
5162             $uncomp = "zcat $qarchive|";
5163         } elsif ($comp eq 'lzop') {
5164             $uncomp = "lzop -d -c $qarchive|";
5165         } else {
5166             die "unknown compression method '$comp'\n";
5167         }
5168
5169     }
5170
5171     my $tmpdir = "/var/tmp/vzdumptmp$$";
5172     rmtree $tmpdir;
5173
5174     # disable interrupts (always do cleanups)
5175     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5176         warn "got interrupt - ignored\n";
5177     };
5178
5179     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5180     POSIX::mkfifo($mapfifo, 0600);
5181     my $fifofh;
5182
5183     my $openfifo = sub {
5184         open($fifofh, '>', $mapfifo) || die $!;
5185     };
5186
5187     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5188
5189     my $oldtimeout;
5190     my $timeout = 5;
5191
5192     my $devinfo = {};
5193
5194     my $rpcenv = PVE::RPCEnvironment::get();
5195
5196     my $conffile = config_file($vmid);
5197     my $tmpfn = "$conffile.$$.tmp";
5198
5199     # Note: $oldconf is undef if VM does not exists
5200     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5201
5202     my $print_devmap = sub {
5203         my $virtdev_hash = {};
5204
5205         my $cfgfn = "$tmpdir/qemu-server.conf";
5206
5207         # we can read the config - that is already extracted
5208         my $fh = IO::File->new($cfgfn, "r") ||
5209             "unable to read qemu-server.conf - $!\n";
5210
5211         while (defined(my $line = <$fh>)) {
5212             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5213                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5214                 die "archive does not contain data for drive '$virtdev'\n"
5215                     if !$devinfo->{$devname};
5216                 if (defined($opts->{storage})) {
5217                     $storeid = $opts->{storage} || 'local';
5218                 } elsif (!$storeid) {
5219                     $storeid = 'local';
5220                 }
5221                 $format = 'raw' if !$format;
5222                 $devinfo->{$devname}->{devname} = $devname;
5223                 $devinfo->{$devname}->{virtdev} = $virtdev;
5224                 $devinfo->{$devname}->{format} = $format;
5225                 $devinfo->{$devname}->{storeid} = $storeid;
5226
5227                 # check permission on storage
5228                 my $pool = $opts->{pool}; # todo: do we need that?
5229                 if ($user ne 'root@pam') {
5230                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5231                 }
5232
5233                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5234             }
5235         }
5236
5237         foreach my $devname (keys %$devinfo) {
5238             die "found no device mapping information for device '$devname'\n"
5239                 if !$devinfo->{$devname}->{virtdev};
5240         }
5241
5242         my $cfg = cfs_read_file('storage.cfg');
5243
5244         # create empty/temp config
5245         if ($oldconf) {
5246             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5247             foreach_drive($oldconf, sub {
5248                 my ($ds, $drive) = @_;
5249
5250                 return if drive_is_cdrom($drive);
5251
5252                 my $volid = $drive->{file};
5253
5254                 return if !$volid || $volid =~ m|^/|;
5255
5256                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5257                 return if !$path || !$owner || ($owner != $vmid);
5258
5259                 # Note: only delete disk we want to restore
5260                 # other volumes will become unused
5261                 if ($virtdev_hash->{$ds}) {
5262                     PVE::Storage::vdisk_free($cfg, $volid);
5263                 }
5264             });
5265         }
5266
5267         my $map = {};
5268         foreach my $virtdev (sort keys %$virtdev_hash) {
5269             my $d = $virtdev_hash->{$virtdev};
5270             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5271             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5272
5273             # test if requested format is supported
5274             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5275             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5276             $d->{format} = $defFormat if !$supported;
5277
5278             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5279                                                   $d->{format}, undef, $alloc_size);
5280             print STDERR "new volume ID is '$volid'\n";
5281             $d->{volid} = $volid;
5282             my $path = PVE::Storage::path($cfg, $volid);
5283
5284             my $write_zeros = 1;
5285             # fixme: what other storages types initialize volumes with zero?
5286             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5287                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5288                 $write_zeros = 0;
5289             }
5290
5291             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5292
5293             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5294             $map->{$virtdev} = $volid;
5295         }
5296
5297         $fh->seek(0, 0) || die "seek failed - $!\n";
5298
5299         my $outfd = new IO::File ($tmpfn, "w") ||
5300             die "unable to write config for VM $vmid\n";
5301
5302         my $cookie = { netcount => 0 };
5303         while (defined(my $line = <$fh>)) {
5304             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5305         }
5306
5307         $fh->close();
5308         $outfd->close();
5309     };
5310
5311     eval {
5312         # enable interrupts
5313         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5314             die "interrupted by signal\n";
5315         };
5316         local $SIG{ALRM} = sub { die "got timeout\n"; };
5317
5318         $oldtimeout = alarm($timeout);
5319
5320         my $parser = sub {
5321             my $line = shift;
5322
5323             print "$line\n";
5324
5325             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5326                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5327                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5328             } elsif ($line =~ m/^CTIME: /) {
5329                 # we correctly received the vma config, so we can disable
5330                 # the timeout now for disk allocation (set to 10 minutes, so
5331                 # that we always timeout if something goes wrong)
5332                 alarm(600);
5333                 &$print_devmap();
5334                 print $fifofh "done\n";
5335                 my $tmp = $oldtimeout || 0;
5336                 $oldtimeout = undef;
5337                 alarm($tmp);
5338                 close($fifofh);
5339             }
5340         };
5341
5342         print "restore vma archive: $cmd\n";
5343         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5344     };
5345     my $err = $@;
5346
5347     alarm($oldtimeout) if $oldtimeout;
5348
5349     unlink $mapfifo;
5350
5351     if ($err) {
5352         rmtree $tmpdir;
5353         unlink $tmpfn;
5354
5355         my $cfg = cfs_read_file('storage.cfg');
5356         foreach my $devname (keys %$devinfo) {
5357             my $volid = $devinfo->{$devname}->{volid};
5358             next if !$volid;
5359             eval {
5360                 if ($volid =~ m|^/|) {
5361                     unlink $volid || die 'unlink failed\n';
5362                 } else {
5363                     PVE::Storage::vdisk_free($cfg, $volid);
5364                 }
5365                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5366             };
5367             print STDERR "unable to cleanup '$volid' - $@" if $@;
5368         }
5369         die $err;
5370     }
5371
5372     rmtree $tmpdir;
5373
5374     rename($tmpfn, $conffile) ||
5375         die "unable to commit configuration file '$conffile'\n";
5376
5377     PVE::Cluster::cfs_update(); # make sure we read new file
5378
5379     eval { rescan($vmid, 1); };
5380     warn $@ if $@;
5381 }
5382
5383 sub restore_tar_archive {
5384     my ($archive, $vmid, $user, $opts) = @_;
5385
5386     if ($archive ne '-') {
5387         my $firstfile = tar_archive_read_firstfile($archive);
5388         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5389             if $firstfile ne 'qemu-server.conf';
5390     }
5391
5392     my $storecfg = cfs_read_file('storage.cfg');
5393
5394     # destroy existing data - keep empty config
5395     my $vmcfgfn = config_file($vmid);
5396     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5397
5398     my $tocmd = "/usr/lib/qemu-server/qmextract";
5399
5400     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5401     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5402     $tocmd .= ' --prealloc' if $opts->{prealloc};
5403     $tocmd .= ' --info' if $opts->{info};
5404
5405     # tar option "xf" does not autodetect compression when read from STDIN,
5406     # so we pipe to zcat
5407     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5408         PVE::Tools::shellquote("--to-command=$tocmd");
5409
5410     my $tmpdir = "/var/tmp/vzdumptmp$$";
5411     mkpath $tmpdir;
5412
5413     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5414     local $ENV{VZDUMP_VMID} = $vmid;
5415     local $ENV{VZDUMP_USER} = $user;
5416
5417     my $conffile = config_file($vmid);
5418     my $tmpfn = "$conffile.$$.tmp";
5419
5420     # disable interrupts (always do cleanups)
5421     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5422         print STDERR "got interrupt - ignored\n";
5423     };
5424
5425     eval {
5426         # enable interrupts
5427         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5428             die "interrupted by signal\n";
5429         };
5430
5431         if ($archive eq '-') {
5432             print "extracting archive from STDIN\n";
5433             run_command($cmd, input => "<&STDIN");
5434         } else {
5435             print "extracting archive '$archive'\n";
5436             run_command($cmd);
5437         }
5438
5439         return if $opts->{info};
5440
5441         # read new mapping
5442         my $map = {};
5443         my $statfile = "$tmpdir/qmrestore.stat";
5444         if (my $fd = IO::File->new($statfile, "r")) {
5445             while (defined (my $line = <$fd>)) {
5446                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5447                     $map->{$1} = $2 if $1;
5448                 } else {
5449                     print STDERR "unable to parse line in statfile - $line\n";
5450                 }
5451             }
5452             $fd->close();
5453         }
5454
5455         my $confsrc = "$tmpdir/qemu-server.conf";
5456
5457         my $srcfd = new IO::File($confsrc, "r") ||
5458             die "unable to open file '$confsrc'\n";
5459
5460         my $outfd = new IO::File ($tmpfn, "w") ||
5461             die "unable to write config for VM $vmid\n";
5462
5463         my $cookie = { netcount => 0 };
5464         while (defined (my $line = <$srcfd>)) {
5465             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5466         }
5467
5468         $srcfd->close();
5469         $outfd->close();
5470     };
5471     my $err = $@;
5472
5473     if ($err) {
5474
5475         unlink $tmpfn;
5476
5477         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5478
5479         die $err;
5480     }
5481
5482     rmtree $tmpdir;
5483
5484     rename $tmpfn, $conffile ||
5485         die "unable to commit configuration file '$conffile'\n";
5486
5487     PVE::Cluster::cfs_update(); # make sure we read new file
5488
5489     eval { rescan($vmid, 1); };
5490     warn $@ if $@;
5491 };
5492
5493
5494 # Internal snapshots
5495
5496 # NOTE: Snapshot create/delete involves several non-atomic
5497 # action, and can take a long time.
5498 # So we try to avoid locking the file and use 'lock' variable
5499 # inside the config file instead.
5500
5501 my $snapshot_copy_config = sub {
5502     my ($source, $dest) = @_;
5503
5504     foreach my $k (keys %$source) {
5505         next if $k eq 'snapshots';
5506         next if $k eq 'snapstate';
5507         next if $k eq 'snaptime';
5508         next if $k eq 'vmstate';
5509         next if $k eq 'lock';
5510         next if $k eq 'digest';
5511         next if $k eq 'description';
5512         next if $k =~ m/^unused\d+$/;
5513
5514         $dest->{$k} = $source->{$k};
5515     }
5516 };
5517
5518 my $snapshot_apply_config = sub {
5519     my ($conf, $snap) = @_;
5520
5521     # copy snapshot list
5522     my $newconf = {
5523         snapshots => $conf->{snapshots},
5524     };
5525
5526     # keep description and list of unused disks
5527     foreach my $k (keys %$conf) {
5528         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5529         $newconf->{$k} = $conf->{$k};
5530     }
5531
5532     &$snapshot_copy_config($snap, $newconf);
5533
5534     return $newconf;
5535 };
5536
5537 sub foreach_writable_storage {
5538     my ($conf, $func) = @_;
5539
5540     my $sidhash = {};
5541
5542     foreach my $ds (keys %$conf) {
5543         next if !valid_drivename($ds);
5544
5545         my $drive = parse_drive($ds, $conf->{$ds});
5546         next if !$drive;
5547         next if drive_is_cdrom($drive);
5548
5549         my $volid = $drive->{file};
5550
5551         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5552         $sidhash->{$sid} = $sid if $sid;
5553     }
5554
5555     foreach my $sid (sort keys %$sidhash) {
5556         &$func($sid);
5557     }
5558 }
5559
5560 my $alloc_vmstate_volid = sub {
5561     my ($storecfg, $vmid, $conf, $snapname) = @_;
5562
5563     # Note: we try to be smart when selecting a $target storage
5564
5565     my $target;
5566
5567     # search shared storage first
5568     foreach_writable_storage($conf, sub {
5569         my ($sid) = @_;
5570         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5571         return if !$scfg->{shared};
5572
5573         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5574     });
5575
5576     if (!$target) {
5577         # now search local storage
5578         foreach_writable_storage($conf, sub {
5579             my ($sid) = @_;
5580             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5581             return if $scfg->{shared};
5582
5583             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5584         });
5585     }
5586
5587     $target = 'local' if !$target;
5588
5589     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5590     # we abort live save after $conf->{memory}, so we need at max twice that space
5591     my $size = $conf->{memory}*2 + $driver_state_size;
5592
5593     my $name = "vm-$vmid-state-$snapname";
5594     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5595     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5596     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5597
5598     return $volid;
5599 };
5600
5601 my $snapshot_prepare = sub {
5602     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5603
5604     my $snap;
5605
5606     my $updatefn =  sub {
5607
5608         my $conf = load_config($vmid);
5609
5610         die "you can't take a snapshot if it's a template\n"
5611             if is_template($conf);
5612
5613         check_lock($conf);
5614
5615         $conf->{lock} = 'snapshot';
5616
5617         die "snapshot name '$snapname' already used\n"
5618             if defined($conf->{snapshots}->{$snapname});
5619
5620         my $storecfg = PVE::Storage::config();
5621         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5622
5623         $snap = $conf->{snapshots}->{$snapname} = {};
5624
5625         if ($save_vmstate && check_running($vmid)) {
5626             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5627         }
5628
5629         &$snapshot_copy_config($conf, $snap);
5630
5631         $snap->{snapstate} = "prepare";
5632         $snap->{snaptime} = time();
5633         $snap->{description} = $comment if $comment;
5634
5635         # always overwrite machine if we save vmstate. This makes sure we
5636         # can restore it later using correct machine type
5637         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5638
5639         update_config_nolock($vmid, $conf, 1);
5640     };
5641
5642     lock_config($vmid, $updatefn);
5643
5644     return $snap;
5645 };
5646
5647 my $snapshot_commit = sub {
5648     my ($vmid, $snapname) = @_;
5649
5650     my $updatefn = sub {
5651
5652         my $conf = load_config($vmid);
5653
5654         die "missing snapshot lock\n"
5655             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5656
5657         my $has_machine_config = defined($conf->{machine});
5658
5659         my $snap = $conf->{snapshots}->{$snapname};
5660
5661         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5662
5663         die "wrong snapshot state\n"
5664             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5665
5666         delete $snap->{snapstate};
5667         delete $conf->{lock};
5668
5669         my $newconf = &$snapshot_apply_config($conf, $snap);
5670
5671         delete $newconf->{machine} if !$has_machine_config;
5672
5673         $newconf->{parent} = $snapname;
5674
5675         update_config_nolock($vmid, $newconf, 1);
5676     };
5677
5678     lock_config($vmid, $updatefn);
5679 };
5680
5681 sub snapshot_rollback {
5682     my ($vmid, $snapname) = @_;
5683
5684     my $prepare = 1;
5685
5686     my $storecfg = PVE::Storage::config();
5687
5688     my $conf = load_config($vmid);
5689
5690     my $get_snapshot_config = sub {
5691
5692         die "you can't rollback if vm is a template\n" if is_template($conf);
5693
5694         my $res = $conf->{snapshots}->{$snapname};
5695
5696         die "snapshot '$snapname' does not exist\n" if !defined($res);
5697
5698         return $res;
5699     };
5700
5701     my $snap = &$get_snapshot_config();
5702
5703     foreach_drive($snap, sub {
5704         my ($ds, $drive) = @_;
5705
5706         return if drive_is_cdrom($drive);
5707
5708         my $volid = $drive->{file};
5709
5710         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5711     });
5712
5713     my $updatefn = sub {
5714
5715         $conf = load_config($vmid);
5716
5717         $snap = &$get_snapshot_config();
5718
5719         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5720             if $snap->{snapstate};
5721
5722         if ($prepare) {
5723             check_lock($conf);
5724             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5725         }
5726
5727         die "unable to rollback vm $vmid: vm is running\n"
5728             if check_running($vmid);
5729
5730         if ($prepare) {
5731             $conf->{lock} = 'rollback';
5732         } else {
5733             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5734             delete $conf->{lock};
5735         }
5736
5737         my $forcemachine;
5738
5739         if (!$prepare) {
5740             my $has_machine_config = defined($conf->{machine});
5741
5742             # copy snapshot config to current config
5743             $conf = &$snapshot_apply_config($conf, $snap);
5744             $conf->{parent} = $snapname;
5745
5746             # Note: old code did not store 'machine', so we try to be smart
5747             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5748             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5749             # we remove the 'machine' configuration if not explicitly specified
5750             # in the original config.
5751             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5752         }
5753
5754         update_config_nolock($vmid, $conf, 1);
5755
5756         if (!$prepare && $snap->{vmstate}) {
5757             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5758             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5759         }
5760     };
5761
5762     lock_config($vmid, $updatefn);
5763
5764     foreach_drive($snap, sub {
5765         my ($ds, $drive) = @_;
5766
5767         return if drive_is_cdrom($drive);
5768
5769         my $volid = $drive->{file};
5770         my $device = "drive-$ds";
5771
5772         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5773     });
5774
5775     $prepare = 0;
5776     lock_config($vmid, $updatefn);
5777 }
5778
5779 my $savevm_wait = sub {
5780     my ($vmid) = @_;
5781
5782     for(;;) {
5783         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5784         if (!$stat->{status}) {
5785             die "savevm not active\n";
5786         } elsif ($stat->{status} eq 'active') {
5787             sleep(1);
5788             next;
5789         } elsif ($stat->{status} eq 'completed') {
5790             last;
5791         } else {
5792             die "query-savevm returned status '$stat->{status}'\n";
5793         }
5794     }
5795 };
5796
5797 sub snapshot_create {
5798     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5799
5800     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5801
5802     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5803
5804     my $config = load_config($vmid);
5805
5806     my $running = check_running($vmid);
5807
5808     my $freezefs = $running && $config->{agent};
5809     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5810
5811     my $drivehash = {};
5812
5813     if ($freezefs) {
5814         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5815         warn "guest-fsfreeze-freeze problems - $@" if $@;
5816     }
5817
5818     eval {
5819         # create internal snapshots of all drives
5820
5821         my $storecfg = PVE::Storage::config();
5822
5823         if ($running) {
5824             if ($snap->{vmstate}) {
5825                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5826                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5827                 &$savevm_wait($vmid);
5828             } else {
5829                 vm_mon_cmd($vmid, "savevm-start");
5830             }
5831         };
5832
5833         foreach_drive($snap, sub {
5834             my ($ds, $drive) = @_;
5835
5836             return if drive_is_cdrom($drive);
5837
5838             my $volid = $drive->{file};
5839             my $device = "drive-$ds";
5840
5841             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5842             $drivehash->{$ds} = 1;
5843        });
5844     };
5845     my $err = $@;
5846
5847     if ($running) {
5848         eval { vm_mon_cmd($vmid, "savevm-end")  };
5849         warn $@ if $@;
5850
5851         if ($freezefs) {
5852             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5853             warn "guest-fsfreeze-thaw problems - $@" if $@;
5854         }
5855
5856         # savevm-end is async, we need to wait
5857         for (;;) {
5858             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5859             if (!$stat->{bytes}) {
5860                 last;
5861             } else {
5862                 print "savevm not yet finished\n";
5863                 sleep(1);
5864                 next;
5865             }
5866         }
5867     }
5868
5869     if ($err) {
5870         warn "snapshot create failed: starting cleanup\n";
5871         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5872         warn $@ if $@;
5873         die $err;
5874     }
5875
5876     &$snapshot_commit($vmid, $snapname);
5877 }
5878
5879 # Note: $drivehash is only set when called from snapshot_create.
5880 sub snapshot_delete {
5881     my ($vmid, $snapname, $force, $drivehash) = @_;
5882
5883     my $prepare = 1;
5884
5885     my $snap;
5886     my $unused = [];
5887
5888     my $unlink_parent = sub {
5889         my ($confref, $new_parent) = @_;
5890
5891         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5892             if ($new_parent) {
5893                 $confref->{parent} = $new_parent;
5894             } else {
5895                 delete $confref->{parent};
5896             }
5897         }
5898     };
5899
5900     my $updatefn =  sub {
5901         my ($remove_drive) = @_;
5902
5903         my $conf = load_config($vmid);
5904
5905         if (!$drivehash) {
5906             check_lock($conf);
5907             die "you can't delete a snapshot if vm is a template\n"
5908                 if is_template($conf);
5909         }
5910
5911         $snap = $conf->{snapshots}->{$snapname};
5912
5913         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5914
5915         # remove parent refs
5916         if (!$prepare) {
5917             &$unlink_parent($conf, $snap->{parent});
5918             foreach my $sn (keys %{$conf->{snapshots}}) {
5919                 next if $sn eq $snapname;
5920                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5921             }
5922         }
5923
5924         if ($remove_drive) {
5925             if ($remove_drive eq 'vmstate') {
5926                 delete $snap->{$remove_drive};
5927             } else {
5928                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5929                 my $volid = $drive->{file};
5930                 delete $snap->{$remove_drive};
5931                 add_unused_volume($conf, $volid);
5932             }
5933         }
5934
5935         if ($prepare) {
5936             $snap->{snapstate} = 'delete';
5937         } else {
5938             delete $conf->{snapshots}->{$snapname};
5939             delete $conf->{lock} if $drivehash;
5940             foreach my $volid (@$unused) {
5941                 add_unused_volume($conf, $volid);
5942             }
5943         }
5944
5945         update_config_nolock($vmid, $conf, 1);
5946     };
5947
5948     lock_config($vmid, $updatefn);
5949
5950     # now remove vmstate file
5951
5952     my $storecfg = PVE::Storage::config();
5953
5954     if ($snap->{vmstate}) {
5955         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5956         if (my $err = $@) {
5957             die $err if !$force;
5958             warn $err;
5959         }
5960         # save changes (remove vmstate from snapshot)
5961         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5962     };
5963
5964     # now remove all internal snapshots
5965     foreach_drive($snap, sub {
5966         my ($ds, $drive) = @_;
5967
5968         return if drive_is_cdrom($drive);
5969
5970         my $volid = $drive->{file};
5971         my $device = "drive-$ds";
5972
5973         if (!$drivehash || $drivehash->{$ds}) {
5974             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5975             if (my $err = $@) {
5976                 die $err if !$force;
5977                 warn $err;
5978             }
5979         }
5980
5981         # save changes (remove drive fron snapshot)
5982         lock_config($vmid, $updatefn, $ds) if !$force;
5983         push @$unused, $volid;
5984     });
5985
5986     # now cleanup config
5987     $prepare = 0;
5988     lock_config($vmid, $updatefn);
5989 }
5990
5991 sub has_feature {
5992     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5993
5994     my $err;
5995     foreach_drive($conf, sub {
5996         my ($ds, $drive) = @_;
5997
5998         return if drive_is_cdrom($drive);
5999         my $volid = $drive->{file};
6000         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6001     });
6002
6003     return $err ? 0 : 1;
6004 }
6005
6006 sub template_create {
6007     my ($vmid, $conf, $disk) = @_;
6008
6009     my $storecfg = PVE::Storage::config();
6010
6011     foreach_drive($conf, sub {
6012         my ($ds, $drive) = @_;
6013
6014         return if drive_is_cdrom($drive);
6015         return if $disk && $ds ne $disk;
6016
6017         my $volid = $drive->{file};
6018         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6019
6020         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6021         $drive->{file} = $voliddst;
6022         $conf->{$ds} = print_drive($vmid, $drive);
6023         update_config_nolock($vmid, $conf, 1);
6024     });
6025 }
6026
6027 sub is_template {
6028     my ($conf) = @_;
6029
6030     return 1 if defined $conf->{template} && $conf->{template} == 1;
6031 }
6032
6033 sub qemu_img_convert {
6034     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6035
6036     my $storecfg = PVE::Storage::config();
6037     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6038     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6039
6040     if ($src_storeid && $dst_storeid) {
6041         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6042         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6043
6044         my $src_format = qemu_img_format($src_scfg, $src_volname);
6045         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6046
6047         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6048         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6049
6050         my $cmd = [];
6051         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6052         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6053         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6054
6055         my $parser = sub {
6056             my $line = shift;
6057             if($line =~ m/\((\S+)\/100\%\)/){
6058                 my $percent = $1;
6059                 my $transferred = int($size * $percent / 100);
6060                 my $remaining = $size - $transferred;
6061
6062                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6063             }
6064
6065         };
6066
6067         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6068         my $err = $@;
6069         die "copy failed: $err" if $err;
6070     }
6071 }
6072
6073 sub qemu_img_format {
6074     my ($scfg, $volname) = @_;
6075
6076     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6077         return $1;
6078     } elsif ($scfg->{type} eq 'iscsi') {
6079         return "host_device";
6080     } else {
6081         return "raw";
6082     }
6083 }
6084
6085 sub qemu_drive_mirror {
6086     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6087
6088     my $count = 0;
6089     my $old_len = 0;
6090     my $frozen = undef;
6091     my $maxwait = 120;
6092
6093     my $storecfg = PVE::Storage::config();
6094     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6095
6096     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6097
6098     my $format;
6099     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6100         $format = $1;
6101     }
6102
6103     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6104
6105     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6106     $opts->{format} = $format if $format;
6107
6108     #fixme : sometime drive-mirror timeout, but works fine after.
6109     # (I have see the problem with big volume > 200GB), so we need to eval
6110     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6111     # ignore errors here
6112
6113     eval {
6114         while (1) {
6115             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6116             my $stat = @$stats[0];
6117             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6118             die "error job is not mirroring" if $stat->{type} ne "mirror";
6119
6120             my $busy = $stat->{busy};
6121
6122             if (my $total = $stat->{len}) {
6123                 my $transferred = $stat->{offset} || 0;
6124                 my $remaining = $total - $transferred;
6125                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6126
6127                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy\n";
6128             }
6129
6130             if ($stat->{len} == $stat->{offset}) {
6131                 if ($busy eq 'false') {
6132
6133                     last if $vmiddst != $vmid;
6134
6135                     # try to switch the disk if source and destination are on the same guest
6136                     eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6137                     last if !$@;
6138                     die $@ if $@ !~ m/cannot be completed/;
6139                 }
6140
6141                 if ($count > $maxwait) {
6142                     # if too much writes to disk occurs at the end of migration
6143                     #the disk needs to be freezed to be able to complete the migration
6144                     vm_suspend($vmid,1);
6145                     $frozen = 1;
6146                 }
6147                 $count ++
6148             }
6149             $old_len = $stat->{offset};
6150             sleep 1;
6151         }
6152
6153         vm_resume($vmid, 1) if $frozen;
6154
6155     };
6156     my $err = $@;
6157
6158     my $cancel_job = sub {
6159         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6160         while (1) {
6161             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6162             my $stat = @$stats[0];
6163             last if !$stat;
6164             sleep 1;
6165         }
6166     };
6167
6168     if ($err) {
6169         eval { &$cancel_job(); };
6170         die "mirroring error: $err";
6171     }
6172
6173     if ($vmiddst != $vmid) {
6174         # if we clone a disk for a new target vm, we don't switch the disk
6175         &$cancel_job(); # so we call block-job-cancel
6176     }
6177 }
6178
6179 sub clone_disk {
6180     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6181         $newvmid, $storage, $format, $full, $newvollist) = @_;
6182
6183     my $newvolid;
6184
6185     if (!$full) {
6186         print "create linked clone of drive $drivename ($drive->{file})\n";
6187         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6188         push @$newvollist, $newvolid;
6189     } else {
6190         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6191         $storeid = $storage if $storage;
6192
6193         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6194         if (!$format) {
6195             $format = $drive->{format} || $defFormat;
6196         }
6197
6198         # test if requested format is supported - else use default
6199         my $supported = grep { $_ eq $format } @$validFormats;
6200         $format = $defFormat if !$supported;
6201
6202         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6203
6204         print "create full clone of drive $drivename ($drive->{file})\n";
6205         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6206         push @$newvollist, $newvolid;
6207
6208         if (!$running || $snapname) {
6209             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6210         } else {
6211             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6212         }
6213     }
6214
6215     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6216
6217     my $disk = $drive;
6218     $disk->{format} = undef;
6219     $disk->{file} = $newvolid;
6220     $disk->{size} = $size;
6221
6222     return $disk;
6223 }
6224
6225 # this only works if VM is running
6226 sub get_current_qemu_machine {
6227     my ($vmid) = @_;
6228
6229     my $cmd = { execute => 'query-machines', arguments => {} };
6230     my $res = vm_qmp_command($vmid, $cmd);
6231
6232     my ($current, $default);
6233     foreach my $e (@$res) {
6234         $default = $e->{name} if $e->{'is-default'};
6235         $current = $e->{name} if $e->{'is-current'};
6236     }
6237
6238     # fallback to the default machine if current is not supported by qemu
6239     return $current || $default || 'pc';
6240 }
6241
6242 sub qemu_machine_feature_enabled {
6243     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6244
6245     my $current_major;
6246     my $current_minor;
6247
6248     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6249
6250         $current_major = $3;
6251         $current_minor = $4;
6252
6253     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6254
6255         $current_major = $1;
6256         $current_minor = $2;
6257     }
6258
6259     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6260
6261
6262 }
6263
6264 sub lspci {
6265
6266     my $devices = {};
6267
6268     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6269             my (undef, $id, $function) = @_;
6270             my $res = { id => $id, function => $function};
6271             push @{$devices->{$id}}, $res;
6272     });
6273
6274     return $devices;
6275 }
6276
6277 sub vm_iothreads_list {
6278     my ($vmid) = @_;
6279
6280     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6281
6282     my $iothreads = {};
6283     foreach my $iothread (@$res) {
6284         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6285     }
6286
6287     return $iothreads;
6288 }
6289
6290 sub scsihw_infos {
6291     my ($conf, $drive) = @_;
6292
6293     my $maxdev = 0;
6294
6295     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6296         $maxdev = 7;
6297     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6298         $maxdev = 1;
6299     } else {
6300         $maxdev = 256;
6301     }
6302
6303     my $controller = int($drive->{index} / $maxdev);
6304     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6305
6306     return ($maxdev, $controller, $controller_prefix);
6307 }
6308
6309 1;