PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     iothread => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Enable iothread dataplane.",
 159         default => 0,
 160     },
 161     onboot => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Specifies whether a VM will be started during system bootup.",
 165         default => 0,
 166     },
 167     autostart => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Automatic restart after crash (currently ignored).",
 171         default => 0,
 172     },
 173     hotplug => {
 174         optional => 1,
 175         type => 'string', format => 'pve-hotplug-features',
 176         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 177         default => 'network,disk,usb',
 178     },
 179     reboot => {
 180         optional => 1,
 181         type => 'boolean',
 182         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 183         default => 1,
 184     },
 185     lock => {
 186         optional => 1,
 187         type => 'string',
 188         description => "Lock/unlock the VM.",
 189         enum => [qw(migrate backup snapshot rollback)],
 190     },
 191     cpulimit => {
 192         optional => 1,
 193         type => 'integer',
 194         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 195         minimum => 0,
 196         default => 0,
 197     },
 198     cpuunits => {
 199         optional => 1,
 200         type => 'integer',
 201         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 202         minimum => 0,
 203         maximum => 500000,
 204         default => 1000,
 205     },
 206     memory => {
 207         optional => 1,
 208         type => 'integer',
 209         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 210         minimum => 16,
 211         default => 512,
 212     },
 213     balloon => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 217         minimum => 0,
 218     },
 219     shares => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 223         minimum => 0,
 224         maximum => 50000,
 225         default => 1000,
 226     },
 227     keyboard => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 231         enum => PVE::Tools::kvmkeymaplist(),
 232         default => 'en-us',
 233     },
 234     name => {
 235         optional => 1,
 236         type => 'string', format => 'dns-name',
 237         description => "Set a name for the VM. Only used on the configuration web interface.",
 238     },
 239     scsihw => {
 240         optional => 1,
 241         type => 'string',
 242         description => "scsi controller model",
 243         enum => [qw(lsi lsi53c810 virtio-scsi-pci megasas pvscsi)],
 244         default => 'lsi',
 245     },
 246     description => {
 247         optional => 1,
 248         type => 'string',
 249         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 250     },
 251     ostype => {
 252         optional => 1,
 253         type => 'string',
 254         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 255         description => <<EODESC,
 256 Used to enable special optimization/features for specific
 257 operating systems:
 258
 259 other  => unspecified OS
 260 wxp    => Microsoft Windows XP
 261 w2k    => Microsoft Windows 2000
 262 w2k3   => Microsoft Windows 2003
 263 w2k8   => Microsoft Windows 2008
 264 wvista => Microsoft Windows Vista
 265 win7   => Microsoft Windows 7
 266 win8   => Microsoft Windows 8/2012
 267 l24    => Linux 2.4 Kernel
 268 l26    => Linux 2.6/3.X Kernel
 269 solaris => solaris/opensolaris/openindiania kernel
 270
 271 other|l24|l26|solaris                       ... no special behaviour
 272 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 273 EODESC
 274     },
 275     boot => {
 276         optional => 1,
 277         type => 'string',
 278         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 279         pattern => '[acdn]{1,4}',
 280         default => 'cdn',
 281     },
 282     bootdisk => {
 283         optional => 1,
 284         type => 'string', format => 'pve-qm-bootdisk',
 285         description => "Enable booting from specified disk.",
 286         pattern => '(ide|sata|scsi|virtio)\d+',
 287     },
 288     smp => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "The number of CPUs. Please use option -sockets instead.",
 292         minimum => 1,
 293         default => 1,
 294     },
 295     sockets => {
 296         optional => 1,
 297         type => 'integer',
 298         description => "The number of CPU sockets.",
 299         minimum => 1,
 300         default => 1,
 301     },
 302     cores => {
 303         optional => 1,
 304         type => 'integer',
 305         description => "The number of cores per socket.",
 306         minimum => 1,
 307         default => 1,
 308     },
 309     numa => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Enable/disable Numa.",
 313         default => 0,
 314     },
 315     vcpus => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "Number of hotplugged vcpus.",
 319         minimum => 1,
 320         default => 0,
 321     },
 322     acpi => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable ACPI.",
 326         default => 1,
 327     },
 328     agent => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable Qemu GuestAgent.",
 332         default => 0,
 333     },
 334     kvm => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Enable/disable KVM hardware virtualization.",
 338         default => 1,
 339     },
 340     tdf => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Enable/disable time drift fix.",
 344         default => 0,
 345     },
 346     localtime => {
 347         optional => 1,
 348         type => 'boolean',
 349         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 350     },
 351     freeze => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 355     },
 356     vga => {
 357         optional => 1,
 358         type => 'string',
 359         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 360         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 361     },
 362     watchdog => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-watchdog',
 365         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 366         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 367     },
 368     startdate => {
 369         optional => 1,
 370         type => 'string',
 371         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 372         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 373         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 374         default => 'now',
 375     },
 376     startup => {
 377         optional => 1,
 378         type => 'string', format => 'pve-qm-startup',
 379         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 380         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 381     },
 382     template => {
 383         optional => 1,
 384         type => 'boolean',
 385         description => "Enable/disable Template.",
 386         default => 0,
 387     },
 388     args => {
 389         optional => 1,
 390         type => 'string',
 391         description => <<EODESCR,
 392 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 393
 394 args: -no-reboot -no-hpet
 395 EODESCR
 396     },
 397     tablet => {
 398         optional => 1,
 399         type => 'boolean',
 400         default => 1,
 401         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 402     },
 403     migrate_speed => {
 404         optional => 1,
 405         type => 'integer',
 406         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 407         minimum => 0,
 408         default => 0,
 409     },
 410     migrate_downtime => {
 411         optional => 1,
 412         type => 'number',
 413         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 414         minimum => 0,
 415         default => 0.1,
 416     },
 417     cdrom => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-drive',
 420         typetext => 'volume',
 421         description => "This is an alias for option -ide2",
 422     },
 423     cpu => {
 424         optional => 1,
 425         description => "Emulated CPU type.",
 426         type => 'string',
 427         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 428         default => 'kvm64',
 429     },
 430     parent => get_standard_option('pve-snapshot-name', {
 431         optional => 1,
 432         description => "Parent snapshot name. This is used internally, and should not be modified.",
 433     }),
 434     snaptime => {
 435         optional => 1,
 436         description => "Timestamp for snapshots.",
 437         type => 'integer',
 438         minimum => 0,
 439     },
 440     vmstate => {
 441         optional => 1,
 442         type => 'string', format => 'pve-volume-id',
 443         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 444     },
 445     machine => {
 446         description => "Specific the Qemu machine type.",
 447         type => 'string',
 448         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 449         maxLength => 40,
 450         optional => 1,
 451     },
 452     smbios1 => {
 453         description => "Specify SMBIOS type 1 fields.",
 454         type => 'string', format => 'pve-qm-smbios1',
 455         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 456         maxLength => 256,
 457         optional => 1,
 458     },
 459 };
 460
 461 # what about other qemu settings ?
 462 #cpu => 'string',
 463 #machine => 'string',
 464 #fda => 'file',
 465 #fdb => 'file',
 466 #mtdblock => 'file',
 467 #sd => 'file',
 468 #pflash => 'file',
 469 #snapshot => 'bool',
 470 #bootp => 'file',
 471 ##tftp => 'dir',
 472 ##smb => 'dir',
 473 #kernel => 'file',
 474 #append => 'string',
 475 #initrd => 'file',
 476 ##soundhw => 'string',
 477
 478 while (my ($k, $v) = each %$confdesc) {
 479     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 480 }
 481
 482 my $MAX_IDE_DISKS = 4;
 483 my $MAX_SCSI_DISKS = 14;
 484 my $MAX_VIRTIO_DISKS = 16;
 485 my $MAX_SATA_DISKS = 6;
 486 my $MAX_USB_DEVICES = 5;
 487 my $MAX_NETS = 32;
 488 my $MAX_UNUSED_DISKS = 8;
 489 my $MAX_HOSTPCI_DEVICES = 4;
 490 my $MAX_SERIAL_PORTS = 4;
 491 my $MAX_PARALLEL_PORTS = 3;
 492 my $MAX_NUMA = 8;
 493 my $MAX_MEM = 4194304;
 494 my $STATICMEM = 1024;
 495
 496 my $numadesc = {
 497     optional => 1,
 498     type => 'string', format => 'pve-qm-numanode',
 499     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 500     description => "numa topology",
 501 };
 502 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 503
 504 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 505     $confdesc->{"numa$i"} = $numadesc;
 506 }
 507
 508 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 509                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 510                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 511 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 512
 513 my $netdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-net',
 516     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 517     description => <<EODESCR,
 518 Specify network devices.
 519
 520 MODEL is one of: $nic_model_list_txt
 521
 522 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 523 automatically generated if not specified.
 524
 525 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 526
 527 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 528
 529 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 530
 531 10.0.2.2   Gateway
 532 10.0.2.3   DNS Server
 533 10.0.2.4   SMB Server
 534
 535 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 536
 537 EODESCR
 538 };
 539 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 540
 541 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 542     $confdesc->{"net$i"} = $netdesc;
 543 }
 544
 545 my $drivename_hash;
 546
 547 my $idedesc = {
 548     optional => 1,
 549     type => 'string', format => 'pve-qm-drive',
 550     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 551     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 552 };
 553 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 554
 555 my $scsidesc = {
 556     optional => 1,
 557     type => 'string', format => 'pve-qm-drive',
 558     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 559     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 560 };
 561 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 562
 563 my $satadesc = {
 564     optional => 1,
 565     type => 'string', format => 'pve-qm-drive',
 566     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 567     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 568 };
 569 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 570
 571 my $virtiodesc = {
 572     optional => 1,
 573     type => 'string', format => 'pve-qm-drive',
 574     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 575     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 576 };
 577 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 578
 579 my $usbdesc = {
 580     optional => 1,
 581     type => 'string', format => 'pve-qm-usb-device',
 582     typetext => 'host=HOSTUSBDEVICE|spice',
 583     description => <<EODESCR,
 584 Configure an USB device (n is 0 to 4). This can be used to
 585 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 586
 587 'bus-port(.port)*' (decimal numbers) or
 588 'vendor_id:product_id' (hexadeciaml numbers)
 589
 590 You can use the 'lsusb -t' command to list existing usb devices.
 591
 592 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 593
 594 The value 'spice' can be used to add a usb redirection devices for spice.
 595
 596 EODESCR
 597 };
 598 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 599
 600 my $hostpcidesc = {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-hostpci',
 603         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 604         description => <<EODESCR,
 605 Map host pci devices. HOSTPCIDEVICE syntax is:
 606
 607 'bus:dev.func' (hexadecimal numbers)
 608
 609 You can us the 'lspci' command to list existing pci devices.
 610
 611 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 612
 613 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 614
 615 Experimental: user reported problems with this option.
 616 EODESCR
 617 };
 618 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 619
 620 my $serialdesc = {
 621         optional => 1,
 622         type => 'string',
 623         pattern => '(/dev/.+|socket)',
 624         description =>  <<EODESCR,
 625 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 626
 627 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 628
 629 Experimental: user reported problems with this option.
 630 EODESCR
 631 };
 632
 633 my $paralleldesc= {
 634         optional => 1,
 635         type => 'string',
 636         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 637         description =>  <<EODESCR,
 638 Map host parallel devices (n is 0 to 2).
 639
 640 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 641
 642 Experimental: user reported problems with this option.
 643 EODESCR
 644 };
 645
 646 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 647     $confdesc->{"parallel$i"} = $paralleldesc;
 648 }
 649
 650 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 651     $confdesc->{"serial$i"} = $serialdesc;
 652 }
 653
 654 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 655     $confdesc->{"hostpci$i"} = $hostpcidesc;
 656 }
 657
 658 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 659     $drivename_hash->{"ide$i"} = 1;
 660     $confdesc->{"ide$i"} = $idedesc;
 661 }
 662
 663 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 664     $drivename_hash->{"sata$i"} = 1;
 665     $confdesc->{"sata$i"} = $satadesc;
 666 }
 667
 668 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 669     $drivename_hash->{"scsi$i"} = 1;
 670     $confdesc->{"scsi$i"} = $scsidesc ;
 671 }
 672
 673 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 674     $drivename_hash->{"virtio$i"} = 1;
 675     $confdesc->{"virtio$i"} = $virtiodesc;
 676 }
 677
 678 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 679     $confdesc->{"usb$i"} = $usbdesc;
 680 }
 681
 682 my $unuseddesc = {
 683     optional => 1,
 684     type => 'string', format => 'pve-volume-id',
 685     description => "Reference to unused volumes.",
 686 };
 687
 688 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 689     $confdesc->{"unused$i"} = $unuseddesc;
 690 }
 691
 692 my $kvm_api_version = 0;
 693
 694 sub kvm_version {
 695
 696     return $kvm_api_version if $kvm_api_version;
 697
 698     my $fh = IO::File->new("</dev/kvm") ||
 699         return 0;
 700
 701     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 702         $kvm_api_version = $v;
 703     }
 704
 705     $fh->close();
 706
 707     return  $kvm_api_version;
 708 }
 709
 710 my $kvm_user_version;
 711
 712 sub kvm_user_version {
 713
 714     return $kvm_user_version if $kvm_user_version;
 715
 716     $kvm_user_version = 'unknown';
 717
 718     my $tmp = `kvm -help 2>/dev/null`;
 719
 720     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 721         $kvm_user_version = $2;
 722     }
 723
 724     return $kvm_user_version;
 725
 726 }
 727
 728 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 729
 730 sub disknames {
 731     # order is important - used to autoselect boot disk
 732     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 733             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 734             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 735             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 736 }
 737
 738 sub valid_drivename {
 739     my $dev = shift;
 740
 741     return defined($drivename_hash->{$dev});
 742 }
 743
 744 sub option_exists {
 745     my $key = shift;
 746     return defined($confdesc->{$key});
 747 }
 748
 749 sub nic_models {
 750     return $nic_model_list;
 751 }
 752
 753 sub os_list_description {
 754
 755     return {
 756         other => 'Other',
 757         wxp => 'Windows XP',
 758         w2k => 'Windows 2000',
 759         w2k3 =>, 'Windows 2003',
 760         w2k8 => 'Windows 2008',
 761         wvista => 'Windows Vista',
 762         win7 => 'Windows 7',
 763         win8 => 'Windows 8/2012',
 764         l24 => 'Linux 2.4',
 765         l26 => 'Linux 2.6',
 766     };
 767 }
 768
 769 my $cdrom_path;
 770
 771 sub get_cdrom_path {
 772
 773     return  $cdrom_path if $cdrom_path;
 774
 775     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 776     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 777     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 778 }
 779
 780 sub get_iso_path {
 781     my ($storecfg, $vmid, $cdrom) = @_;
 782
 783     if ($cdrom eq 'cdrom') {
 784         return get_cdrom_path();
 785     } elsif ($cdrom eq 'none') {
 786         return '';
 787     } elsif ($cdrom =~ m|^/|) {
 788         return $cdrom;
 789     } else {
 790         return PVE::Storage::path($storecfg, $cdrom);
 791     }
 792 }
 793
 794 # try to convert old style file names to volume IDs
 795 sub filename_to_volume_id {
 796     my ($vmid, $file, $media) = @_;
 797
 798     if (!($file eq 'none' || $file eq 'cdrom' ||
 799           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 800
 801         return undef if $file =~ m|/|;
 802
 803         if ($media && $media eq 'cdrom') {
 804             $file = "local:iso/$file";
 805         } else {
 806             $file = "local:$vmid/$file";
 807         }
 808     }
 809
 810     return $file;
 811 }
 812
 813 sub verify_media_type {
 814     my ($opt, $vtype, $media) = @_;
 815
 816     return if !$media;
 817
 818     my $etype;
 819     if ($media eq 'disk') {
 820         $etype = 'images';
 821     } elsif ($media eq 'cdrom') {
 822         $etype = 'iso';
 823     } else {
 824         die "internal error";
 825     }
 826
 827     return if ($vtype eq $etype);
 828
 829     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 830 }
 831
 832 sub cleanup_drive_path {
 833     my ($opt, $storecfg, $drive) = @_;
 834
 835     # try to convert filesystem paths to volume IDs
 836
 837     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 838         ($drive->{file} !~ m|^/dev/.+|) &&
 839         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 840         ($drive->{file} !~ m/^\d+$/)) {
 841         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 842         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 843         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 844         verify_media_type($opt, $vtype, $drive->{media});
 845         $drive->{file} = $volid;
 846     }
 847
 848     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 849 }
 850
 851 sub create_conf_nolock {
 852     my ($vmid, $settings) = @_;
 853
 854     my $filename = config_file($vmid);
 855
 856     die "configuration file '$filename' already exists\n" if -f $filename;
 857
 858     my $defaults = load_defaults();
 859
 860     $settings->{name} = "vm$vmid" if !$settings->{name};
 861     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 862
 863     my $data = '';
 864     foreach my $opt (keys %$settings) {
 865         next if !$confdesc->{$opt};
 866
 867         my $value = $settings->{$opt};
 868         next if !$value;
 869
 870         $data .= "$opt: $value\n";
 871     }
 872
 873     PVE::Tools::file_set_contents($filename, $data);
 874 }
 875
 876 sub parse_hotplug_features {
 877     my ($data) = @_;
 878
 879     my $res = {};
 880
 881     return $res if $data eq '0';
 882
 883     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 884
 885     foreach my $feature (PVE::Tools::split_list($data)) {
 886         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 887             $res->{$1} = 1;
 888         } else {
 889             warn "ignoring unknown hotplug feature '$feature'\n";
 890         }
 891     }
 892     return $res;
 893 }
 894
 895 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 896 sub pve_verify_hotplug_features {
 897     my ($value, $noerr) = @_;
 898
 899     return $value if parse_hotplug_features($value);
 900
 901     return undef if $noerr;
 902
 903     die "unable to parse hotplug option\n";
 904 }
 905
 906 my $parse_size = sub {
 907     my ($value) = @_;
 908
 909     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 910     my ($size, $unit) = ($1, $3);
 911     if ($unit) {
 912         if ($unit eq 'K') {
 913             $size = $size * 1024;
 914         } elsif ($unit eq 'M') {
 915             $size = $size * 1024 * 1024;
 916         } elsif ($unit eq 'G') {
 917             $size = $size * 1024 * 1024 * 1024;
 918         }
 919     }
 920     return int($size);
 921 };
 922
 923 my $format_size = sub {
 924     my ($size) = @_;
 925
 926     $size = int($size);
 927
 928     my $kb = int($size/1024);
 929     return $size if $kb*1024 != $size;
 930
 931     my $mb = int($kb/1024);
 932     return "${kb}K" if $mb*1024 != $kb;
 933
 934     my $gb = int($mb/1024);
 935     return "${mb}M" if $gb*1024 != $mb;
 936
 937     return "${gb}G";
 938 };
 939
 940 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 941 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 942 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 943 #        [,aio=native|threads][,discard=ignore|on]
 944
 945 sub parse_drive {
 946     my ($key, $data) = @_;
 947
 948     my $res = {};
 949
 950     # $key may be undefined - used to verify JSON parameters
 951     if (!defined($key)) {
 952         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 953         $res->{index} = 0;
 954     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 955         $res->{interface} = $1;
 956         $res->{index} = $2;
 957     } else {
 958         return undef;
 959     }
 960
 961     foreach my $p (split (/,/, $data)) {
 962         next if $p =~ m/^\s*$/;
 963
 964         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard)=(.+)$/) {
 965             my ($k, $v) = ($1, $2);
 966
 967             $k = 'file' if $k eq 'volume';
 968
 969             return undef if defined $res->{$k};
 970
 971             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 972                 return undef if !$v || $v !~ m/^\d+/;
 973                 $k = "m$k";
 974                 $v = sprintf("%.3f", $v / (1024*1024));
 975             }
 976             $res->{$k} = $v;
 977         } else {
 978             if (!$res->{file} && $p !~ m/=/) {
 979                 $res->{file} = $p;
 980             } else {
 981                 return undef;
 982             }
 983         }
 984     }
 985
 986     return undef if !$res->{file};
 987
 988     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 989         $res->{format} = $1;
 990     }
 991
 992     return undef if $res->{cache} &&
 993         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 994     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 995     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 996     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 997     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 998     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 999     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
1000     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
1001     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
1002     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
1003     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
1004     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
1005     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
1006
1007     return undef if $res->{mbps_rd} && $res->{mbps};
1008     return undef if $res->{mbps_wr} && $res->{mbps};
1009
1010     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1011     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1012     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1013     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1014     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1015     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1016
1017     return undef if $res->{iops_rd} && $res->{iops};
1018     return undef if $res->{iops_wr} && $res->{iops};
1019
1020
1021     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1022     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1023     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1024     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1025     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1026     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1027
1028
1029     if ($res->{size}) {
1030         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1031     }
1032
1033     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1034         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1035         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1036         return undef if $res->{interface} eq 'virtio';
1037     }
1038
1039     # rerror does not work with scsi drives
1040     if ($res->{rerror}) {
1041         return undef if $res->{interface} eq 'scsi';
1042     }
1043
1044     return $res;
1045 }
1046
1047 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1048
1049 sub print_drive {
1050     my ($vmid, $drive) = @_;
1051
1052     my $opts = '';
1053     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup') {
1054         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1055     }
1056
1057     if ($drive->{size}) {
1058         $opts .= ",size=" . &$format_size($drive->{size});
1059     }
1060
1061     return "$drive->{file}$opts";
1062 }
1063
1064 sub scsi_inquiry {
1065     my($fh, $noerr) = @_;
1066
1067     my $SG_IO = 0x2285;
1068     my $SG_GET_VERSION_NUM = 0x2282;
1069
1070     my $versionbuf = "\x00" x 8;
1071     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1072     if (!$ret) {
1073         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1074         return undef;
1075     }
1076     my $version = unpack("I", $versionbuf);
1077     if ($version < 30000) {
1078         die "scsi generic interface too old\n"  if !$noerr;
1079         return undef;
1080     }
1081
1082     my $buf = "\x00" x 36;
1083     my $sensebuf = "\x00" x 8;
1084     my $cmd = pack("C x3 C x1", 0x12, 36);
1085
1086     # see /usr/include/scsi/sg.h
1087     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1088
1089     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1090                       length($sensebuf), 0, length($buf), $buf,
1091                       $cmd, $sensebuf, 6000);
1092
1093     $ret = ioctl($fh, $SG_IO, $packet);
1094     if (!$ret) {
1095         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1096         return undef;
1097     }
1098
1099     my @res = unpack($sg_io_hdr_t, $packet);
1100     if ($res[17] || $res[18]) {
1101         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1102         return undef;
1103     }
1104
1105     my $res = {};
1106     (my $byte0, my $byte1, $res->{vendor},
1107      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1108
1109     $res->{removable} = $byte1 & 128 ? 1 : 0;
1110     $res->{type} = $byte0 & 31;
1111
1112     return $res;
1113 }
1114
1115 sub path_is_scsi {
1116     my ($path) = @_;
1117
1118     my $fh = IO::File->new("+<$path") || return undef;
1119     my $res = scsi_inquiry($fh, 1);
1120     close($fh);
1121
1122     return $res;
1123 }
1124
1125 sub machine_type_is_q35 {
1126     my ($conf) = @_;
1127
1128     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1129 }
1130
1131 sub print_tabletdevice_full {
1132     my ($conf) = @_;
1133
1134     my $q35 = machine_type_is_q35($conf);
1135
1136     # we use uhci for old VMs because tablet driver was buggy in older qemu
1137     my $usbbus = $q35 ? "ehci" : "uhci";
1138
1139     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1140 }
1141
1142 sub print_drivedevice_full {
1143     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1144
1145     my $device = '';
1146     my $maxdev = 0;
1147
1148     if ($drive->{interface} eq 'virtio') {
1149         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1150         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1151         $device .= ",iothread=iothread0" if $conf->{iothread};
1152     } elsif ($drive->{interface} eq 'scsi') {
1153         $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
1154         my $controller = int($drive->{index} / $maxdev);
1155         my $unit = $drive->{index} % $maxdev;
1156         my $devicetype = 'hd';
1157         my $path = '';
1158         if (drive_is_cdrom($drive)) {
1159               $devicetype = 'cd';
1160         } else {
1161               if ($drive->{file} =~ m|^/|) {
1162                   $path = $drive->{file};
1163               } else {
1164                   $path = PVE::Storage::path($storecfg, $drive->{file});
1165               }
1166
1167               if($path =~ m/^iscsi\:\/\//){
1168                   $devicetype = 'generic';
1169               } else {
1170                   if (my $info = path_is_scsi($path)) {
1171                       if ($info->{type} == 0) {
1172                           $devicetype = 'block';
1173                       } elsif ($info->{type} == 1) { # tape
1174                           $devicetype = 'generic';
1175                       }
1176                   }
1177               }
1178          }
1179
1180         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1181             $device = "scsi-$devicetype,bus=scsihw$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1182         } else {
1183             $device = "scsi-$devicetype,bus=scsihw$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1184         }
1185
1186     } elsif ($drive->{interface} eq 'ide'){
1187         $maxdev = 2;
1188         my $controller = int($drive->{index} / $maxdev);
1189         my $unit = $drive->{index} % $maxdev;
1190         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1191
1192         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1193     } elsif ($drive->{interface} eq 'sata'){
1194         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1195         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1196         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1197     } elsif ($drive->{interface} eq 'usb') {
1198         die "implement me";
1199         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1200     } else {
1201         die "unsupported interface type";
1202     }
1203
1204     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1205
1206     return $device;
1207 }
1208
1209 sub get_initiator_name {
1210     my $initiator;
1211
1212     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1213     while (defined(my $line = <$fh>)) {
1214         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1215         $initiator = $1;
1216         last;
1217     }
1218     $fh->close();
1219
1220     return $initiator;
1221 }
1222
1223 sub print_drive_full {
1224     my ($storecfg, $vmid, $drive) = @_;
1225
1226     my $opts = '';
1227     foreach my $o (@qemu_drive_options) {
1228         next if $o eq 'bootindex';
1229         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1230     }
1231
1232     foreach my $o (qw(bps bps_rd bps_wr)) {
1233         my $v = $drive->{"m$o"};
1234         $opts .= ",$o=" . int($v*1024*1024) if $v;
1235     }
1236
1237     # use linux-aio by default (qemu default is threads)
1238     $opts .= ",aio=native" if !$drive->{aio};
1239
1240     my $path;
1241     my $volid = $drive->{file};
1242     if (drive_is_cdrom($drive)) {
1243         $path = get_iso_path($storecfg, $vmid, $volid);
1244     } else {
1245         if ($volid =~ m|^/|) {
1246             $path = $volid;
1247         } else {
1248             $path = PVE::Storage::path($storecfg, $volid);
1249         }
1250     }
1251
1252     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1253
1254     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1255     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1256
1257     my $pathinfo = $path ? "file=$path," : '';
1258
1259     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1260 }
1261
1262 sub print_netdevice_full {
1263     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1264
1265     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1266
1267     my $device = $net->{model};
1268     if ($net->{model} eq 'virtio') {
1269          $device = 'virtio-net-pci';
1270      };
1271
1272     my $pciaddr = print_pci_addr("$netid", $bridges);
1273     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1274     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1275         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1276         my $vectors = $net->{queues} * 2 + 2;
1277         $tmpstr .= ",vectors=$vectors,mq=on";
1278     }
1279     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1280     return $tmpstr;
1281 }
1282
1283 sub print_netdev_full {
1284     my ($vmid, $conf, $net, $netid) = @_;
1285
1286     my $i = '';
1287     if ($netid =~ m/^net(\d+)$/) {
1288         $i = int($1);
1289     }
1290
1291     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1292
1293     my $ifname = "tap${vmid}i$i";
1294
1295     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1296     die "interface name '$ifname' is too long (max 15 character)\n"
1297         if length($ifname) >= 16;
1298
1299     my $vhostparam = '';
1300     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1301
1302     my $vmname = $conf->{name} || "vm$vmid";
1303
1304     my $netdev = "";
1305
1306     if ($net->{bridge}) {
1307         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1308     } else {
1309         $netdev = "type=user,id=$netid,hostname=$vmname";
1310     }
1311
1312     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1313
1314     return $netdev;
1315 }
1316
1317 sub drive_is_cdrom {
1318     my ($drive) = @_;
1319
1320     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1321
1322 }
1323
1324 sub parse_numa {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     foreach my $kvp (split(/,/, $data)) {
1330
1331         if ($kvp =~ m/^memory=(\S+)$/) {
1332             $res->{memory} = $1;
1333         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1334             $res->{policy} = $1;
1335         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1336             $res->{cpus}->{start} = $1;
1337             $res->{cpus}->{end} = $3;
1338         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1339             $res->{hostnodes}->{start} = $1;
1340             $res->{hostnodes}->{end} = $3;
1341         } else {
1342             return undef;
1343         }
1344     }
1345
1346     return $res;
1347 }
1348
1349 sub parse_hostpci {
1350     my ($value) = @_;
1351
1352     return undef if !$value;
1353
1354
1355     my @list = split(/,/, $value);
1356     my $found;
1357
1358     my $res = {};
1359     foreach my $kv (@list) {
1360
1361         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1362             $found = 1;
1363             if(defined($4)){
1364                 push @{$res->{pciid}}, { id => $2 , function => $4};
1365
1366             }else{
1367                 my $pcidevices = lspci($2);
1368                 $res->{pciid} = $pcidevices->{$2};
1369             }
1370         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1371             $res->{driver} = $1;
1372         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1373             $res->{rombar} = $1;
1374         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1375             $res->{'x-vga'} = $1;
1376         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1377             $res->{pcie} = 1 if $1 == 1;
1378         } else {
1379             warn "unknown hostpci setting '$kv'\n";
1380         }
1381     }
1382
1383     return undef if !$found;
1384
1385     return $res;
1386 }
1387
1388 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1389 sub parse_net {
1390     my ($data) = @_;
1391
1392     my $res = {};
1393
1394     foreach my $kvp (split(/,/, $data)) {
1395
1396         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1397             my $model = lc($1);
1398             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1399             $res->{model} = $model;
1400             $res->{macaddr} = $mac;
1401         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1402             $res->{bridge} = $1;
1403         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1404             $res->{queues} = $1;
1405         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1406             $res->{rate} = $1;
1407         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1408             $res->{tag} = $1;
1409         } elsif ($kvp =~ m/^firewall=([01])$/) {
1410             $res->{firewall} = $1;
1411         } elsif ($kvp =~ m/^link_down=([01])$/) {
1412             $res->{link_down} = $1;
1413         } else {
1414             return undef;
1415         }
1416
1417     }
1418
1419     return undef if !$res->{model};
1420
1421     return $res;
1422 }
1423
1424 sub print_net {
1425     my $net = shift;
1426
1427     my $res = "$net->{model}";
1428     $res .= "=$net->{macaddr}" if $net->{macaddr};
1429     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1430     $res .= ",rate=$net->{rate}" if $net->{rate};
1431     $res .= ",tag=$net->{tag}" if $net->{tag};
1432     $res .= ",firewall=1" if $net->{firewall};
1433     $res .= ",link_down=1" if $net->{link_down};
1434
1435     return $res;
1436 }
1437
1438 sub add_random_macs {
1439     my ($settings) = @_;
1440
1441     foreach my $opt (keys %$settings) {
1442         next if $opt !~ m/^net(\d+)$/;
1443         my $net = parse_net($settings->{$opt});
1444         next if !$net;
1445         $settings->{$opt} = print_net($net);
1446     }
1447 }
1448
1449 sub add_unused_volume {
1450     my ($config, $volid) = @_;
1451
1452     my $key;
1453     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1454         my $test = "unused$ind";
1455         if (my $vid = $config->{$test}) {
1456             return if $vid eq $volid; # do not add duplicates
1457         } else {
1458             $key = $test;
1459         }
1460     }
1461
1462     die "To many unused volume - please delete them first.\n" if !$key;
1463
1464     $config->{$key} = $volid;
1465
1466     return $key;
1467 }
1468
1469 sub vm_is_volid_owner {
1470     my ($storecfg, $vmid, $volid) = @_;
1471
1472     if ($volid !~  m|^/|) {
1473         my ($path, $owner);
1474         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1475         if ($owner && ($owner == $vmid)) {
1476             return 1;
1477         }
1478     }
1479
1480     return undef;
1481 }
1482
1483 sub vmconfig_delete_pending_option {
1484     my ($conf, $key) = @_;
1485
1486     delete $conf->{pending}->{$key};
1487     my $pending_delete_hash = { $key => 1 };
1488     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1489         $pending_delete_hash->{$opt} = 1;
1490     }
1491     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1492 }
1493
1494 sub vmconfig_undelete_pending_option {
1495     my ($conf, $key) = @_;
1496
1497     my $pending_delete_hash = {};
1498     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1499         $pending_delete_hash->{$opt} = 1;
1500     }
1501     delete $pending_delete_hash->{$key};
1502
1503     my @keylist = keys %$pending_delete_hash;
1504     if (scalar(@keylist)) {
1505         $conf->{pending}->{delete} = join(',', @keylist);
1506     } else {
1507         delete $conf->{pending}->{delete};
1508     }
1509 }
1510
1511 sub vmconfig_register_unused_drive {
1512     my ($storecfg, $vmid, $conf, $drive) = @_;
1513
1514     if (!drive_is_cdrom($drive)) {
1515         my $volid = $drive->{file};
1516         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1517             add_unused_volume($conf, $volid, $vmid);
1518         }
1519     }
1520 }
1521
1522 sub vmconfig_cleanup_pending {
1523     my ($conf) = @_;
1524
1525     # remove pending changes when nothing changed
1526     my $changes;
1527     foreach my $opt (keys %{$conf->{pending}}) {
1528         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1529             $changes = 1;
1530             delete $conf->{pending}->{$opt};
1531         }
1532     }
1533
1534     # remove delete if option is not set
1535     my $pending_delete_hash = {};
1536     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1537         if (defined($conf->{$opt})) {
1538             $pending_delete_hash->{$opt} = 1;
1539         } else {
1540             $changes = 1;
1541         }
1542     }
1543
1544     my @keylist = keys %$pending_delete_hash;
1545     if (scalar(@keylist)) {
1546         $conf->{pending}->{delete} = join(',', @keylist);
1547     } else {
1548         delete $conf->{pending}->{delete};
1549     }
1550
1551     return $changes;
1552 }
1553
1554 my $valid_smbios1_options = {
1555     manufacturer => '\S+',
1556     product => '\S+',
1557     version => '\S+',
1558     serial => '\S+',
1559     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1560     sku => '\S+',
1561     family => '\S+',
1562 };
1563
1564 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1565 sub parse_smbios1 {
1566     my ($data) = @_;
1567
1568     my $res = {};
1569
1570     foreach my $kvp (split(/,/, $data)) {
1571         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1572         my ($k, $v) = split(/=/, $kvp);
1573         return undef if !defined($k) || !defined($v);
1574         return undef if !$valid_smbios1_options->{$k};
1575         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1576         $res->{$k} = $v;
1577     }
1578
1579     return $res;
1580 }
1581
1582 sub print_smbios1 {
1583     my ($smbios1) = @_;
1584
1585     my $data = '';
1586     foreach my $k (keys %$smbios1) {
1587         next if !defined($smbios1->{$k});
1588         next if !$valid_smbios1_options->{$k};
1589         $data .= ',' if $data;
1590         $data .= "$k=$smbios1->{$k}";
1591     }
1592     return $data;
1593 }
1594
1595 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1596 sub verify_smbios1 {
1597     my ($value, $noerr) = @_;
1598
1599     return $value if parse_smbios1($value);
1600
1601     return undef if $noerr;
1602
1603     die "unable to parse smbios (type 1) options\n";
1604 }
1605
1606 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1607 sub verify_bootdisk {
1608     my ($value, $noerr) = @_;
1609
1610     return $value if valid_drivename($value);
1611
1612     return undef if $noerr;
1613
1614     die "invalid boot disk '$value'\n";
1615 }
1616
1617 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1618 sub verify_numa {
1619     my ($value, $noerr) = @_;
1620
1621     return $value if parse_numa($value);
1622
1623     return undef if $noerr;
1624
1625     die "unable to parse numa options\n";
1626 }
1627
1628 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1629 sub verify_net {
1630     my ($value, $noerr) = @_;
1631
1632     return $value if parse_net($value);
1633
1634     return undef if $noerr;
1635
1636     die "unable to parse network options\n";
1637 }
1638
1639 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1640 sub verify_drive {
1641     my ($value, $noerr) = @_;
1642
1643     return $value if parse_drive(undef, $value);
1644
1645     return undef if $noerr;
1646
1647     die "unable to parse drive options\n";
1648 }
1649
1650 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1651 sub verify_hostpci {
1652     my ($value, $noerr) = @_;
1653
1654     return $value if parse_hostpci($value);
1655
1656     return undef if $noerr;
1657
1658     die "unable to parse pci id\n";
1659 }
1660
1661 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1662 sub verify_watchdog {
1663     my ($value, $noerr) = @_;
1664
1665     return $value if parse_watchdog($value);
1666
1667     return undef if $noerr;
1668
1669     die "unable to parse watchdog options\n";
1670 }
1671
1672 sub parse_watchdog {
1673     my ($value) = @_;
1674
1675     return undef if !$value;
1676
1677     my $res = {};
1678
1679     foreach my $p (split(/,/, $value)) {
1680         next if $p =~ m/^\s*$/;
1681
1682         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1683             $res->{model} = $2;
1684         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1685             $res->{action} = $2;
1686         } else {
1687             return undef;
1688         }
1689     }
1690
1691     return $res;
1692 }
1693
1694 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1695 sub verify_startup {
1696     my ($value, $noerr) = @_;
1697
1698     return $value if parse_startup($value);
1699
1700     return undef if $noerr;
1701
1702     die "unable to parse startup options\n";
1703 }
1704
1705 sub parse_startup {
1706     my ($value) = @_;
1707
1708     return undef if !$value;
1709
1710     my $res = {};
1711
1712     foreach my $p (split(/,/, $value)) {
1713         next if $p =~ m/^\s*$/;
1714
1715         if ($p =~ m/^(order=)?(\d+)$/) {
1716             $res->{order} = $2;
1717         } elsif ($p =~ m/^up=(\d+)$/) {
1718             $res->{up} = $1;
1719         } elsif ($p =~ m/^down=(\d+)$/) {
1720             $res->{down} = $1;
1721         } else {
1722             return undef;
1723         }
1724     }
1725
1726     return $res;
1727 }
1728
1729 sub parse_usb_device {
1730     my ($value) = @_;
1731
1732     return undef if !$value;
1733
1734     my @dl = split(/,/, $value);
1735     my $found;
1736
1737     my $res = {};
1738     foreach my $v (@dl) {
1739         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1740             $found = 1;
1741             $res->{vendorid} = $2;
1742             $res->{productid} = $4;
1743         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1744             $found = 1;
1745             $res->{hostbus} = $1;
1746             $res->{hostport} = $2;
1747         } elsif ($v =~ m/^spice$/) {
1748             $found = 1;
1749             $res->{spice} = 1;
1750         } else {
1751             return undef;
1752         }
1753     }
1754     return undef if !$found;
1755
1756     return $res;
1757 }
1758
1759 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1760 sub verify_usb_device {
1761     my ($value, $noerr) = @_;
1762
1763     return $value if parse_usb_device($value);
1764
1765     return undef if $noerr;
1766
1767     die "unable to parse usb device\n";
1768 }
1769
1770 # add JSON properties for create and set function
1771 sub json_config_properties {
1772     my $prop = shift;
1773
1774     foreach my $opt (keys %$confdesc) {
1775         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1776         $prop->{$opt} = $confdesc->{$opt};
1777     }
1778
1779     return $prop;
1780 }
1781
1782 sub check_type {
1783     my ($key, $value) = @_;
1784
1785     die "unknown setting '$key'\n" if !$confdesc->{$key};
1786
1787     my $type = $confdesc->{$key}->{type};
1788
1789     if (!defined($value)) {
1790         die "got undefined value\n";
1791     }
1792
1793     if ($value =~ m/[\n\r]/) {
1794         die "property contains a line feed\n";
1795     }
1796
1797     if ($type eq 'boolean') {
1798         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1799         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1800         die "type check ('boolean') failed - got '$value'\n";
1801     } elsif ($type eq 'integer') {
1802         return int($1) if $value =~ m/^(\d+)$/;
1803         die "type check ('integer') failed - got '$value'\n";
1804     } elsif ($type eq 'number') {
1805         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1806         die "type check ('number') failed - got '$value'\n";
1807     } elsif ($type eq 'string') {
1808         if (my $fmt = $confdesc->{$key}->{format}) {
1809             if ($fmt eq 'pve-qm-drive') {
1810                 # special case - we need to pass $key to parse_drive()
1811                 my $drive = parse_drive($key, $value);
1812                 return $value if $drive;
1813                 die "unable to parse drive options\n";
1814             }
1815             PVE::JSONSchema::check_format($fmt, $value);
1816             return $value;
1817         }
1818         $value =~ s/^\"(.*)\"$/$1/;
1819         return $value;
1820     } else {
1821         die "internal error"
1822     }
1823 }
1824
1825 sub lock_config_full {
1826     my ($vmid, $timeout, $code, @param) = @_;
1827
1828     my $filename = config_file_lock($vmid);
1829
1830     my $res = lock_file($filename, $timeout, $code, @param);
1831
1832     die $@ if $@;
1833
1834     return $res;
1835 }
1836
1837 sub lock_config_mode {
1838     my ($vmid, $timeout, $shared, $code, @param) = @_;
1839
1840     my $filename = config_file_lock($vmid);
1841
1842     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1843
1844     die $@ if $@;
1845
1846     return $res;
1847 }
1848
1849 sub lock_config {
1850     my ($vmid, $code, @param) = @_;
1851
1852     return lock_config_full($vmid, 10, $code, @param);
1853 }
1854
1855 sub cfs_config_path {
1856     my ($vmid, $node) = @_;
1857
1858     $node = $nodename if !$node;
1859     return "nodes/$node/qemu-server/$vmid.conf";
1860 }
1861
1862 sub check_iommu_support{
1863     #fixme : need to check IOMMU support
1864     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1865
1866     my $iommu=1;
1867     return $iommu;
1868
1869 }
1870
1871 sub config_file {
1872     my ($vmid, $node) = @_;
1873
1874     my $cfspath = cfs_config_path($vmid, $node);
1875     return "/etc/pve/$cfspath";
1876 }
1877
1878 sub config_file_lock {
1879     my ($vmid) = @_;
1880
1881     return "$lock_dir/lock-$vmid.conf";
1882 }
1883
1884 sub touch_config {
1885     my ($vmid) = @_;
1886
1887     my $conf = config_file($vmid);
1888     utime undef, undef, $conf;
1889 }
1890
1891 sub destroy_vm {
1892     my ($storecfg, $vmid, $keep_empty_config) = @_;
1893
1894     my $conffile = config_file($vmid);
1895
1896     my $conf = load_config($vmid);
1897
1898     check_lock($conf);
1899
1900     # only remove disks owned by this VM
1901     foreach_drive($conf, sub {
1902         my ($ds, $drive) = @_;
1903
1904         return if drive_is_cdrom($drive);
1905
1906         my $volid = $drive->{file};
1907
1908         return if !$volid || $volid =~ m|^/|;
1909
1910         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1911         return if !$path || !$owner || ($owner != $vmid);
1912
1913         PVE::Storage::vdisk_free($storecfg, $volid);
1914     });
1915
1916     if ($keep_empty_config) {
1917         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1918     } else {
1919         unlink $conffile;
1920     }
1921
1922     # also remove unused disk
1923     eval {
1924         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1925
1926         eval {
1927             PVE::Storage::foreach_volid($dl, sub {
1928                 my ($volid, $sid, $volname, $d) = @_;
1929                 PVE::Storage::vdisk_free($storecfg, $volid);
1930             });
1931         };
1932         warn $@ if $@;
1933
1934     };
1935     warn $@ if $@;
1936 }
1937
1938 sub load_config {
1939     my ($vmid, $node) = @_;
1940
1941     my $cfspath = cfs_config_path($vmid, $node);
1942
1943     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1944
1945     die "no such VM ('$vmid')\n" if !defined($conf);
1946
1947     return $conf;
1948 }
1949
1950 sub parse_vm_config {
1951     my ($filename, $raw) = @_;
1952
1953     return undef if !defined($raw);
1954
1955     my $res = {
1956         digest => Digest::SHA::sha1_hex($raw),
1957         snapshots => {},
1958         pending => {},
1959     };
1960
1961     $filename =~ m|/qemu-server/(\d+)\.conf$|
1962         || die "got strange filename '$filename'";
1963
1964     my $vmid = $1;
1965
1966     my $conf = $res;
1967     my $descr = '';
1968     my $section = '';
1969
1970     my @lines = split(/\n/, $raw);
1971     foreach my $line (@lines) {
1972         next if $line =~ m/^\s*$/;
1973
1974         if ($line =~ m/^\[PENDING\]\s*$/i) {
1975             $section = 'pending';
1976             $conf->{description} = $descr if $descr;
1977             $descr = '';
1978             $conf = $res->{$section} = {};
1979             next;
1980
1981         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1982             $section = $1;
1983             $conf->{description} = $descr if $descr;
1984             $descr = '';
1985             $conf = $res->{snapshots}->{$section} = {};
1986             next;
1987         }
1988
1989         if ($line =~ m/^\#(.*)\s*$/) {
1990             $descr .= PVE::Tools::decode_text($1) . "\n";
1991             next;
1992         }
1993
1994         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1995             $descr .= PVE::Tools::decode_text($2);
1996         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1997             $conf->{snapstate} = $1;
1998         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1999             my $key = $1;
2000             my $value = $2;
2001             $conf->{$key} = $value;
2002         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2003             my $value = $1;
2004             if ($section eq 'pending') {
2005                 $conf->{delete} = $value; # we parse this later
2006             } else {
2007                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2008             }
2009         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2010             my $key = $1;
2011             my $value = $2;
2012             eval { $value = check_type($key, $value); };
2013             if ($@) {
2014                 warn "vm $vmid - unable to parse value of '$key' - $@";
2015             } else {
2016                 my $fmt = $confdesc->{$key}->{format};
2017                 if ($fmt && $fmt eq 'pve-qm-drive') {
2018                     my $v = parse_drive($key, $value);
2019                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2020                         $v->{file} = $volid;
2021                         $value = print_drive($vmid, $v);
2022                     } else {
2023                         warn "vm $vmid - unable to parse value of '$key'\n";
2024                         next;
2025                     }
2026                 }
2027
2028                 if ($key eq 'cdrom') {
2029                     $conf->{ide2} = $value;
2030                 } else {
2031                     $conf->{$key} = $value;
2032                 }
2033             }
2034         }
2035     }
2036
2037     $conf->{description} = $descr if $descr;
2038
2039     delete $res->{snapstate}; # just to be sure
2040
2041     return $res;
2042 }
2043
2044 sub write_vm_config {
2045     my ($filename, $conf) = @_;
2046
2047     delete $conf->{snapstate}; # just to be sure
2048
2049     if ($conf->{cdrom}) {
2050         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2051         $conf->{ide2} = $conf->{cdrom};
2052         delete $conf->{cdrom};
2053     }
2054
2055     # we do not use 'smp' any longer
2056     if ($conf->{sockets}) {
2057         delete $conf->{smp};
2058     } elsif ($conf->{smp}) {
2059         $conf->{sockets} = $conf->{smp};
2060         delete $conf->{cores};
2061         delete $conf->{smp};
2062     }
2063
2064     my $used_volids = {};
2065
2066     my $cleanup_config = sub {
2067         my ($cref, $pending, $snapname) = @_;
2068
2069         foreach my $key (keys %$cref) {
2070             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2071                 $key eq 'snapstate' || $key eq 'pending';
2072             my $value = $cref->{$key};
2073             if ($key eq 'delete') {
2074                 die "propertry 'delete' is only allowed in [PENDING]\n"
2075                     if !$pending;
2076                 # fixme: check syntax?
2077                 next;
2078             }
2079             eval { $value = check_type($key, $value); };
2080             die "unable to parse value of '$key' - $@" if $@;
2081
2082             $cref->{$key} = $value;
2083
2084             if (!$snapname && valid_drivename($key)) {
2085                 my $drive = parse_drive($key, $value);
2086                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2087             }
2088         }
2089     };
2090
2091     &$cleanup_config($conf);
2092
2093     &$cleanup_config($conf->{pending}, 1);
2094
2095     foreach my $snapname (keys %{$conf->{snapshots}}) {
2096         die "internal error" if $snapname eq 'pending';
2097         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2098     }
2099
2100     # remove 'unusedX' settings if we re-add a volume
2101     foreach my $key (keys %$conf) {
2102         my $value = $conf->{$key};
2103         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2104             delete $conf->{$key};
2105         }
2106     }
2107
2108     my $generate_raw_config = sub {
2109         my ($conf) = @_;
2110
2111         my $raw = '';
2112
2113         # add description as comment to top of file
2114         my $descr = $conf->{description} || '';
2115         foreach my $cl (split(/\n/, $descr)) {
2116             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2117         }
2118
2119         foreach my $key (sort keys %$conf) {
2120             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2121             $raw .= "$key: $conf->{$key}\n";
2122         }
2123         return $raw;
2124     };
2125
2126     my $raw = &$generate_raw_config($conf);
2127
2128     if (scalar(keys %{$conf->{pending}})){
2129         $raw .= "\n[PENDING]\n";
2130         $raw .= &$generate_raw_config($conf->{pending});
2131     }
2132
2133     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2134         $raw .= "\n[$snapname]\n";
2135         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2136     }
2137
2138     return $raw;
2139 }
2140
2141 sub update_config_nolock {
2142     my ($vmid, $conf, $skiplock) = @_;
2143
2144     check_lock($conf) if !$skiplock;
2145
2146     my $cfspath = cfs_config_path($vmid);
2147
2148     PVE::Cluster::cfs_write_file($cfspath, $conf);
2149 }
2150
2151 sub update_config {
2152     my ($vmid, $conf, $skiplock) = @_;
2153
2154     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2155 }
2156
2157 sub load_defaults {
2158
2159     my $res = {};
2160
2161     # we use static defaults from our JSON schema configuration
2162     foreach my $key (keys %$confdesc) {
2163         if (defined(my $default = $confdesc->{$key}->{default})) {
2164             $res->{$key} = $default;
2165         }
2166     }
2167
2168     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2169     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2170
2171     return $res;
2172 }
2173
2174 sub config_list {
2175     my $vmlist = PVE::Cluster::get_vmlist();
2176     my $res = {};
2177     return $res if !$vmlist || !$vmlist->{ids};
2178     my $ids = $vmlist->{ids};
2179
2180     foreach my $vmid (keys %$ids) {
2181         my $d = $ids->{$vmid};
2182         next if !$d->{node} || $d->{node} ne $nodename;
2183         next if !$d->{type} || $d->{type} ne 'qemu';
2184         $res->{$vmid}->{exists} = 1;
2185     }
2186     return $res;
2187 }
2188
2189 # test if VM uses local resources (to prevent migration)
2190 sub check_local_resources {
2191     my ($conf, $noerr) = @_;
2192
2193     my $loc_res = 0;
2194
2195     $loc_res = 1 if $conf->{hostusb}; # old syntax
2196     $loc_res = 1 if $conf->{hostpci}; # old syntax
2197
2198     foreach my $k (keys %$conf) {
2199         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2200         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2201     }
2202
2203     die "VM uses local resources\n" if $loc_res && !$noerr;
2204
2205     return $loc_res;
2206 }
2207
2208 # check if used storages are available on all nodes (use by migrate)
2209 sub check_storage_availability {
2210     my ($storecfg, $conf, $node) = @_;
2211
2212     foreach_drive($conf, sub {
2213         my ($ds, $drive) = @_;
2214
2215         my $volid = $drive->{file};
2216         return if !$volid;
2217
2218         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2219         return if !$sid;
2220
2221         # check if storage is available on both nodes
2222         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2223         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2224    });
2225 }
2226
2227 # list nodes where all VM images are available (used by has_feature API)
2228 sub shared_nodes {
2229     my ($conf, $storecfg) = @_;
2230
2231     my $nodelist = PVE::Cluster::get_nodelist();
2232     my $nodehash = { map { $_ => 1 } @$nodelist };
2233     my $nodename = PVE::INotify::nodename();
2234
2235     foreach_drive($conf, sub {
2236         my ($ds, $drive) = @_;
2237
2238         my $volid = $drive->{file};
2239         return if !$volid;
2240
2241         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2242         if ($storeid) {
2243             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2244             if ($scfg->{disable}) {
2245                 $nodehash = {};
2246             } elsif (my $avail = $scfg->{nodes}) {
2247                 foreach my $node (keys %$nodehash) {
2248                     delete $nodehash->{$node} if !$avail->{$node};
2249                 }
2250             } elsif (!$scfg->{shared}) {
2251                 foreach my $node (keys %$nodehash) {
2252                     delete $nodehash->{$node} if $node ne $nodename
2253                 }
2254             }
2255         }
2256     });
2257
2258     return $nodehash
2259 }
2260
2261 sub check_lock {
2262     my ($conf) = @_;
2263
2264     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2265 }
2266
2267 sub check_cmdline {
2268     my ($pidfile, $pid) = @_;
2269
2270     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2271     if (defined($fh)) {
2272         my $line = <$fh>;
2273         $fh->close;
2274         return undef if !$line;
2275         my @param = split(/\0/, $line);
2276
2277         my $cmd = $param[0];
2278         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2279
2280         for (my $i = 0; $i < scalar (@param); $i++) {
2281             my $p = $param[$i];
2282             next if !$p;
2283             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2284                 my $p = $param[$i+1];
2285                 return 1 if $p && ($p eq $pidfile);
2286                 return undef;
2287             }
2288         }
2289     }
2290     return undef;
2291 }
2292
2293 sub check_running {
2294     my ($vmid, $nocheck, $node) = @_;
2295
2296     my $filename = config_file($vmid, $node);
2297
2298     die "unable to find configuration file for VM $vmid - no such machine\n"
2299         if !$nocheck && ! -f $filename;
2300
2301     my $pidfile = pidfile_name($vmid);
2302
2303     if (my $fd = IO::File->new("<$pidfile")) {
2304         my $st = stat($fd);
2305         my $line = <$fd>;
2306         close($fd);
2307
2308         my $mtime = $st->mtime;
2309         if ($mtime > time()) {
2310             warn "file '$filename' modified in future\n";
2311         }
2312
2313         if ($line =~ m/^(\d+)$/) {
2314             my $pid = $1;
2315             if (check_cmdline($pidfile, $pid)) {
2316                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2317                     return $pid;
2318                 }
2319             }
2320         }
2321     }
2322
2323     return undef;
2324 }
2325
2326 sub vzlist {
2327
2328     my $vzlist = config_list();
2329
2330     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2331
2332     while (defined(my $de = $fd->read)) {
2333         next if $de !~ m/^(\d+)\.pid$/;
2334         my $vmid = $1;
2335         next if !defined($vzlist->{$vmid});
2336         if (my $pid = check_running($vmid)) {
2337             $vzlist->{$vmid}->{pid} = $pid;
2338         }
2339     }
2340
2341     return $vzlist;
2342 }
2343
2344 sub disksize {
2345     my ($storecfg, $conf) = @_;
2346
2347     my $bootdisk = $conf->{bootdisk};
2348     return undef if !$bootdisk;
2349     return undef if !valid_drivename($bootdisk);
2350
2351     return undef if !$conf->{$bootdisk};
2352
2353     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2354     return undef if !defined($drive);
2355
2356     return undef if drive_is_cdrom($drive);
2357
2358     my $volid = $drive->{file};
2359     return undef if !$volid;
2360
2361     return $drive->{size};
2362 }
2363
2364 my $last_proc_pid_stat;
2365
2366 # get VM status information
2367 # This must be fast and should not block ($full == false)
2368 # We only query KVM using QMP if $full == true (this can be slow)
2369 sub vmstatus {
2370     my ($opt_vmid, $full) = @_;
2371
2372     my $res = {};
2373
2374     my $storecfg = PVE::Storage::config();
2375
2376     my $list = vzlist();
2377     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2378
2379     my $cpucount = $cpuinfo->{cpus} || 1;
2380
2381     foreach my $vmid (keys %$list) {
2382         next if $opt_vmid && ($vmid ne $opt_vmid);
2383
2384         my $cfspath = cfs_config_path($vmid);
2385         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2386
2387         my $d = {};
2388         $d->{pid} = $list->{$vmid}->{pid};
2389
2390         # fixme: better status?
2391         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2392
2393         my $size = disksize($storecfg, $conf);
2394         if (defined($size)) {
2395             $d->{disk} = 0; # no info available
2396             $d->{maxdisk} = $size;
2397         } else {
2398             $d->{disk} = 0;
2399             $d->{maxdisk} = 0;
2400         }
2401
2402         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2403         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2404
2405         $d->{name} = $conf->{name} || "VM $vmid";
2406         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2407
2408         if ($conf->{balloon}) {
2409             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2410             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2411         }
2412
2413         $d->{uptime} = 0;
2414         $d->{cpu} = 0;
2415         $d->{mem} = 0;
2416
2417         $d->{netout} = 0;
2418         $d->{netin} = 0;
2419
2420         $d->{diskread} = 0;
2421         $d->{diskwrite} = 0;
2422
2423         $d->{template} = is_template($conf);
2424
2425         $res->{$vmid} = $d;
2426     }
2427
2428     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2429     foreach my $dev (keys %$netdev) {
2430         next if $dev !~ m/^tap([1-9]\d*)i/;
2431         my $vmid = $1;
2432         my $d = $res->{$vmid};
2433         next if !$d;
2434
2435         $d->{netout} += $netdev->{$dev}->{receive};
2436         $d->{netin} += $netdev->{$dev}->{transmit};
2437     }
2438
2439     my $ctime = gettimeofday;
2440
2441     foreach my $vmid (keys %$list) {
2442
2443         my $d = $res->{$vmid};
2444         my $pid = $d->{pid};
2445         next if !$pid;
2446
2447         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2448         next if !$pstat; # not running
2449
2450         my $used = $pstat->{utime} + $pstat->{stime};
2451
2452         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2453
2454         if ($pstat->{vsize}) {
2455             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2456         }
2457
2458         my $old = $last_proc_pid_stat->{$pid};
2459         if (!$old) {
2460             $last_proc_pid_stat->{$pid} = {
2461                 time => $ctime,
2462                 used => $used,
2463                 cpu => 0,
2464             };
2465             next;
2466         }
2467
2468         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2469
2470         if ($dtime > 1000) {
2471             my $dutime = $used -  $old->{used};
2472
2473             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2474             $last_proc_pid_stat->{$pid} = {
2475                 time => $ctime,
2476                 used => $used,
2477                 cpu => $d->{cpu},
2478             };
2479         } else {
2480             $d->{cpu} = $old->{cpu};
2481         }
2482     }
2483
2484     return $res if !$full;
2485
2486     my $qmpclient = PVE::QMPClient->new();
2487
2488     my $ballooncb = sub {
2489         my ($vmid, $resp) = @_;
2490
2491         my $info = $resp->{'return'};
2492         return if !$info->{max_mem};
2493
2494         my $d = $res->{$vmid};
2495
2496         # use memory assigned to VM
2497         $d->{maxmem} = $info->{max_mem};
2498         $d->{balloon} = $info->{actual};
2499
2500         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2501             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2502             $d->{freemem} = $info->{free_mem};
2503         }
2504
2505     };
2506
2507     my $blockstatscb = sub {
2508         my ($vmid, $resp) = @_;
2509         my $data = $resp->{'return'} || [];
2510         my $totalrdbytes = 0;
2511         my $totalwrbytes = 0;
2512         for my $blockstat (@$data) {
2513             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2514             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2515         }
2516         $res->{$vmid}->{diskread} = $totalrdbytes;
2517         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2518     };
2519
2520     my $statuscb = sub {
2521         my ($vmid, $resp) = @_;
2522
2523         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2524         # this fails if ballon driver is not loaded, so this must be
2525         # the last commnand (following command are aborted if this fails).
2526         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2527
2528         my $status = 'unknown';
2529         if (!defined($status = $resp->{'return'}->{status})) {
2530             warn "unable to get VM status\n";
2531             return;
2532         }
2533
2534         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2535     };
2536
2537     foreach my $vmid (keys %$list) {
2538         next if $opt_vmid && ($vmid ne $opt_vmid);
2539         next if !$res->{$vmid}->{pid}; # not running
2540         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2541     }
2542
2543     $qmpclient->queue_execute(undef, 1);
2544
2545     foreach my $vmid (keys %$list) {
2546         next if $opt_vmid && ($vmid ne $opt_vmid);
2547         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2548     }
2549
2550     return $res;
2551 }
2552
2553 sub foreach_drive {
2554     my ($conf, $func) = @_;
2555
2556     foreach my $ds (keys %$conf) {
2557         next if !valid_drivename($ds);
2558
2559         my $drive = parse_drive($ds, $conf->{$ds});
2560         next if !$drive;
2561
2562         &$func($ds, $drive);
2563     }
2564 }
2565
2566 sub foreach_volid {
2567     my ($conf, $func) = @_;
2568
2569     my $volhash = {};
2570
2571     my $test_volid = sub {
2572         my ($volid, $is_cdrom) = @_;
2573
2574         return if !$volid;
2575
2576         $volhash->{$volid} = $is_cdrom || 0;
2577     };
2578
2579     foreach_drive($conf, sub {
2580         my ($ds, $drive) = @_;
2581         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2582     });
2583
2584     foreach my $snapname (keys %{$conf->{snapshots}}) {
2585         my $snap = $conf->{snapshots}->{$snapname};
2586         &$test_volid($snap->{vmstate}, 0);
2587         foreach_drive($snap, sub {
2588             my ($ds, $drive) = @_;
2589             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2590         });
2591     }
2592
2593     foreach my $volid (keys %$volhash) {
2594         &$func($volid, $volhash->{$volid});
2595     }
2596 }
2597
2598 sub vga_conf_has_spice {
2599     my ($vga) = @_;
2600
2601     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2602
2603     return $1 || 1;
2604 }
2605
2606 sub config_to_command {
2607     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2608
2609     my $cmd = [];
2610     my $globalFlags = [];
2611     my $machineFlags = [];
2612     my $rtcFlags = [];
2613     my $cpuFlags = [];
2614     my $devices = [];
2615     my $pciaddr = '';
2616     my $bridges = {};
2617     my $kvmver = kvm_user_version();
2618     my $vernum = 0; # unknown
2619     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2620         $vernum = $1*1000000+$2*1000;
2621     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2622         $vernum = $1*1000000+$2*1000+$3;
2623     }
2624
2625     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2626
2627     my $have_ovz = -f '/proc/vz/vestat';
2628
2629     my $q35 = machine_type_is_q35($conf);
2630     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2631
2632     push @$cmd, '/usr/bin/kvm';
2633
2634     push @$cmd, '-id', $vmid;
2635
2636     my $use_virtio = 0;
2637
2638     my $qmpsocket = qmp_socket($vmid);
2639     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2640     push @$cmd, '-mon', "chardev=qmp,mode=control";
2641
2642     my $socket = vnc_socket($vmid);
2643     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2644
2645     push @$cmd, '-pidfile' , pidfile_name($vmid);
2646
2647     push @$cmd, '-daemonize';
2648
2649     if ($conf->{smbios1}) {
2650         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2651     }
2652
2653     push @$cmd, '-object', "iothread,id=iothread0" if $conf->{iothread};
2654
2655     if ($q35) {
2656         # the q35 chipset support native usb2, so we enable usb controller
2657         # by default for this machine type
2658         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2659     } else {
2660         $pciaddr = print_pci_addr("piix3", $bridges);
2661         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2662
2663         my $use_usb2 = 0;
2664         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2665             next if !$conf->{"usb$i"};
2666             $use_usb2 = 1;
2667         }
2668         # include usb device config
2669         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2670     }
2671
2672     my $vga = $conf->{vga};
2673
2674     my $qxlnum = vga_conf_has_spice($vga);
2675     $vga = 'qxl' if $qxlnum;
2676
2677     if (!$vga) {
2678         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2679                                 $conf->{ostype} eq 'win7' ||
2680                                 $conf->{ostype} eq 'w2k8')) {
2681             $vga = 'std';
2682         } else {
2683             $vga = 'cirrus';
2684         }
2685     }
2686
2687     # enable absolute mouse coordinates (needed by vnc)
2688     my $tablet;
2689     if (defined($conf->{tablet})) {
2690         $tablet = $conf->{tablet};
2691     } else {
2692         $tablet = $defaults->{tablet};
2693         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2694         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2695     }
2696
2697     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2698
2699     # host pci devices
2700     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2701         my $d = parse_hostpci($conf->{"hostpci$i"});
2702         next if !$d;
2703
2704         my $pcie = $d->{pcie};
2705         if($pcie){
2706             die "q35 machine model is not enabled" if !$q35;
2707             $pciaddr = print_pcie_addr("hostpci$i");
2708         }else{
2709             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2710         }
2711
2712         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2713         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2714         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2715         if ($xvga && $xvga ne '') {
2716             push @$cpuFlags, 'kvm=off';
2717             $vga = 'none';
2718         }
2719         $driver = "vfio-pci" if $xvga ne '';
2720         my $pcidevices = $d->{pciid};
2721         my $multifunction = 1 if @$pcidevices > 1;
2722
2723         my $j=0;
2724         foreach my $pcidevice (@$pcidevices) {
2725
2726             my $id = "hostpci$i";
2727             $id .= ".$j" if $multifunction;
2728             my $addr = $pciaddr;
2729             $addr .= ".$j" if $multifunction;
2730             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2731
2732             if($j == 0){
2733                 $devicestr .= "$rombar$xvga";
2734                 $devicestr .= ",multifunction=on" if $multifunction;
2735             }
2736
2737             push @$devices, '-device', $devicestr;
2738             $j++;
2739         }
2740     }
2741
2742     # usb devices
2743     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2744         my $d = parse_usb_device($conf->{"usb$i"});
2745         next if !$d;
2746         if ($d->{vendorid} && $d->{productid}) {
2747             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2748         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2749             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2750         } elsif ($d->{spice}) {
2751             # usb redir support for spice
2752             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2753             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2754         }
2755     }
2756
2757     # serial devices
2758     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2759         if (my $path = $conf->{"serial$i"}) {
2760             if ($path eq 'socket') {
2761                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2762                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2763                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2764             } else {
2765                 die "no such serial device\n" if ! -c $path;
2766                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2767                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2768             }
2769         }
2770     }
2771
2772     # parallel devices
2773     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2774         if (my $path = $conf->{"parallel$i"}) {
2775             die "no such parallel device\n" if ! -c $path;
2776             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2777             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2778             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2779         }
2780     }
2781
2782     my $vmname = $conf->{name} || "vm$vmid";
2783
2784     push @$cmd, '-name', $vmname;
2785
2786     my $sockets = 1;
2787     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2788     $sockets = $conf->{sockets} if  $conf->{sockets};
2789
2790     my $cores = $conf->{cores} || 1;
2791
2792     my $maxcpus = $sockets * $cores;
2793
2794     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2795
2796     my $allowed_vcpus = $cpuinfo->{cpus};
2797
2798     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2799         if ($allowed_vcpus < $maxcpus);
2800
2801     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2802
2803     push @$cmd, '-nodefaults';
2804
2805     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2806
2807     my $bootindex_hash = {};
2808     my $i = 1;
2809     foreach my $o (split(//, $bootorder)) {
2810         $bootindex_hash->{$o} = $i*100;
2811         $i++;
2812     }
2813
2814     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2815
2816     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2817
2818     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2819
2820     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2821
2822     # time drift fix
2823     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2824
2825     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2826     my $useLocaltime = $conf->{localtime};
2827
2828     if (my $ost = $conf->{ostype}) {
2829         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2830
2831         if ($ost =~ m/^w/) { # windows
2832             $useLocaltime = 1 if !defined($conf->{localtime});
2833
2834             # use time drift fix when acpi is enabled
2835             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2836                 $tdf = 1 if !defined($conf->{tdf});
2837             }
2838         }
2839
2840         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2841             $ost eq 'wvista') {
2842             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2843             push @$cmd, '-no-hpet';
2844             #push @$cpuFlags , 'hv_vapic" if !$nokvm;  #fixme, my win2008R2 hang at boot with this
2845             push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2846         }
2847
2848         if ($ost eq 'win7' || $ost eq 'win8') {
2849             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2850         }
2851     }
2852
2853     push @$rtcFlags, 'driftfix=slew' if $tdf;
2854
2855     if ($nokvm) {
2856         push @$machineFlags, 'accel=tcg';
2857     } else {
2858         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2859     }
2860
2861     my $machine_type = $forcemachine || $conf->{machine};
2862     if ($machine_type) {
2863         push @$machineFlags, "type=${machine_type}";
2864     }
2865
2866     if ($conf->{startdate}) {
2867         push @$rtcFlags, "base=$conf->{startdate}";
2868     } elsif ($useLocaltime) {
2869         push @$rtcFlags, 'base=localtime';
2870     }
2871
2872     my $cpu = $nokvm ? "qemu64" : "kvm64";
2873     $cpu = $conf->{cpu} if $conf->{cpu};
2874
2875     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2876
2877     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2878
2879     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2880
2881     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2882
2883     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2884
2885     # Note: enforce needs kernel 3.10, so we do not use it for now
2886     # push @$cmd, '-cpu', "$cpu,enforce";
2887     push @$cmd, '-cpu', $cpu;
2888
2889     my $memory = $conf->{memory} || $defaults->{memory};
2890     my $static_memory = 0;
2891     my $dimm_memory = 0;
2892
2893     if ($hotplug_features->{memory}) {
2894         die "Numa need to be enabled for memory hotplug" if !$conf->{numa};
2895         die "Total memory is bigger than $MAX_MEM MB" if $memory > $MAX_MEM;
2896         die "memory should be a multiple of 128!" if ($memory % 128 != 0);
2897         $static_memory = $STATICMEM;
2898         die "minimum memory must be $static_memory"."MB" if($memory < $static_memory);
2899         $dimm_memory = $memory - $static_memory;
2900         push @$cmd, '-m', "size=".$static_memory.",slots=255,maxmem=".$MAX_MEM."M";
2901
2902     } else {
2903
2904         $static_memory = $memory;
2905         push @$cmd, '-m', $static_memory;
2906     }
2907
2908     if ($conf->{numa}) {
2909
2910         my $numa_totalmemory = undef;
2911         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2912             next if !$conf->{"numa$i"};
2913             my $numa = parse_numa($conf->{"numa$i"});
2914             next if !$numa;
2915             # memory
2916             die "missing numa node$i memory value\n" if !$numa->{memory};
2917             my $numa_memory = $numa->{memory};
2918             $numa_totalmemory += $numa_memory;
2919             my $numa_object = "memory-backend-ram,id=ram-node$i,size=$numa_memory"."M";
2920
2921             # cpus
2922             my $cpus_start = $numa->{cpus}->{start};
2923             die "missing numa node$i cpus\n" if !defined($cpus_start);
2924             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2925             my $cpus = $cpus_start;
2926             if (defined($cpus_end)) {
2927                 $cpus .= "-$cpus_end";
2928                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2929             }
2930
2931             # hostnodes
2932             my $hostnodes_start = $numa->{hostnodes}->{start};
2933             if (defined($hostnodes_start)) {
2934                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2935                 my $hostnodes = $hostnodes_start;
2936                 if (defined($hostnodes_end)) {
2937                     $hostnodes .= "-$hostnodes_end";
2938                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2939                 }
2940
2941                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2942                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2943                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2944                 }
2945
2946                 # policy
2947                 my $policy = $numa->{policy};
2948                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2949                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2950             }
2951
2952             push @$cmd, '-object', $numa_object;
2953             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2954         }
2955
2956         die "total memory for NUMA nodes must be equal to vm static memory\n"
2957             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2958
2959         #if no custom tology, we split memory and cores across numa nodes
2960         if(!$numa_totalmemory) {
2961
2962             my $numa_memory = ($static_memory / $sockets) . "M";
2963
2964             for (my $i = 0; $i < $sockets; $i++)  {
2965
2966                 my $cpustart = ($cores * $i);
2967                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2968                 my $cpus = $cpustart;
2969                 $cpus .= "-$cpuend" if $cpuend;
2970
2971                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2972                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2973             }
2974         }
2975     }
2976
2977     if ($hotplug_features->{memory}) {
2978         my $dimm_id = 0;
2979         my $dimm_size = 512;
2980         my $current_size = $static_memory;
2981         for (my $j = 0; $j < 8; $j++) {
2982             for (my $i = 0; $i < 32; $i++) {
2983                 my $name = "dimm${dimm_id}";
2984                 $dimm_id++;
2985                 last if $current_size >= $memory;
2986                 my $numanode = $i % $sockets;
2987                 push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=$dimm_size"."M";
2988                 push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2989                 $current_size += $dimm_size;
2990                 #if dimm_memory is not aligned to dimm map
2991                 if($current_size > $memory) {
2992                     $conf->{memory} = $current_size;
2993                     update_config_nolock($vmid, $conf, 1);
2994                 }
2995              }
2996              $dimm_size *= 2;
2997         }
2998     }
2999
3000     push @$cmd, '-S' if $conf->{freeze};
3001
3002     # set keyboard layout
3003     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3004     push @$cmd, '-k', $kb if $kb;
3005
3006     # enable sound
3007     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3008     #push @$cmd, '-soundhw', 'es1370';
3009     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3010
3011     if($conf->{agent}) {
3012         my $qgasocket = qmp_socket($vmid, 1);
3013         my $pciaddr = print_pci_addr("qga0", $bridges);
3014         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3015         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3016         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3017     }
3018
3019     my $spice_port;
3020
3021     if ($qxlnum) {
3022         if ($qxlnum > 1) {
3023             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3024                 for(my $i = 1; $i < $qxlnum; $i++){
3025                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3026                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3027                 }
3028             } else {
3029                 # assume other OS works like Linux
3030                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3031                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3032             }
3033         }
3034
3035         my $pciaddr = print_pci_addr("spice", $bridges);
3036
3037         $spice_port = PVE::Tools::next_spice_port();
3038
3039         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3040
3041         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3042         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3043         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3044     }
3045
3046     # enable balloon by default, unless explicitly disabled
3047     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3048         $pciaddr = print_pci_addr("balloon0", $bridges);
3049         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3050     }
3051
3052     if ($conf->{watchdog}) {
3053         my $wdopts = parse_watchdog($conf->{watchdog});
3054         $pciaddr = print_pci_addr("watchdog", $bridges);
3055         my $watchdog = $wdopts->{model} || 'i6300esb';
3056         push @$devices, '-device', "$watchdog$pciaddr";
3057         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3058     }
3059
3060     my $vollist = [];
3061     my $scsicontroller = {};
3062     my $ahcicontroller = {};
3063     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3064
3065     # Add iscsi initiator name if available
3066     if (my $initiator = get_initiator_name()) {
3067         push @$devices, '-iscsi', "initiator-name=$initiator";
3068     }
3069
3070     foreach_drive($conf, sub {
3071         my ($ds, $drive) = @_;
3072
3073         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3074             push @$vollist, $drive->{file};
3075         }
3076
3077         $use_virtio = 1 if $ds =~ m/^virtio/;
3078
3079         if (drive_is_cdrom ($drive)) {
3080             if ($bootindex_hash->{d}) {
3081                 $drive->{bootindex} = $bootindex_hash->{d};
3082                 $bootindex_hash->{d} += 1;
3083             }
3084         } else {
3085             if ($bootindex_hash->{c}) {
3086                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3087                 $bootindex_hash->{c} += 1;
3088             }
3089         }
3090
3091         if ($drive->{interface} eq 'scsi') {
3092
3093             my $maxdev = ($scsihw !~ m/^lsi/) ? 256 : 7;
3094             my $controller = int($drive->{index} / $maxdev);
3095             $pciaddr = print_pci_addr("scsihw$controller", $bridges);
3096             push @$devices, '-device', "$scsihw,id=scsihw$controller$pciaddr" if !$scsicontroller->{$controller};
3097             $scsicontroller->{$controller}=1;
3098         }
3099
3100         if ($drive->{interface} eq 'sata') {
3101            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3102            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3103            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3104            $ahcicontroller->{$controller}=1;
3105         }
3106
3107         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3108         push @$devices, '-drive',$drive_cmd;
3109         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3110     });
3111
3112     for (my $i = 0; $i < $MAX_NETS; $i++) {
3113          next if !$conf->{"net$i"};
3114          my $d = parse_net($conf->{"net$i"});
3115          next if !$d;
3116
3117          $use_virtio = 1 if $d->{model} eq 'virtio';
3118
3119          if ($bootindex_hash->{n}) {
3120             $d->{bootindex} = $bootindex_hash->{n};
3121             $bootindex_hash->{n} += 1;
3122          }
3123
3124          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3125          push @$devices, '-netdev', $netdevfull;
3126
3127          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3128          push @$devices, '-device', $netdevicefull;
3129     }
3130
3131     if (!$q35) {
3132         # add pci bridges
3133         while (my ($k, $v) = each %$bridges) {
3134             $pciaddr = print_pci_addr("pci.$k");
3135             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3136         }
3137     }
3138
3139     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3140     # when the VM uses virtio devices.
3141     if (!$use_virtio && $have_ovz) {
3142
3143         my $cpuunits = defined($conf->{cpuunits}) ?
3144             $conf->{cpuunits} : $defaults->{cpuunits};
3145
3146         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3147
3148         # fixme: cpulimit is currently ignored
3149         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3150     }
3151
3152     # add custom args
3153     if ($conf->{args}) {
3154         my $aa = PVE::Tools::split_args($conf->{args});
3155         push @$cmd, @$aa;
3156     }
3157
3158     push @$cmd, @$devices;
3159     push @$cmd, '-rtc', join(',', @$rtcFlags)
3160         if scalar(@$rtcFlags);
3161     push @$cmd, '-machine', join(',', @$machineFlags)
3162         if scalar(@$machineFlags);
3163     push @$cmd, '-global', join(',', @$globalFlags)
3164         if scalar(@$globalFlags);
3165
3166     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3167 }
3168
3169 sub vnc_socket {
3170     my ($vmid) = @_;
3171     return "${var_run_tmpdir}/$vmid.vnc";
3172 }
3173
3174 sub spice_port {
3175     my ($vmid) = @_;
3176
3177     my $res = vm_mon_cmd($vmid, 'query-spice');
3178
3179     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3180 }
3181
3182 sub qmp_socket {
3183     my ($vmid, $qga) = @_;
3184     my $sockettype = $qga ? 'qga' : 'qmp';
3185     return "${var_run_tmpdir}/$vmid.$sockettype";
3186 }
3187
3188 sub pidfile_name {
3189     my ($vmid) = @_;
3190     return "${var_run_tmpdir}/$vmid.pid";
3191 }
3192
3193 sub vm_devices_list {
3194     my ($vmid) = @_;
3195
3196     my $res = vm_mon_cmd($vmid, 'query-pci');
3197     my $devices = {};
3198     foreach my $pcibus (@$res) {
3199         foreach my $device (@{$pcibus->{devices}}) {
3200             next if !$device->{'qdev_id'};
3201             $devices->{$device->{'qdev_id'}} = 1;
3202         }
3203     }
3204
3205     my $resblock = vm_mon_cmd($vmid, 'query-block');
3206     foreach my $block (@$resblock) {
3207         if($block->{device} =~ m/^drive-(\S+)/){
3208                 $devices->{$1} = 1;
3209         }
3210     }
3211
3212     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3213     foreach my $mice (@$resmice) {
3214         if ($mice->{name} eq 'QEMU HID Tablet') {
3215             $devices->{tablet} = 1;
3216             last;
3217         }
3218     }
3219
3220     return $devices;
3221 }
3222
3223 sub vm_deviceplug {
3224     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3225
3226     my $q35 = machine_type_is_q35($conf);
3227
3228     my $devices_list = vm_devices_list($vmid);
3229     return 1 if defined($devices_list->{$deviceid});
3230
3231     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3232
3233     if ($deviceid eq 'tablet') {
3234
3235         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3236
3237     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3238
3239         qemu_driveadd($storecfg, $vmid, $device);
3240         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3241
3242         qemu_deviceadd($vmid, $devicefull);
3243         eval { qemu_deviceaddverify($vmid, $deviceid); };
3244         if (my $err = $@) {
3245             eval { qemu_drivedel($vmid, $deviceid); };
3246             warn $@ if $@;
3247             die $err;
3248         }
3249
3250     } elsif ($deviceid =~ m/^(scsihw)(\d+)$/) {
3251
3252         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3253         my $pciaddr = print_pci_addr($deviceid);
3254         my $devicefull = "$scsihw,id=$deviceid$pciaddr";
3255
3256         qemu_deviceadd($vmid, $devicefull);
3257         qemu_deviceaddverify($vmid, $deviceid);
3258
3259     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3260
3261         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3262         qemu_driveadd($storecfg, $vmid, $device);
3263
3264         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3265         eval { qemu_deviceadd($vmid, $devicefull); };
3266         if (my $err = $@) {
3267             eval { qemu_drivedel($vmid, $deviceid); };
3268             warn $@ if $@;
3269             die $err;
3270         }
3271
3272     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3273
3274         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3275         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3276         qemu_deviceadd($vmid, $netdevicefull);
3277         eval { qemu_deviceaddverify($vmid, $deviceid); };
3278         if (my $err = $@) {
3279             eval { qemu_netdevdel($vmid, $deviceid); };
3280             warn $@ if $@;
3281             die $err;
3282         }
3283
3284     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3285
3286         my $bridgeid = $2;
3287         my $pciaddr = print_pci_addr($deviceid);
3288         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3289
3290         qemu_deviceadd($vmid, $devicefull);
3291         qemu_deviceaddverify($vmid, $deviceid);
3292
3293     } else {
3294         die "can't hotplug device '$deviceid'\n";
3295     }
3296
3297     return 1;
3298 }
3299
3300 # fixme: this should raise exceptions on error!
3301 sub vm_deviceunplug {
3302     my ($vmid, $conf, $deviceid) = @_;
3303
3304     my $devices_list = vm_devices_list($vmid);
3305     return 1 if !defined($devices_list->{$deviceid});
3306
3307     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3308
3309     if ($deviceid eq 'tablet') {
3310
3311         qemu_devicedel($vmid, $deviceid);
3312
3313     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3314
3315         qemu_devicedel($vmid, $deviceid);
3316         qemu_devicedelverify($vmid, $deviceid);
3317         qemu_drivedel($vmid, $deviceid);
3318
3319     } elsif ($deviceid =~ m/^(lsi)(\d+)$/) {
3320
3321         qemu_devicedel($vmid, $deviceid);
3322
3323     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3324
3325         qemu_devicedel($vmid, $deviceid);
3326         qemu_drivedel($vmid, $deviceid);
3327
3328     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3329
3330         qemu_devicedel($vmid, $deviceid);
3331         qemu_devicedelverify($vmid, $deviceid);
3332         qemu_netdevdel($vmid, $deviceid);
3333
3334     } else {
3335         die "can't unplug device '$deviceid'\n";
3336     }
3337
3338     return 1;
3339 }
3340
3341 sub qemu_deviceadd {
3342     my ($vmid, $devicefull) = @_;
3343
3344     $devicefull = "driver=".$devicefull;
3345     my %options =  split(/[=,]/, $devicefull);
3346
3347     vm_mon_cmd($vmid, "device_add" , %options);
3348 }
3349
3350 sub qemu_devicedel {
3351     my ($vmid, $deviceid) = @_;
3352
3353     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3354 }
3355
3356 sub qemu_objectadd {
3357     my($vmid, $objectid, $qomtype) = @_;
3358
3359     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3360
3361     return 1;
3362 }
3363
3364 sub qemu_objectdel {
3365     my($vmid, $objectid) = @_;
3366
3367     vm_mon_cmd($vmid, "object-del", id => $objectid);
3368
3369     return 1;
3370 }
3371
3372 sub qemu_driveadd {
3373     my ($storecfg, $vmid, $device) = @_;
3374
3375     my $drive = print_drive_full($storecfg, $vmid, $device);
3376     my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
3377
3378     # If the command succeeds qemu prints: "OK"
3379     return 1 if $ret =~ m/OK/s;
3380
3381     die "adding drive failed: $ret\n";
3382 }
3383
3384 sub qemu_drivedel {
3385     my($vmid, $deviceid) = @_;
3386
3387     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3388     $ret =~ s/^\s+//;
3389
3390     return 1 if $ret eq "";
3391
3392     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3393     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3394
3395     die "deleting drive $deviceid failed : $ret\n";
3396 }
3397
3398 sub qemu_deviceaddverify {
3399     my ($vmid, $deviceid) = @_;
3400
3401     for (my $i = 0; $i <= 5; $i++) {
3402          my $devices_list = vm_devices_list($vmid);
3403          return 1 if defined($devices_list->{$deviceid});
3404          sleep 1;
3405     }
3406
3407     die "error on hotplug device '$deviceid'\n";
3408 }
3409
3410
3411 sub qemu_devicedelverify {
3412     my ($vmid, $deviceid) = @_;
3413
3414     # need to verify that the device is correctly removed as device_del
3415     # is async and empty return is not reliable
3416
3417     for (my $i = 0; $i <= 5; $i++) {
3418          my $devices_list = vm_devices_list($vmid);
3419          return 1 if !defined($devices_list->{$deviceid});
3420          sleep 1;
3421     }
3422
3423     die "error on hot-unplugging device '$deviceid'\n";
3424 }
3425
3426 sub qemu_findorcreatescsihw {
3427     my ($storecfg, $conf, $vmid, $device) = @_;
3428
3429     my $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
3430     my $controller = int($device->{index} / $maxdev);
3431     my $scsihwid="scsihw$controller";
3432     my $devices_list = vm_devices_list($vmid);
3433
3434     if(!defined($devices_list->{$scsihwid})) {
3435         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
3436     }
3437
3438     return 1;
3439 }
3440
3441 sub qemu_add_pci_bridge {
3442     my ($storecfg, $conf, $vmid, $device) = @_;
3443
3444     my $bridges = {};
3445
3446     my $bridgeid;
3447
3448     print_pci_addr($device, $bridges);
3449
3450     while (my ($k, $v) = each %$bridges) {
3451         $bridgeid = $k;
3452     }
3453     return 1 if !defined($bridgeid) || $bridgeid < 1;
3454
3455     my $bridge = "pci.$bridgeid";
3456     my $devices_list = vm_devices_list($vmid);
3457
3458     if (!defined($devices_list->{$bridge})) {
3459         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3460     }
3461
3462     return 1;
3463 }
3464
3465 sub qemu_set_link_status {
3466     my ($vmid, $device, $up) = @_;
3467
3468     vm_mon_cmd($vmid, "set_link", name => $device,
3469                up => $up ? JSON::true : JSON::false);
3470 }
3471
3472 sub qemu_netdevadd {
3473     my ($vmid, $conf, $device, $deviceid) = @_;
3474
3475     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3476     my %options =  split(/[=,]/, $netdev);
3477
3478     vm_mon_cmd($vmid, "netdev_add",  %options);
3479     return 1;
3480 }
3481
3482 sub qemu_netdevdel {
3483     my ($vmid, $deviceid) = @_;
3484
3485     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3486 }
3487
3488 sub qemu_cpu_hotplug {
3489     my ($vmid, $conf, $vcpus) = @_;
3490
3491     my $sockets = 1;
3492     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3493     $sockets = $conf->{sockets} if  $conf->{sockets};
3494     my $cores = $conf->{cores} || 1;
3495     my $maxcpus = $sockets * $cores;
3496
3497     $vcpus = $maxcpus if !$vcpus;
3498
3499     die "you can't add more vcpus than maxcpus\n"
3500         if $vcpus > $maxcpus;
3501
3502     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3503     die "online cpu unplug is not yet possible\n"
3504         if $vcpus < $currentvcpus;
3505
3506     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3507     die "vcpus in running vm is different than configuration\n"
3508         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3509
3510     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3511         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3512     }
3513 }
3514
3515 sub qemu_memory_hotplug {
3516     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3517
3518     return $value if !check_running($vmid);
3519
3520     my $memory = $conf->{memory} || $defaults->{memory};
3521     $value = $defaults->{memory} if !$value;
3522     return $value if $value == $memory;
3523
3524     my $static_memory = $STATICMEM;
3525     my $dimm_memory = $memory - $static_memory;
3526
3527     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3528     die "memory unplug is not yet available" if $value < $memory;
3529     die "memory should be a multiple of 128!\n" if ($value % 128 != 0);
3530     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3531
3532
3533     my $sockets = 1;
3534     $sockets = $conf->{sockets} if $conf->{sockets};
3535
3536     my $dimm_id = 0;
3537     my $current_size = $static_memory;
3538     my $dimm_size = 512;
3539     for (my $j = 0; $j < 8; $j++) {
3540         for (my $i = 0; $i < 32; $i++) {
3541             my $name = "dimm${dimm_id}";
3542             $dimm_id++;
3543             $current_size += $dimm_size;
3544             next if $current_size <= $memory;
3545             my $numanode = $i % $sockets;
3546
3547             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3548             if (my $err = $@) {
3549                 eval { qemu_objectdel($vmid, "mem-$name"); };
3550                 die $err;
3551             }
3552
3553             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3554             if (my $err = $@) {
3555                 eval { qemu_objectdel($vmid, "mem-$name"); };
3556                 die $err;
3557             }
3558             #update conf after each succesful module hotplug
3559             $conf->{$opt} = $current_size;
3560             update_config_nolock($vmid, $conf, 1);
3561
3562             return $current_size if $current_size >= $value;
3563         }
3564         $dimm_size *= 2;
3565     }
3566
3567 }
3568
3569 sub qemu_block_set_io_throttle {
3570     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3571
3572     return if !check_running($vmid) ;
3573
3574     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3575
3576 }
3577
3578 # old code, only used to shutdown old VM after update
3579 sub __read_avail {
3580     my ($fh, $timeout) = @_;
3581
3582     my $sel = new IO::Select;
3583     $sel->add($fh);
3584
3585     my $res = '';
3586     my $buf;
3587
3588     my @ready;
3589     while (scalar (@ready = $sel->can_read($timeout))) {
3590         my $count;
3591         if ($count = $fh->sysread($buf, 8192)) {
3592             if ($buf =~ /^(.*)\(qemu\) $/s) {
3593                 $res .= $1;
3594                 last;
3595             } else {
3596                 $res .= $buf;
3597             }
3598         } else {
3599             if (!defined($count)) {
3600                 die "$!\n";
3601             }
3602             last;
3603         }
3604     }
3605
3606     die "monitor read timeout\n" if !scalar(@ready);
3607
3608     return $res;
3609 }
3610
3611 # old code, only used to shutdown old VM after update
3612 sub vm_monitor_command {
3613     my ($vmid, $cmdstr, $nocheck) = @_;
3614
3615     my $res;
3616
3617     eval {
3618         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3619
3620         my $sname = "${var_run_tmpdir}/$vmid.mon";
3621
3622         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3623             die "unable to connect to VM $vmid socket - $!\n";
3624
3625         my $timeout = 3;
3626
3627         # hack: migrate sometime blocks the monitor (when migrate_downtime
3628         # is set)
3629         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3630             $timeout = 60*60; # 1 hour
3631         }
3632
3633         # read banner;
3634         my $data = __read_avail($sock, $timeout);
3635
3636         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3637             die "got unexpected qemu monitor banner\n";
3638         }
3639
3640         my $sel = new IO::Select;
3641         $sel->add($sock);
3642
3643         if (!scalar(my @ready = $sel->can_write($timeout))) {
3644             die "monitor write error - timeout";
3645         }
3646
3647         my $fullcmd = "$cmdstr\r";
3648
3649         # syslog('info', "VM $vmid monitor command: $cmdstr");
3650
3651         my $b;
3652         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3653             die "monitor write error - $!";
3654         }
3655
3656         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3657
3658         $timeout = 20;
3659
3660         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3661             $timeout = 60*60; # 1 hour
3662         } elsif ($cmdstr =~ m/^(eject|change)/) {
3663             $timeout = 60; # note: cdrom mount command is slow
3664         }
3665         if ($res = __read_avail($sock, $timeout)) {
3666
3667             my @lines = split("\r?\n", $res);
3668
3669             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3670
3671             $res = join("\n", @lines);
3672             $res .= "\n";
3673         }
3674     };
3675
3676     my $err = $@;
3677
3678     if ($err) {
3679         syslog("err", "VM $vmid monitor command failed - $err");
3680         die $err;
3681     }
3682
3683     return $res;
3684 }
3685
3686 sub qemu_block_resize {
3687     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3688
3689     my $running = check_running($vmid);
3690
3691     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3692
3693     return if !$running;
3694
3695     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3696
3697 }
3698
3699 sub qemu_volume_snapshot {
3700     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3701
3702     my $running = check_running($vmid);
3703
3704     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3705
3706     return if !$running;
3707
3708     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3709
3710 }
3711
3712 sub qemu_volume_snapshot_delete {
3713     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3714
3715     my $running = check_running($vmid);
3716
3717     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3718
3719     return if !$running;
3720
3721     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3722 }
3723
3724 sub set_migration_caps {
3725     my ($vmid) = @_;
3726
3727     my $cap_ref = [];
3728
3729     my $enabled_cap = {
3730         "auto-converge" => 1,
3731         "xbzrle" => 0,
3732         "x-rdma-pin-all" => 0,
3733         "zero-blocks" => 0,
3734     };
3735
3736     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3737
3738     for my $supported_capability (@$supported_capabilities) {
3739         push @$cap_ref, {
3740             capability => $supported_capability->{capability},
3741             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3742         };
3743     }
3744
3745     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3746 }
3747
3748 my $fast_plug_option = {
3749     'name' => 1,
3750     'hotplug' => 1,
3751     'onboot' => 1,
3752     'shares' => 1,
3753     'startup' => 1,
3754 };
3755
3756 # hotplug changes in [PENDING]
3757 # $selection hash can be used to only apply specified options, for
3758 # example: { cores => 1 } (only apply changed 'cores')
3759 # $errors ref is used to return error messages
3760 sub vmconfig_hotplug_pending {
3761     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3762
3763     my $defaults = load_defaults();
3764
3765     # commit values which do not have any impact on running VM first
3766     # Note: those option cannot raise errors, we we do not care about
3767     # $selection and always apply them.
3768
3769     my $add_error = sub {
3770         my ($opt, $msg) = @_;
3771         $errors->{$opt} = "hotplug problem - $msg";
3772     };
3773
3774     my $changes = 0;
3775     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3776         if ($fast_plug_option->{$opt}) {
3777             $conf->{$opt} = $conf->{pending}->{$opt};
3778             delete $conf->{pending}->{$opt};
3779             $changes = 1;
3780         }
3781     }
3782
3783     if ($changes) {
3784         update_config_nolock($vmid, $conf, 1);
3785         $conf = load_config($vmid); # update/reload
3786     }
3787
3788     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3789
3790     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3791     foreach my $opt (@delete) {
3792         next if $selection && !$selection->{$opt};
3793         eval {
3794             if ($opt eq 'tablet') {
3795                 die "skip\n" if !$hotplug_features->{usb};
3796                 if ($defaults->{tablet}) {
3797                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3798                 } else {
3799                     vm_deviceunplug($vmid, $conf, $opt);
3800                 }
3801             } elsif ($opt eq 'vcpus') {
3802                 die "skip\n" if !$hotplug_features->{cpu};
3803                 qemu_cpu_hotplug($vmid, $conf, undef);
3804             } elsif ($opt eq 'balloon') {
3805                 # enable balloon device is not hotpluggable
3806                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3807             } elsif ($fast_plug_option->{$opt}) {
3808                 # do nothing
3809             } elsif ($opt =~ m/^net(\d+)$/) {
3810                 die "skip\n" if !$hotplug_features->{network};
3811                 vm_deviceunplug($vmid, $conf, $opt);
3812             } elsif (valid_drivename($opt)) {
3813                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3814                 vm_deviceunplug($vmid, $conf, $opt);
3815                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3816             } elsif ($opt =~ m/^memory$/) {
3817                 die "skip\n" if !$hotplug_features->{memory};
3818                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3819             } else {
3820                 die "skip\n";
3821             }
3822         };
3823         if (my $err = $@) {
3824             &$add_error($opt, $err) if $err ne "skip\n";
3825         } else {
3826             # save new config if hotplug was successful
3827             delete $conf->{$opt};
3828             vmconfig_undelete_pending_option($conf, $opt);
3829             update_config_nolock($vmid, $conf, 1);
3830             $conf = load_config($vmid); # update/reload
3831         }
3832     }
3833
3834     foreach my $opt (keys %{$conf->{pending}}) {
3835         next if $selection && !$selection->{$opt};
3836         my $value = $conf->{pending}->{$opt};
3837         eval {
3838             if ($opt eq 'tablet') {
3839                 die "skip\n" if !$hotplug_features->{usb};
3840                 if ($value == 1) {
3841                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3842                 } elsif ($value == 0) {
3843                     vm_deviceunplug($vmid, $conf, $opt);
3844                 }
3845             } elsif ($opt eq 'vcpus') {
3846                 die "skip\n" if !$hotplug_features->{cpu};
3847                 qemu_cpu_hotplug($vmid, $conf, $value);
3848             } elsif ($opt eq 'balloon') {
3849                 # enable/disable balloning device is not hotpluggable
3850                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3851                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3852                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3853
3854                 # allow manual ballooning if shares is set to zero
3855                 if (!(defined($conf->{shares}) && ($conf->{shares} == 0))) {
3856                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3857                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3858                 }
3859             } elsif ($opt =~ m/^net(\d+)$/) {
3860                 # some changes can be done without hotplug
3861                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3862                                     $vmid, $opt, $value);
3863             } elsif (valid_drivename($opt)) {
3864                 # some changes can be done without hotplug
3865                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3866                                      $vmid, $opt, $value, 1);
3867             } elsif ($opt =~ m/^memory$/) { #dimms
3868                 die "skip\n" if !$hotplug_features->{memory};
3869                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3870             } else {
3871                 die "skip\n";  # skip non-hot-pluggable options
3872             }
3873         };
3874         if (my $err = $@) {
3875             &$add_error($opt, $err) if $err ne "skip\n";
3876         } else {
3877             # save new config if hotplug was successful
3878             $conf->{$opt} = $value;
3879             delete $conf->{pending}->{$opt};
3880             update_config_nolock($vmid, $conf, 1);
3881             $conf = load_config($vmid); # update/reload
3882         }
3883     }
3884 }
3885
3886 sub vmconfig_apply_pending {
3887     my ($vmid, $conf, $storecfg) = @_;
3888
3889     # cold plug
3890
3891     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3892     foreach my $opt (@delete) { # delete
3893         die "internal error" if $opt =~ m/^unused/;
3894         $conf = load_config($vmid); # update/reload
3895         if (!defined($conf->{$opt})) {
3896             vmconfig_undelete_pending_option($conf, $opt);
3897             update_config_nolock($vmid, $conf, 1);
3898         } elsif (valid_drivename($opt)) {
3899             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3900             vmconfig_undelete_pending_option($conf, $opt);
3901             delete $conf->{$opt};
3902             update_config_nolock($vmid, $conf, 1);
3903         } else {
3904             vmconfig_undelete_pending_option($conf, $opt);
3905             delete $conf->{$opt};
3906             update_config_nolock($vmid, $conf, 1);
3907         }
3908     }
3909
3910     $conf = load_config($vmid); # update/reload
3911
3912     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3913         $conf = load_config($vmid); # update/reload
3914
3915         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3916             # skip if nothing changed
3917         } elsif (valid_drivename($opt)) {
3918             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3919                 if defined($conf->{$opt});
3920             $conf->{$opt} = $conf->{pending}->{$opt};
3921         } else {
3922             $conf->{$opt} = $conf->{pending}->{$opt};
3923         }
3924
3925         delete $conf->{pending}->{$opt};
3926         update_config_nolock($vmid, $conf, 1);
3927     }
3928 }
3929
3930 my $safe_num_ne = sub {
3931     my ($a, $b) = @_;
3932
3933     return 0 if !defined($a) && !defined($b);
3934     return 1 if !defined($a);
3935     return 1 if !defined($b);
3936
3937     return $a != $b;
3938 };
3939
3940 my $safe_string_ne = sub {
3941     my ($a, $b) = @_;
3942
3943     return 0 if !defined($a) && !defined($b);
3944     return 1 if !defined($a);
3945     return 1 if !defined($b);
3946
3947     return $a ne $b;
3948 };
3949
3950 sub vmconfig_update_net {
3951     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
3952
3953     my $newnet = parse_net($value);
3954
3955     if ($conf->{$opt}) {
3956         my $oldnet = parse_net($conf->{$opt});
3957
3958         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
3959             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
3960             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
3961             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
3962
3963             # for non online change, we try to hot-unplug
3964             die "skip\n" if !$hotplug;
3965             vm_deviceunplug($vmid, $conf, $opt);
3966         } else {
3967
3968             die "internal error" if $opt !~ m/net(\d+)/;
3969             my $iface = "tap${vmid}i$1";
3970
3971             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
3972                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
3973             }
3974
3975             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
3976                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
3977                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
3978                 PVE::Network::tap_unplug($iface);
3979                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
3980             }
3981
3982             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
3983                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
3984             }
3985
3986             return 1;
3987         }
3988     }
3989
3990     if ($hotplug) {
3991         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
3992     } else {
3993         die "skip\n";
3994     }
3995 }
3996
3997 sub vmconfig_update_disk {
3998     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
3999
4000     # fixme: do we need force?
4001
4002     my $drive = parse_drive($opt, $value);
4003
4004     if ($conf->{$opt}) {
4005
4006         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4007
4008             my $media = $drive->{media} || 'disk';
4009             my $oldmedia = $old_drive->{media} || 'disk';
4010             die "unable to change media type\n" if $media ne $oldmedia;
4011
4012             if (!drive_is_cdrom($old_drive)) {
4013
4014                 if ($drive->{file} ne $old_drive->{file}) {
4015
4016                     die "skip\n" if !$hotplug;
4017
4018                     # unplug and register as unused
4019                     vm_deviceunplug($vmid, $conf, $opt);
4020                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4021
4022                 } else {
4023                     # update existing disk
4024
4025                     # skip non hotpluggable value
4026                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4027                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4028                         die "skip\n";
4029                     }
4030
4031                     # apply throttle
4032                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4033                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4034                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4035                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4036                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4037                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4038                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4039                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4040                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4041                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4042                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4043                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4044
4045                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4046                                                    ($drive->{mbps} || 0)*1024*1024,
4047                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4048                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4049                                                    $drive->{iops} || 0,
4050                                                    $drive->{iops_rd} || 0,
4051                                                    $drive->{iops_wr} || 0,
4052                                                    ($drive->{mbps_max} || 0)*1024*1024,
4053                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4054                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4055                                                    $drive->{iops_max} || 0,
4056                                                    $drive->{iops_rd_max} || 0,
4057                                                    $drive->{iops_wr_max} || 0);
4058
4059                     }
4060
4061                     return 1;
4062                 }
4063             }
4064         }
4065     }
4066
4067     if (drive_is_cdrom($drive)) { # cdrom
4068
4069         if ($drive->{file} eq 'none') {
4070             vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4071         } else {
4072             my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4073             vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4074             vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4075         }
4076
4077     } else {
4078         die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4079         # hotplug new disks
4080         vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4081     }
4082 }
4083
4084 sub vm_start {
4085     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4086
4087     lock_config($vmid, sub {
4088         my $conf = load_config($vmid, $migratedfrom);
4089
4090         die "you can't start a vm if it's a template\n" if is_template($conf);
4091
4092         check_lock($conf) if !$skiplock;
4093
4094         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4095
4096         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4097             vmconfig_apply_pending($vmid, $conf, $storecfg);
4098             $conf = load_config($vmid); # update/reload
4099         }
4100
4101         my $defaults = load_defaults();
4102
4103         # set environment variable useful inside network script
4104         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4105
4106         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4107
4108         my $migrate_port = 0;
4109         my $migrate_uri;
4110         if ($statefile) {
4111             if ($statefile eq 'tcp') {
4112                 my $localip = "localhost";
4113                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4114                 if ($datacenterconf->{migration_unsecure}) {
4115                         my $nodename = PVE::INotify::nodename();
4116                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4117                 }
4118                 $migrate_port = PVE::Tools::next_migrate_port();
4119                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4120                 push @$cmd, '-incoming', $migrate_uri;
4121                 push @$cmd, '-S';
4122             } else {
4123                 push @$cmd, '-loadstate', $statefile;
4124             }
4125         } elsif ($paused) {
4126             push @$cmd, '-S';
4127         }
4128
4129         # host pci devices
4130         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4131           my $d = parse_hostpci($conf->{"hostpci$i"});
4132           next if !$d;
4133           my $pcidevices = $d->{pciid};
4134           foreach my $pcidevice (@$pcidevices) {
4135                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4136
4137                 my $info = pci_device_info("0000:$pciid");
4138                 die "IOMMU not present\n" if !check_iommu_support();
4139                 die "no pci device info for device '$pciid'\n" if !$info;
4140
4141                 if ($d->{driver} && $d->{driver} eq "vfio") {
4142                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4143                 } else {
4144                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4145                 }
4146
4147                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4148           }
4149         }
4150
4151         PVE::Storage::activate_volumes($storecfg, $vollist);
4152
4153         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4154                     umask => 0077); };
4155         my $err = $@;
4156         die "start failed: $err" if $err;
4157
4158         print "migration listens on $migrate_uri\n" if $migrate_uri;
4159
4160         if ($statefile && $statefile ne 'tcp')  {
4161             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4162             warn $@ if $@;
4163         }
4164
4165         if ($migratedfrom) {
4166
4167             eval {
4168                 set_migration_caps($vmid);
4169             };
4170             warn $@ if $@;
4171
4172             if ($spice_port) {
4173                 print "spice listens on port $spice_port\n";
4174                 if ($spice_ticket) {
4175                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4176                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4177                 }
4178             }
4179
4180         } else {
4181
4182             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4183                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4184                     if $conf->{balloon};
4185                 vm_mon_cmd_nocheck($vmid, 'qom-set',
4186                             path => "machine/peripheral/balloon0",
4187                             property => "guest-stats-polling-interval",
4188                             value => 2);
4189             }
4190
4191             foreach my $opt (keys %$conf) {
4192                 next if $opt !~  m/^net\d+$/;
4193                 my $nicconf = parse_net($conf->{$opt});
4194                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4195             }
4196         }
4197     });
4198 }
4199
4200 sub vm_mon_cmd {
4201     my ($vmid, $execute, %params) = @_;
4202
4203     my $cmd = { execute => $execute, arguments => \%params };
4204     vm_qmp_command($vmid, $cmd);
4205 }
4206
4207 sub vm_mon_cmd_nocheck {
4208     my ($vmid, $execute, %params) = @_;
4209
4210     my $cmd = { execute => $execute, arguments => \%params };
4211     vm_qmp_command($vmid, $cmd, 1);
4212 }
4213
4214 sub vm_qmp_command {
4215     my ($vmid, $cmd, $nocheck) = @_;
4216
4217     my $res;
4218
4219     my $timeout;
4220     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4221         $timeout = $cmd->{arguments}->{timeout};
4222         delete $cmd->{arguments}->{timeout};
4223     }
4224
4225     eval {
4226         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4227         my $sname = qmp_socket($vmid);
4228         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4229             my $qmpclient = PVE::QMPClient->new();
4230
4231             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4232         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4233             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4234                 if scalar(%{$cmd->{arguments}});
4235             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4236         } else {
4237             die "unable to open monitor socket\n";
4238         }
4239     };
4240     if (my $err = $@) {
4241         syslog("err", "VM $vmid qmp command failed - $err");
4242         die $err;
4243     }
4244
4245     return $res;
4246 }
4247
4248 sub vm_human_monitor_command {
4249     my ($vmid, $cmdline) = @_;
4250
4251     my $res;
4252
4253     my $cmd = {
4254         execute => 'human-monitor-command',
4255         arguments => { 'command-line' => $cmdline},
4256     };
4257
4258     return vm_qmp_command($vmid, $cmd);
4259 }
4260
4261 sub vm_commandline {
4262     my ($storecfg, $vmid) = @_;
4263
4264     my $conf = load_config($vmid);
4265
4266     my $defaults = load_defaults();
4267
4268     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4269
4270     return join(' ', @$cmd);
4271 }
4272
4273 sub vm_reset {
4274     my ($vmid, $skiplock) = @_;
4275
4276     lock_config($vmid, sub {
4277
4278         my $conf = load_config($vmid);
4279
4280         check_lock($conf) if !$skiplock;
4281
4282         vm_mon_cmd($vmid, "system_reset");
4283     });
4284 }
4285
4286 sub get_vm_volumes {
4287     my ($conf) = @_;
4288
4289     my $vollist = [];
4290     foreach_volid($conf, sub {
4291         my ($volid, $is_cdrom) = @_;
4292
4293         return if $volid =~ m|^/|;
4294
4295         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4296         return if !$sid;
4297
4298         push @$vollist, $volid;
4299     });
4300
4301     return $vollist;
4302 }
4303
4304 sub vm_stop_cleanup {
4305     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4306
4307     eval {
4308         fairsched_rmnod($vmid); # try to destroy group
4309
4310         if (!$keepActive) {
4311             my $vollist = get_vm_volumes($conf);
4312             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4313         }
4314
4315         foreach my $ext (qw(mon qmp pid vnc qga)) {
4316             unlink "/var/run/qemu-server/${vmid}.$ext";
4317         }
4318
4319         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4320     };
4321     warn $@ if $@; # avoid errors - just warn
4322 }
4323
4324 # Note: use $nockeck to skip tests if VM configuration file exists.
4325 # We need that when migration VMs to other nodes (files already moved)
4326 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4327 sub vm_stop {
4328     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4329
4330     $force = 1 if !defined($force) && !$shutdown;
4331
4332     if ($migratedfrom){
4333         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4334         kill 15, $pid if $pid;
4335         my $conf = load_config($vmid, $migratedfrom);
4336         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4337         return;
4338     }
4339
4340     lock_config($vmid, sub {
4341
4342         my $pid = check_running($vmid, $nocheck);
4343         return if !$pid;
4344
4345         my $conf;
4346         if (!$nocheck) {
4347             $conf = load_config($vmid);
4348             check_lock($conf) if !$skiplock;
4349             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4350                 my $opts = parse_startup($conf->{startup});
4351                 $timeout = $opts->{down} if $opts->{down};
4352             }
4353         }
4354
4355         $timeout = 60 if !defined($timeout);
4356
4357         eval {
4358             if ($shutdown) {
4359                 if (defined($conf) && $conf->{agent}) {
4360                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4361                 } else {
4362                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4363                 }
4364             } else {
4365                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4366             }
4367         };
4368         my $err = $@;
4369
4370         if (!$err) {
4371             my $count = 0;
4372             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4373                 $count++;
4374                 sleep 1;
4375             }
4376
4377             if ($count >= $timeout) {
4378                 if ($force) {
4379                     warn "VM still running - terminating now with SIGTERM\n";
4380                     kill 15, $pid;
4381                 } else {
4382                     die "VM quit/powerdown failed - got timeout\n";
4383                 }
4384             } else {
4385                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4386                 return;
4387             }
4388         } else {
4389             if ($force) {
4390                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4391                 kill 15, $pid;
4392             } else {
4393                 die "VM quit/powerdown failed\n";
4394             }
4395         }
4396
4397         # wait again
4398         $timeout = 10;
4399
4400         my $count = 0;
4401         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4402             $count++;
4403             sleep 1;
4404         }
4405
4406         if ($count >= $timeout) {
4407             warn "VM still running - terminating now with SIGKILL\n";
4408             kill 9, $pid;
4409             sleep 1;
4410         }
4411
4412         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4413    });
4414 }
4415
4416 sub vm_suspend {
4417     my ($vmid, $skiplock) = @_;
4418
4419     lock_config($vmid, sub {
4420
4421         my $conf = load_config($vmid);
4422
4423         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4424
4425         vm_mon_cmd($vmid, "stop");
4426     });
4427 }
4428
4429 sub vm_resume {
4430     my ($vmid, $skiplock) = @_;
4431
4432     lock_config($vmid, sub {
4433
4434         my $conf = load_config($vmid);
4435
4436         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4437
4438         vm_mon_cmd($vmid, "cont");
4439     });
4440 }
4441
4442 sub vm_sendkey {
4443     my ($vmid, $skiplock, $key) = @_;
4444
4445     lock_config($vmid, sub {
4446
4447         my $conf = load_config($vmid);
4448
4449         # there is no qmp command, so we use the human monitor command
4450         vm_human_monitor_command($vmid, "sendkey $key");
4451     });
4452 }
4453
4454 sub vm_destroy {
4455     my ($storecfg, $vmid, $skiplock) = @_;
4456
4457     lock_config($vmid, sub {
4458
4459         my $conf = load_config($vmid);
4460
4461         check_lock($conf) if !$skiplock;
4462
4463         if (!check_running($vmid)) {
4464             fairsched_rmnod($vmid); # try to destroy group
4465             destroy_vm($storecfg, $vmid);
4466         } else {
4467             die "VM $vmid is running - destroy failed\n";
4468         }
4469     });
4470 }
4471
4472 # pci helpers
4473
4474 sub file_write {
4475     my ($filename, $buf) = @_;
4476
4477     my $fh = IO::File->new($filename, "w");
4478     return undef if !$fh;
4479
4480     my $res = print $fh $buf;
4481
4482     $fh->close();
4483
4484     return $res;
4485 }
4486
4487 sub pci_device_info {
4488     my ($name) = @_;
4489
4490     my $res;
4491
4492     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4493     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4494
4495     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4496     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4497
4498     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4499     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4500
4501     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4502     return undef if !defined($product) || $product !~ s/^0x//;
4503
4504     $res = {
4505         name => $name,
4506         vendor => $vendor,
4507         product => $product,
4508         domain => $domain,
4509         bus => $bus,
4510         slot => $slot,
4511         func => $func,
4512         irq => $irq,
4513         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4514     };
4515
4516     return $res;
4517 }
4518
4519 sub pci_dev_reset {
4520     my ($dev) = @_;
4521
4522     my $name = $dev->{name};
4523
4524     my $fn = "$pcisysfs/devices/$name/reset";
4525
4526     return file_write($fn, "1");
4527 }
4528
4529 sub pci_dev_bind_to_stub {
4530     my ($dev) = @_;
4531
4532     my $name = $dev->{name};
4533
4534     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4535     return 1 if -d $testdir;
4536
4537     my $data = "$dev->{vendor} $dev->{product}";
4538     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4539
4540     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4541     if (!file_write($fn, $name)) {
4542         return undef if -f $fn;
4543     }
4544
4545     $fn = "$pcisysfs/drivers/pci-stub/bind";
4546     if (! -d $testdir) {
4547         return undef if !file_write($fn, $name);
4548     }
4549
4550     return -d $testdir;
4551 }
4552
4553 sub pci_dev_bind_to_vfio {
4554     my ($dev) = @_;
4555
4556     my $name = $dev->{name};
4557
4558     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4559
4560     if (!-d $vfio_basedir) {
4561         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4562     }
4563     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4564
4565     my $testdir = "$vfio_basedir/$name";
4566     return 1 if -d $testdir;
4567
4568     my $data = "$dev->{vendor} $dev->{product}";
4569     return undef if !file_write("$vfio_basedir/new_id", $data);
4570
4571     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4572     if (!file_write($fn, $name)) {
4573         return undef if -f $fn;
4574     }
4575
4576     $fn = "$vfio_basedir/bind";
4577     if (! -d $testdir) {
4578         return undef if !file_write($fn, $name);
4579     }
4580
4581     return -d $testdir;
4582 }
4583
4584 sub pci_dev_group_bind_to_vfio {
4585     my ($pciid) = @_;
4586
4587     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4588
4589     if (!-d $vfio_basedir) {
4590         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4591     }
4592     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4593
4594     # get IOMMU group devices
4595     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4596       my @devs = grep /^0000:/, readdir($D);
4597     closedir($D);
4598
4599     foreach my $pciid (@devs) {
4600         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4601         my $info = pci_device_info($1);
4602         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4603     }
4604
4605     return 1;
4606 }
4607
4608 sub print_pci_addr {
4609     my ($id, $bridges) = @_;
4610
4611     my $res = '';
4612     my $devices = {
4613         piix3 => { bus => 0, addr => 1 },
4614         #addr2 : first videocard
4615         balloon0 => { bus => 0, addr => 3 },
4616         watchdog => { bus => 0, addr => 4 },
4617         scsihw0 => { bus => 0, addr => 5 },
4618         scsihw1 => { bus => 0, addr => 6 },
4619         ahci0 => { bus => 0, addr => 7 },
4620         qga0 => { bus => 0, addr => 8 },
4621         spice => { bus => 0, addr => 9 },
4622         virtio0 => { bus => 0, addr => 10 },
4623         virtio1 => { bus => 0, addr => 11 },
4624         virtio2 => { bus => 0, addr => 12 },
4625         virtio3 => { bus => 0, addr => 13 },
4626         virtio4 => { bus => 0, addr => 14 },
4627         virtio5 => { bus => 0, addr => 15 },
4628         hostpci0 => { bus => 0, addr => 16 },
4629         hostpci1 => { bus => 0, addr => 17 },
4630         net0 => { bus => 0, addr => 18 },
4631         net1 => { bus => 0, addr => 19 },
4632         net2 => { bus => 0, addr => 20 },
4633         net3 => { bus => 0, addr => 21 },
4634         net4 => { bus => 0, addr => 22 },
4635         net5 => { bus => 0, addr => 23 },
4636         vga1 => { bus => 0, addr => 24 },
4637         vga2 => { bus => 0, addr => 25 },
4638         vga3 => { bus => 0, addr => 26 },
4639         hostpci2 => { bus => 0, addr => 27 },
4640         hostpci3 => { bus => 0, addr => 28 },
4641         #addr29 : usb-host (pve-usb.cfg)
4642         'pci.1' => { bus => 0, addr => 30 },
4643         'pci.2' => { bus => 0, addr => 31 },
4644         'net6' => { bus => 1, addr => 1 },
4645         'net7' => { bus => 1, addr => 2 },
4646         'net8' => { bus => 1, addr => 3 },
4647         'net9' => { bus => 1, addr => 4 },
4648         'net10' => { bus => 1, addr => 5 },
4649         'net11' => { bus => 1, addr => 6 },
4650         'net12' => { bus => 1, addr => 7 },
4651         'net13' => { bus => 1, addr => 8 },
4652         'net14' => { bus => 1, addr => 9 },
4653         'net15' => { bus => 1, addr => 10 },
4654         'net16' => { bus => 1, addr => 11 },
4655         'net17' => { bus => 1, addr => 12 },
4656         'net18' => { bus => 1, addr => 13 },
4657         'net19' => { bus => 1, addr => 14 },
4658         'net20' => { bus => 1, addr => 15 },
4659         'net21' => { bus => 1, addr => 16 },
4660         'net22' => { bus => 1, addr => 17 },
4661         'net23' => { bus => 1, addr => 18 },
4662         'net24' => { bus => 1, addr => 19 },
4663         'net25' => { bus => 1, addr => 20 },
4664         'net26' => { bus => 1, addr => 21 },
4665         'net27' => { bus => 1, addr => 22 },
4666         'net28' => { bus => 1, addr => 23 },
4667         'net29' => { bus => 1, addr => 24 },
4668         'net30' => { bus => 1, addr => 25 },
4669         'net31' => { bus => 1, addr => 26 },
4670         'virtio6' => { bus => 2, addr => 1 },
4671         'virtio7' => { bus => 2, addr => 2 },
4672         'virtio8' => { bus => 2, addr => 3 },
4673         'virtio9' => { bus => 2, addr => 4 },
4674         'virtio10' => { bus => 2, addr => 5 },
4675         'virtio11' => { bus => 2, addr => 6 },
4676         'virtio12' => { bus => 2, addr => 7 },
4677         'virtio13' => { bus => 2, addr => 8 },
4678         'virtio14' => { bus => 2, addr => 9 },
4679         'virtio15' => { bus => 2, addr => 10 },
4680     };
4681
4682     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4683            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4684            my $bus = $devices->{$id}->{bus};
4685            $res = ",bus=pci.$bus,addr=$addr";
4686            $bridges->{$bus} = 1 if $bridges;
4687     }
4688     return $res;
4689
4690 }
4691
4692 sub print_pcie_addr {
4693     my ($id) = @_;
4694
4695     my $res = '';
4696     my $devices = {
4697         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4698         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4699         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4700         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4701     };
4702
4703     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4704            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4705            my $bus = $devices->{$id}->{bus};
4706            $res = ",bus=$bus,addr=$addr";
4707     }
4708     return $res;
4709
4710 }
4711
4712 # vzdump restore implementaion
4713
4714 sub tar_archive_read_firstfile {
4715     my $archive = shift;
4716
4717     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4718
4719     # try to detect archive type first
4720     my $pid = open (TMP, "tar tf '$archive'|") ||
4721         die "unable to open file '$archive'\n";
4722     my $firstfile = <TMP>;
4723     kill 15, $pid;
4724     close TMP;
4725
4726     die "ERROR: archive contaions no data\n" if !$firstfile;
4727     chomp $firstfile;
4728
4729     return $firstfile;
4730 }
4731
4732 sub tar_restore_cleanup {
4733     my ($storecfg, $statfile) = @_;
4734
4735     print STDERR "starting cleanup\n";
4736
4737     if (my $fd = IO::File->new($statfile, "r")) {
4738         while (defined(my $line = <$fd>)) {
4739             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4740                 my $volid = $2;
4741                 eval {
4742                     if ($volid =~ m|^/|) {
4743                         unlink $volid || die 'unlink failed\n';
4744                     } else {
4745                         PVE::Storage::vdisk_free($storecfg, $volid);
4746                     }
4747                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4748                 };
4749                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4750             } else {
4751                 print STDERR "unable to parse line in statfile - $line";
4752             }
4753         }
4754         $fd->close();
4755     }
4756 }
4757
4758 sub restore_archive {
4759     my ($archive, $vmid, $user, $opts) = @_;
4760
4761     my $format = $opts->{format};
4762     my $comp;
4763
4764     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4765         $format = 'tar' if !$format;
4766         $comp = 'gzip';
4767     } elsif ($archive =~ m/\.tar$/) {
4768         $format = 'tar' if !$format;
4769     } elsif ($archive =~ m/.tar.lzo$/) {
4770         $format = 'tar' if !$format;
4771         $comp = 'lzop';
4772     } elsif ($archive =~ m/\.vma$/) {
4773         $format = 'vma' if !$format;
4774     } elsif ($archive =~ m/\.vma\.gz$/) {
4775         $format = 'vma' if !$format;
4776         $comp = 'gzip';
4777     } elsif ($archive =~ m/\.vma\.lzo$/) {
4778         $format = 'vma' if !$format;
4779         $comp = 'lzop';
4780     } else {
4781         $format = 'vma' if !$format; # default
4782     }
4783
4784     # try to detect archive format
4785     if ($format eq 'tar') {
4786         return restore_tar_archive($archive, $vmid, $user, $opts);
4787     } else {
4788         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4789     }
4790 }
4791
4792 sub restore_update_config_line {
4793     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4794
4795     return if $line =~ m/^\#qmdump\#/;
4796     return if $line =~ m/^\#vzdump\#/;
4797     return if $line =~ m/^lock:/;
4798     return if $line =~ m/^unused\d+:/;
4799     return if $line =~ m/^parent:/;
4800     return if $line =~ m/^template:/; # restored VM is never a template
4801
4802     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4803         # try to convert old 1.X settings
4804         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4805         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4806             my ($model, $macaddr) = split(/\=/, $devconfig);
4807             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4808             my $net = {
4809                 model => $model,
4810                 bridge => "vmbr$ind",
4811                 macaddr => $macaddr,
4812             };
4813             my $netstr = print_net($net);
4814
4815             print $outfd "net$cookie->{netcount}: $netstr\n";
4816             $cookie->{netcount}++;
4817         }
4818     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4819         my ($id, $netstr) = ($1, $2);
4820         my $net = parse_net($netstr);
4821         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4822         $netstr = print_net($net);
4823         print $outfd "$id: $netstr\n";
4824     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4825         my $virtdev = $1;
4826         my $value = $3;
4827         if ($line =~ m/backup=no/) {
4828             print $outfd "#$line";
4829         } elsif ($virtdev && $map->{$virtdev}) {
4830             my $di = parse_drive($virtdev, $value);
4831             delete $di->{format}; # format can change on restore
4832             $di->{file} = $map->{$virtdev};
4833             $value = print_drive($vmid, $di);
4834             print $outfd "$virtdev: $value\n";
4835         } else {
4836             print $outfd $line;
4837         }
4838     } else {
4839         print $outfd $line;
4840     }
4841 }
4842
4843 sub scan_volids {
4844     my ($cfg, $vmid) = @_;
4845
4846     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4847
4848     my $volid_hash = {};
4849     foreach my $storeid (keys %$info) {
4850         foreach my $item (@{$info->{$storeid}}) {
4851             next if !($item->{volid} && $item->{size});
4852             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4853             $volid_hash->{$item->{volid}} = $item;
4854         }
4855     }
4856
4857     return $volid_hash;
4858 }
4859
4860 sub get_used_paths {
4861     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4862
4863     my $used_path = {};
4864
4865     my $scan_config = sub {
4866         my ($cref, $snapname) = @_;
4867
4868         foreach my $key (keys %$cref) {
4869             my $value = $cref->{$key};
4870             if (valid_drivename($key)) {
4871                 next if $skip_drive && $key eq $skip_drive;
4872                 my $drive = parse_drive($key, $value);
4873                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4874                 if ($drive->{file} =~ m!^/!) {
4875                     $used_path->{$drive->{file}}++; # = 1;
4876                 } else {
4877                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4878                     next if !$storeid;
4879                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4880                     next if !$scfg;
4881                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4882                     $used_path->{$path}++; # = 1;
4883                 }
4884             }
4885         }
4886     };
4887
4888     &$scan_config($conf);
4889
4890     undef $skip_drive;
4891
4892     if ($scan_snapshots) {
4893         foreach my $snapname (keys %{$conf->{snapshots}}) {
4894             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4895         }
4896     }
4897
4898     return $used_path;
4899 }
4900
4901 sub update_disksize {
4902     my ($vmid, $conf, $volid_hash) = @_;
4903
4904     my $changes;
4905
4906     my $used = {};
4907
4908     # Note: it is allowed to define multiple storages with same path (alias), so
4909     # we need to check both 'volid' and real 'path' (two different volid can point
4910     # to the same path).
4911
4912     my $usedpath = {};
4913
4914     # update size info
4915     foreach my $opt (keys %$conf) {
4916         if (valid_drivename($opt)) {
4917             my $drive = parse_drive($opt, $conf->{$opt});
4918             my $volid = $drive->{file};
4919             next if !$volid;
4920
4921             $used->{$volid} = 1;
4922             if ($volid_hash->{$volid} &&
4923                 (my $path = $volid_hash->{$volid}->{path})) {
4924                 $usedpath->{$path} = 1;
4925             }
4926
4927             next if drive_is_cdrom($drive);
4928             next if !$volid_hash->{$volid};
4929
4930             $drive->{size} = $volid_hash->{$volid}->{size};
4931             my $new = print_drive($vmid, $drive);
4932             if ($new ne $conf->{$opt}) {
4933                 $changes = 1;
4934                 $conf->{$opt} = $new;
4935             }
4936         }
4937     }
4938
4939     # remove 'unusedX' entry if volume is used
4940     foreach my $opt (keys %$conf) {
4941         next if $opt !~ m/^unused\d+$/;
4942         my $volid = $conf->{$opt};
4943         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
4944         if ($used->{$volid} || ($path && $usedpath->{$path})) {
4945             $changes = 1;
4946             delete $conf->{$opt};
4947         }
4948     }
4949
4950     foreach my $volid (sort keys %$volid_hash) {
4951         next if $volid =~ m/vm-$vmid-state-/;
4952         next if $used->{$volid};
4953         my $path = $volid_hash->{$volid}->{path};
4954         next if !$path; # just to be sure
4955         next if $usedpath->{$path};
4956         $changes = 1;
4957         add_unused_volume($conf, $volid);
4958         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
4959     }
4960
4961     return $changes;
4962 }
4963
4964 sub rescan {
4965     my ($vmid, $nolock) = @_;
4966
4967     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
4968
4969     my $volid_hash = scan_volids($cfg, $vmid);
4970
4971     my $updatefn =  sub {
4972         my ($vmid) = @_;
4973
4974         my $conf = load_config($vmid);
4975
4976         check_lock($conf);
4977
4978         my $vm_volids = {};
4979         foreach my $volid (keys %$volid_hash) {
4980             my $info = $volid_hash->{$volid};
4981             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
4982         }
4983
4984         my $changes = update_disksize($vmid, $conf, $vm_volids);
4985
4986         update_config_nolock($vmid, $conf, 1) if $changes;
4987     };
4988
4989     if (defined($vmid)) {
4990         if ($nolock) {
4991             &$updatefn($vmid);
4992         } else {
4993             lock_config($vmid, $updatefn, $vmid);
4994         }
4995     } else {
4996         my $vmlist = config_list();
4997         foreach my $vmid (keys %$vmlist) {
4998             if ($nolock) {
4999                 &$updatefn($vmid);
5000             } else {
5001                 lock_config($vmid, $updatefn, $vmid);
5002             }
5003         }
5004     }
5005 }
5006
5007 sub restore_vma_archive {
5008     my ($archive, $vmid, $user, $opts, $comp) = @_;
5009
5010     my $input = $archive eq '-' ? "<&STDIN" : undef;
5011     my $readfrom = $archive;
5012
5013     my $uncomp = '';
5014     if ($comp) {
5015         $readfrom = '-';
5016         my $qarchive = PVE::Tools::shellquote($archive);
5017         if ($comp eq 'gzip') {
5018             $uncomp = "zcat $qarchive|";
5019         } elsif ($comp eq 'lzop') {
5020             $uncomp = "lzop -d -c $qarchive|";
5021         } else {
5022             die "unknown compression method '$comp'\n";
5023         }
5024
5025     }
5026
5027     my $tmpdir = "/var/tmp/vzdumptmp$$";
5028     rmtree $tmpdir;
5029
5030     # disable interrupts (always do cleanups)
5031     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5032         warn "got interrupt - ignored\n";
5033     };
5034
5035     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5036     POSIX::mkfifo($mapfifo, 0600);
5037     my $fifofh;
5038
5039     my $openfifo = sub {
5040         open($fifofh, '>', $mapfifo) || die $!;
5041     };
5042
5043     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5044
5045     my $oldtimeout;
5046     my $timeout = 5;
5047
5048     my $devinfo = {};
5049
5050     my $rpcenv = PVE::RPCEnvironment::get();
5051
5052     my $conffile = config_file($vmid);
5053     my $tmpfn = "$conffile.$$.tmp";
5054
5055     # Note: $oldconf is undef if VM does not exists
5056     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5057
5058     my $print_devmap = sub {
5059         my $virtdev_hash = {};
5060
5061         my $cfgfn = "$tmpdir/qemu-server.conf";
5062
5063         # we can read the config - that is already extracted
5064         my $fh = IO::File->new($cfgfn, "r") ||
5065             "unable to read qemu-server.conf - $!\n";
5066
5067         while (defined(my $line = <$fh>)) {
5068             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5069                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5070                 die "archive does not contain data for drive '$virtdev'\n"
5071                     if !$devinfo->{$devname};
5072                 if (defined($opts->{storage})) {
5073                     $storeid = $opts->{storage} || 'local';
5074                 } elsif (!$storeid) {
5075                     $storeid = 'local';
5076                 }
5077                 $format = 'raw' if !$format;
5078                 $devinfo->{$devname}->{devname} = $devname;
5079                 $devinfo->{$devname}->{virtdev} = $virtdev;
5080                 $devinfo->{$devname}->{format} = $format;
5081                 $devinfo->{$devname}->{storeid} = $storeid;
5082
5083                 # check permission on storage
5084                 my $pool = $opts->{pool}; # todo: do we need that?
5085                 if ($user ne 'root@pam') {
5086                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5087                 }
5088
5089                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5090             }
5091         }
5092
5093         foreach my $devname (keys %$devinfo) {
5094             die "found no device mapping information for device '$devname'\n"
5095                 if !$devinfo->{$devname}->{virtdev};
5096         }
5097
5098         my $cfg = cfs_read_file('storage.cfg');
5099
5100         # create empty/temp config
5101         if ($oldconf) {
5102             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5103             foreach_drive($oldconf, sub {
5104                 my ($ds, $drive) = @_;
5105
5106                 return if drive_is_cdrom($drive);
5107
5108                 my $volid = $drive->{file};
5109
5110                 return if !$volid || $volid =~ m|^/|;
5111
5112                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5113                 return if !$path || !$owner || ($owner != $vmid);
5114
5115                 # Note: only delete disk we want to restore
5116                 # other volumes will become unused
5117                 if ($virtdev_hash->{$ds}) {
5118                     PVE::Storage::vdisk_free($cfg, $volid);
5119                 }
5120             });
5121         }
5122
5123         my $map = {};
5124         foreach my $virtdev (sort keys %$virtdev_hash) {
5125             my $d = $virtdev_hash->{$virtdev};
5126             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5127             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5128
5129             # test if requested format is supported
5130             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5131             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5132             $d->{format} = $defFormat if !$supported;
5133
5134             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5135                                                   $d->{format}, undef, $alloc_size);
5136             print STDERR "new volume ID is '$volid'\n";
5137             $d->{volid} = $volid;
5138             my $path = PVE::Storage::path($cfg, $volid);
5139
5140             my $write_zeros = 1;
5141             # fixme: what other storages types initialize volumes with zero?
5142             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5143                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5144                 $write_zeros = 0;
5145             }
5146
5147             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5148
5149             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5150             $map->{$virtdev} = $volid;
5151         }
5152
5153         $fh->seek(0, 0) || die "seek failed - $!\n";
5154
5155         my $outfd = new IO::File ($tmpfn, "w") ||
5156             die "unable to write config for VM $vmid\n";
5157
5158         my $cookie = { netcount => 0 };
5159         while (defined(my $line = <$fh>)) {
5160             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5161         }
5162
5163         $fh->close();
5164         $outfd->close();
5165     };
5166
5167     eval {
5168         # enable interrupts
5169         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5170             die "interrupted by signal\n";
5171         };
5172         local $SIG{ALRM} = sub { die "got timeout\n"; };
5173
5174         $oldtimeout = alarm($timeout);
5175
5176         my $parser = sub {
5177             my $line = shift;
5178
5179             print "$line\n";
5180
5181             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5182                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5183                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5184             } elsif ($line =~ m/^CTIME: /) {
5185                 # we correctly received the vma config, so we can disable
5186                 # the timeout now for disk allocation (set to 10 minutes, so
5187                 # that we always timeout if something goes wrong)
5188                 alarm(600);
5189                 &$print_devmap();
5190                 print $fifofh "done\n";
5191                 my $tmp = $oldtimeout || 0;
5192                 $oldtimeout = undef;
5193                 alarm($tmp);
5194                 close($fifofh);
5195             }
5196         };
5197
5198         print "restore vma archive: $cmd\n";
5199         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5200     };
5201     my $err = $@;
5202
5203     alarm($oldtimeout) if $oldtimeout;
5204
5205     unlink $mapfifo;
5206
5207     if ($err) {
5208         rmtree $tmpdir;
5209         unlink $tmpfn;
5210
5211         my $cfg = cfs_read_file('storage.cfg');
5212         foreach my $devname (keys %$devinfo) {
5213             my $volid = $devinfo->{$devname}->{volid};
5214             next if !$volid;
5215             eval {
5216                 if ($volid =~ m|^/|) {
5217                     unlink $volid || die 'unlink failed\n';
5218                 } else {
5219                     PVE::Storage::vdisk_free($cfg, $volid);
5220                 }
5221                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5222             };
5223             print STDERR "unable to cleanup '$volid' - $@" if $@;
5224         }
5225         die $err;
5226     }
5227
5228     rmtree $tmpdir;
5229
5230     rename($tmpfn, $conffile) ||
5231         die "unable to commit configuration file '$conffile'\n";
5232
5233     PVE::Cluster::cfs_update(); # make sure we read new file
5234
5235     eval { rescan($vmid, 1); };
5236     warn $@ if $@;
5237 }
5238
5239 sub restore_tar_archive {
5240     my ($archive, $vmid, $user, $opts) = @_;
5241
5242     if ($archive ne '-') {
5243         my $firstfile = tar_archive_read_firstfile($archive);
5244         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5245             if $firstfile ne 'qemu-server.conf';
5246     }
5247
5248     my $storecfg = cfs_read_file('storage.cfg');
5249
5250     # destroy existing data - keep empty config
5251     my $vmcfgfn = config_file($vmid);
5252     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5253
5254     my $tocmd = "/usr/lib/qemu-server/qmextract";
5255
5256     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5257     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5258     $tocmd .= ' --prealloc' if $opts->{prealloc};
5259     $tocmd .= ' --info' if $opts->{info};
5260
5261     # tar option "xf" does not autodetect compression when read from STDIN,
5262     # so we pipe to zcat
5263     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5264         PVE::Tools::shellquote("--to-command=$tocmd");
5265
5266     my $tmpdir = "/var/tmp/vzdumptmp$$";
5267     mkpath $tmpdir;
5268
5269     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5270     local $ENV{VZDUMP_VMID} = $vmid;
5271     local $ENV{VZDUMP_USER} = $user;
5272
5273     my $conffile = config_file($vmid);
5274     my $tmpfn = "$conffile.$$.tmp";
5275
5276     # disable interrupts (always do cleanups)
5277     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5278         print STDERR "got interrupt - ignored\n";
5279     };
5280
5281     eval {
5282         # enable interrupts
5283         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5284             die "interrupted by signal\n";
5285         };
5286
5287         if ($archive eq '-') {
5288             print "extracting archive from STDIN\n";
5289             run_command($cmd, input => "<&STDIN");
5290         } else {
5291             print "extracting archive '$archive'\n";
5292             run_command($cmd);
5293         }
5294
5295         return if $opts->{info};
5296
5297         # read new mapping
5298         my $map = {};
5299         my $statfile = "$tmpdir/qmrestore.stat";
5300         if (my $fd = IO::File->new($statfile, "r")) {
5301             while (defined (my $line = <$fd>)) {
5302                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5303                     $map->{$1} = $2 if $1;
5304                 } else {
5305                     print STDERR "unable to parse line in statfile - $line\n";
5306                 }
5307             }
5308             $fd->close();
5309         }
5310
5311         my $confsrc = "$tmpdir/qemu-server.conf";
5312
5313         my $srcfd = new IO::File($confsrc, "r") ||
5314             die "unable to open file '$confsrc'\n";
5315
5316         my $outfd = new IO::File ($tmpfn, "w") ||
5317             die "unable to write config for VM $vmid\n";
5318
5319         my $cookie = { netcount => 0 };
5320         while (defined (my $line = <$srcfd>)) {
5321             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5322         }
5323
5324         $srcfd->close();
5325         $outfd->close();
5326     };
5327     my $err = $@;
5328
5329     if ($err) {
5330
5331         unlink $tmpfn;
5332
5333         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5334
5335         die $err;
5336     }
5337
5338     rmtree $tmpdir;
5339
5340     rename $tmpfn, $conffile ||
5341         die "unable to commit configuration file '$conffile'\n";
5342
5343     PVE::Cluster::cfs_update(); # make sure we read new file
5344
5345     eval { rescan($vmid, 1); };
5346     warn $@ if $@;
5347 };
5348
5349
5350 # Internal snapshots
5351
5352 # NOTE: Snapshot create/delete involves several non-atomic
5353 # action, and can take a long time.
5354 # So we try to avoid locking the file and use 'lock' variable
5355 # inside the config file instead.
5356
5357 my $snapshot_copy_config = sub {
5358     my ($source, $dest) = @_;
5359
5360     foreach my $k (keys %$source) {
5361         next if $k eq 'snapshots';
5362         next if $k eq 'snapstate';
5363         next if $k eq 'snaptime';
5364         next if $k eq 'vmstate';
5365         next if $k eq 'lock';
5366         next if $k eq 'digest';
5367         next if $k eq 'description';
5368         next if $k =~ m/^unused\d+$/;
5369
5370         $dest->{$k} = $source->{$k};
5371     }
5372 };
5373
5374 my $snapshot_apply_config = sub {
5375     my ($conf, $snap) = @_;
5376
5377     # copy snapshot list
5378     my $newconf = {
5379         snapshots => $conf->{snapshots},
5380     };
5381
5382     # keep description and list of unused disks
5383     foreach my $k (keys %$conf) {
5384         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5385         $newconf->{$k} = $conf->{$k};
5386     }
5387
5388     &$snapshot_copy_config($snap, $newconf);
5389
5390     return $newconf;
5391 };
5392
5393 sub foreach_writable_storage {
5394     my ($conf, $func) = @_;
5395
5396     my $sidhash = {};
5397
5398     foreach my $ds (keys %$conf) {
5399         next if !valid_drivename($ds);
5400
5401         my $drive = parse_drive($ds, $conf->{$ds});
5402         next if !$drive;
5403         next if drive_is_cdrom($drive);
5404
5405         my $volid = $drive->{file};
5406
5407         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5408         $sidhash->{$sid} = $sid if $sid;
5409     }
5410
5411     foreach my $sid (sort keys %$sidhash) {
5412         &$func($sid);
5413     }
5414 }
5415
5416 my $alloc_vmstate_volid = sub {
5417     my ($storecfg, $vmid, $conf, $snapname) = @_;
5418
5419     # Note: we try to be smart when selecting a $target storage
5420
5421     my $target;
5422
5423     # search shared storage first
5424     foreach_writable_storage($conf, sub {
5425         my ($sid) = @_;
5426         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5427         return if !$scfg->{shared};
5428
5429         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5430     });
5431
5432     if (!$target) {
5433         # now search local storage
5434         foreach_writable_storage($conf, sub {
5435             my ($sid) = @_;
5436             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5437             return if $scfg->{shared};
5438
5439             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5440         });
5441     }
5442
5443     $target = 'local' if !$target;
5444
5445     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5446     # we abort live save after $conf->{memory}, so we need at max twice that space
5447     my $size = $conf->{memory}*2 + $driver_state_size;
5448
5449     my $name = "vm-$vmid-state-$snapname";
5450     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5451     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5452     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5453
5454     return $volid;
5455 };
5456
5457 my $snapshot_prepare = sub {
5458     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5459
5460     my $snap;
5461
5462     my $updatefn =  sub {
5463
5464         my $conf = load_config($vmid);
5465
5466         die "you can't take a snapshot if it's a template\n"
5467             if is_template($conf);
5468
5469         check_lock($conf);
5470
5471         $conf->{lock} = 'snapshot';
5472
5473         die "snapshot name '$snapname' already used\n"
5474             if defined($conf->{snapshots}->{$snapname});
5475
5476         my $storecfg = PVE::Storage::config();
5477         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5478
5479         $snap = $conf->{snapshots}->{$snapname} = {};
5480
5481         if ($save_vmstate && check_running($vmid)) {
5482             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5483         }
5484
5485         &$snapshot_copy_config($conf, $snap);
5486
5487         $snap->{snapstate} = "prepare";
5488         $snap->{snaptime} = time();
5489         $snap->{description} = $comment if $comment;
5490
5491         # always overwrite machine if we save vmstate. This makes sure we
5492         # can restore it later using correct machine type
5493         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5494
5495         update_config_nolock($vmid, $conf, 1);
5496     };
5497
5498     lock_config($vmid, $updatefn);
5499
5500     return $snap;
5501 };
5502
5503 my $snapshot_commit = sub {
5504     my ($vmid, $snapname) = @_;
5505
5506     my $updatefn = sub {
5507
5508         my $conf = load_config($vmid);
5509
5510         die "missing snapshot lock\n"
5511             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5512
5513         my $has_machine_config = defined($conf->{machine});
5514
5515         my $snap = $conf->{snapshots}->{$snapname};
5516
5517         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5518
5519         die "wrong snapshot state\n"
5520             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5521
5522         delete $snap->{snapstate};
5523         delete $conf->{lock};
5524
5525         my $newconf = &$snapshot_apply_config($conf, $snap);
5526
5527         delete $newconf->{machine} if !$has_machine_config;
5528
5529         $newconf->{parent} = $snapname;
5530
5531         update_config_nolock($vmid, $newconf, 1);
5532     };
5533
5534     lock_config($vmid, $updatefn);
5535 };
5536
5537 sub snapshot_rollback {
5538     my ($vmid, $snapname) = @_;
5539
5540     my $snap;
5541
5542     my $prepare = 1;
5543
5544     my $storecfg = PVE::Storage::config();
5545
5546     my $updatefn = sub {
5547
5548         my $conf = load_config($vmid);
5549
5550         die "you can't rollback if vm is a template\n" if is_template($conf);
5551
5552         $snap = $conf->{snapshots}->{$snapname};
5553
5554         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5555
5556         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5557             if $snap->{snapstate};
5558
5559         if ($prepare) {
5560             check_lock($conf);
5561             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5562         }
5563
5564         die "unable to rollback vm $vmid: vm is running\n"
5565             if check_running($vmid);
5566
5567         if ($prepare) {
5568             $conf->{lock} = 'rollback';
5569         } else {
5570             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5571             delete $conf->{lock};
5572         }
5573
5574         my $forcemachine;
5575
5576         if (!$prepare) {
5577             my $has_machine_config = defined($conf->{machine});
5578
5579             # copy snapshot config to current config
5580             $conf = &$snapshot_apply_config($conf, $snap);
5581             $conf->{parent} = $snapname;
5582
5583             # Note: old code did not store 'machine', so we try to be smart
5584             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5585             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5586             # we remove the 'machine' configuration if not explicitly specified
5587             # in the original config.
5588             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5589         }
5590
5591         update_config_nolock($vmid, $conf, 1);
5592
5593         if (!$prepare && $snap->{vmstate}) {
5594             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5595             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5596         }
5597     };
5598
5599     lock_config($vmid, $updatefn);
5600
5601     foreach_drive($snap, sub {
5602         my ($ds, $drive) = @_;
5603
5604         return if drive_is_cdrom($drive);
5605
5606         my $volid = $drive->{file};
5607         my $device = "drive-$ds";
5608
5609         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5610     });
5611
5612     $prepare = 0;
5613     lock_config($vmid, $updatefn);
5614 }
5615
5616 my $savevm_wait = sub {
5617     my ($vmid) = @_;
5618
5619     for(;;) {
5620         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5621         if (!$stat->{status}) {
5622             die "savevm not active\n";
5623         } elsif ($stat->{status} eq 'active') {
5624             sleep(1);
5625             next;
5626         } elsif ($stat->{status} eq 'completed') {
5627             last;
5628         } else {
5629             die "query-savevm returned status '$stat->{status}'\n";
5630         }
5631     }
5632 };
5633
5634 sub snapshot_create {
5635     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5636
5637     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5638
5639     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5640
5641     my $config = load_config($vmid);
5642
5643     my $running = check_running($vmid);
5644
5645     my $freezefs = $running && $config->{agent};
5646     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5647
5648     my $drivehash = {};
5649
5650     if ($freezefs) {
5651         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5652         warn "guest-fsfreeze-freeze problems - $@" if $@;
5653     }
5654
5655     eval {
5656         # create internal snapshots of all drives
5657
5658         my $storecfg = PVE::Storage::config();
5659
5660         if ($running) {
5661             if ($snap->{vmstate}) {
5662                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5663                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5664                 &$savevm_wait($vmid);
5665             } else {
5666                 vm_mon_cmd($vmid, "savevm-start");
5667             }
5668         };
5669
5670         foreach_drive($snap, sub {
5671             my ($ds, $drive) = @_;
5672
5673             return if drive_is_cdrom($drive);
5674
5675             my $volid = $drive->{file};
5676             my $device = "drive-$ds";
5677
5678             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5679             $drivehash->{$ds} = 1;
5680        });
5681     };
5682     my $err = $@;
5683
5684     if ($running) {
5685         eval { vm_mon_cmd($vmid, "savevm-end")  };
5686         warn $@ if $@;
5687
5688         if ($freezefs) {
5689             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5690             warn "guest-fsfreeze-thaw problems - $@" if $@;
5691         }
5692
5693         # savevm-end is async, we need to wait
5694         for (;;) {
5695             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5696             if (!$stat->{bytes}) {
5697                 last;
5698             } else {
5699                 print "savevm not yet finished\n";
5700                 sleep(1);
5701                 next;
5702             }
5703         }
5704     }
5705
5706     if ($err) {
5707         warn "snapshot create failed: starting cleanup\n";
5708         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5709         warn $@ if $@;
5710         die $err;
5711     }
5712
5713     &$snapshot_commit($vmid, $snapname);
5714 }
5715
5716 # Note: $drivehash is only set when called from snapshot_create.
5717 sub snapshot_delete {
5718     my ($vmid, $snapname, $force, $drivehash) = @_;
5719
5720     my $prepare = 1;
5721
5722     my $snap;
5723     my $unused = [];
5724
5725     my $unlink_parent = sub {
5726         my ($confref, $new_parent) = @_;
5727
5728         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5729             if ($new_parent) {
5730                 $confref->{parent} = $new_parent;
5731             } else {
5732                 delete $confref->{parent};
5733             }
5734         }
5735     };
5736
5737     my $updatefn =  sub {
5738         my ($remove_drive) = @_;
5739
5740         my $conf = load_config($vmid);
5741
5742         if (!$drivehash) {
5743             check_lock($conf);
5744             die "you can't delete a snapshot if vm is a template\n"
5745                 if is_template($conf);
5746         }
5747
5748         $snap = $conf->{snapshots}->{$snapname};
5749
5750         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5751
5752         # remove parent refs
5753         if (!$prepare) {
5754             &$unlink_parent($conf, $snap->{parent});
5755             foreach my $sn (keys %{$conf->{snapshots}}) {
5756                 next if $sn eq $snapname;
5757                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5758             }
5759         }
5760
5761         if ($remove_drive) {
5762             if ($remove_drive eq 'vmstate') {
5763                 delete $snap->{$remove_drive};
5764             } else {
5765                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5766                 my $volid = $drive->{file};
5767                 delete $snap->{$remove_drive};
5768                 add_unused_volume($conf, $volid);
5769             }
5770         }
5771
5772         if ($prepare) {
5773             $snap->{snapstate} = 'delete';
5774         } else {
5775             delete $conf->{snapshots}->{$snapname};
5776             delete $conf->{lock} if $drivehash;
5777             foreach my $volid (@$unused) {
5778                 add_unused_volume($conf, $volid);
5779             }
5780         }
5781
5782         update_config_nolock($vmid, $conf, 1);
5783     };
5784
5785     lock_config($vmid, $updatefn);
5786
5787     # now remove vmstate file
5788
5789     my $storecfg = PVE::Storage::config();
5790
5791     if ($snap->{vmstate}) {
5792         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5793         if (my $err = $@) {
5794             die $err if !$force;
5795             warn $err;
5796         }
5797         # save changes (remove vmstate from snapshot)
5798         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5799     };
5800
5801     # now remove all internal snapshots
5802     foreach_drive($snap, sub {
5803         my ($ds, $drive) = @_;
5804
5805         return if drive_is_cdrom($drive);
5806
5807         my $volid = $drive->{file};
5808         my $device = "drive-$ds";
5809
5810         if (!$drivehash || $drivehash->{$ds}) {
5811             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5812             if (my $err = $@) {
5813                 die $err if !$force;
5814                 warn $err;
5815             }
5816         }
5817
5818         # save changes (remove drive fron snapshot)
5819         lock_config($vmid, $updatefn, $ds) if !$force;
5820         push @$unused, $volid;
5821     });
5822
5823     # now cleanup config
5824     $prepare = 0;
5825     lock_config($vmid, $updatefn);
5826 }
5827
5828 sub has_feature {
5829     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5830
5831     my $err;
5832     foreach_drive($conf, sub {
5833         my ($ds, $drive) = @_;
5834
5835         return if drive_is_cdrom($drive);
5836         my $volid = $drive->{file};
5837         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5838     });
5839
5840     return $err ? 0 : 1;
5841 }
5842
5843 sub template_create {
5844     my ($vmid, $conf, $disk) = @_;
5845
5846     my $storecfg = PVE::Storage::config();
5847
5848     foreach_drive($conf, sub {
5849         my ($ds, $drive) = @_;
5850
5851         return if drive_is_cdrom($drive);
5852         return if $disk && $ds ne $disk;
5853
5854         my $volid = $drive->{file};
5855         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5856
5857         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5858         $drive->{file} = $voliddst;
5859         $conf->{$ds} = print_drive($vmid, $drive);
5860         update_config_nolock($vmid, $conf, 1);
5861     });
5862 }
5863
5864 sub is_template {
5865     my ($conf) = @_;
5866
5867     return 1 if defined $conf->{template} && $conf->{template} == 1;
5868 }
5869
5870 sub qemu_img_convert {
5871     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5872
5873     my $storecfg = PVE::Storage::config();
5874     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5875     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5876
5877     if ($src_storeid && $dst_storeid) {
5878         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5879         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5880
5881         my $src_format = qemu_img_format($src_scfg, $src_volname);
5882         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5883
5884         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5885         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5886
5887         my $cmd = [];
5888         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5889         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5890         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5891
5892         my $parser = sub {
5893             my $line = shift;
5894             if($line =~ m/\((\S+)\/100\%\)/){
5895                 my $percent = $1;
5896                 my $transferred = int($size * $percent / 100);
5897                 my $remaining = $size - $transferred;
5898
5899                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5900             }
5901
5902         };
5903
5904         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5905         my $err = $@;
5906         die "copy failed: $err" if $err;
5907     }
5908 }
5909
5910 sub qemu_img_format {
5911     my ($scfg, $volname) = @_;
5912
5913     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
5914         return $1;
5915     } elsif ($scfg->{type} eq 'iscsi') {
5916         return "host_device";
5917     } else {
5918         return "raw";
5919     }
5920 }
5921
5922 sub qemu_drive_mirror {
5923     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
5924
5925     my $count = 0;
5926     my $old_len = 0;
5927     my $frozen = undef;
5928     my $maxwait = 120;
5929
5930     my $storecfg = PVE::Storage::config();
5931     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5932
5933     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5934
5935     my $format;
5936     if ($dst_volname =~ m/\.(raw|qcow2)$/){
5937         $format = $1;
5938     }
5939
5940     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5941
5942     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
5943     $opts->{format} = $format if $format;
5944
5945     #fixme : sometime drive-mirror timeout, but works fine after.
5946     # (I have see the problem with big volume > 200GB), so we need to eval
5947     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
5948     # ignore errors here
5949
5950     eval {
5951         while (1) {
5952             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5953             my $stat = @$stats[0];
5954             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5955             die "error job is not mirroring" if $stat->{type} ne "mirror";
5956
5957             my $busy = $stat->{busy};
5958
5959             if (my $total = $stat->{len}) {
5960                 my $transferred = $stat->{offset} || 0;
5961                 my $remaining = $total - $transferred;
5962                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5963
5964                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy\n";
5965             }
5966
5967             if ($stat->{len} == $stat->{offset}) {
5968                 if ($busy eq 'false') {
5969
5970                     last if $vmiddst != $vmid;
5971
5972                     # try to switch the disk if source and destination are on the same guest
5973                     eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5974                     last if !$@;
5975                     die $@ if $@ !~ m/cannot be completed/;
5976                 }
5977
5978                 if ($count > $maxwait) {
5979                     # if too much writes to disk occurs at the end of migration
5980                     #the disk needs to be freezed to be able to complete the migration
5981                     vm_suspend($vmid,1);
5982                     $frozen = 1;
5983                 }
5984                 $count ++
5985             }
5986             $old_len = $stat->{offset};
5987             sleep 1;
5988         }
5989
5990         vm_resume($vmid, 1) if $frozen;
5991
5992     };
5993     my $err = $@;
5994
5995     my $cancel_job = sub {
5996         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5997         while (1) {
5998             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5999             my $stat = @$stats[0];
6000             last if !$stat;
6001             sleep 1;
6002         }
6003     };
6004
6005     if ($err) {
6006         eval { &$cancel_job(); };
6007         die "mirroring error: $err";
6008     }
6009
6010     if ($vmiddst != $vmid) {
6011         # if we clone a disk for a new target vm, we don't switch the disk
6012         &$cancel_job(); # so we call block-job-cancel
6013     }
6014 }
6015
6016 sub clone_disk {
6017     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6018         $newvmid, $storage, $format, $full, $newvollist) = @_;
6019
6020     my $newvolid;
6021
6022     if (!$full) {
6023         print "create linked clone of drive $drivename ($drive->{file})\n";
6024         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6025         push @$newvollist, $newvolid;
6026     } else {
6027         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6028         $storeid = $storage if $storage;
6029
6030         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6031         if (!$format) {
6032             $format = $drive->{format} || $defFormat;
6033         }
6034
6035         # test if requested format is supported - else use default
6036         my $supported = grep { $_ eq $format } @$validFormats;
6037         $format = $defFormat if !$supported;
6038
6039         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6040
6041         print "create full clone of drive $drivename ($drive->{file})\n";
6042         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6043         push @$newvollist, $newvolid;
6044
6045         if (!$running || $snapname) {
6046             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6047         } else {
6048             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6049         }
6050     }
6051
6052     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6053
6054     my $disk = $drive;
6055     $disk->{format} = undef;
6056     $disk->{file} = $newvolid;
6057     $disk->{size} = $size;
6058
6059     return $disk;
6060 }
6061
6062 # this only works if VM is running
6063 sub get_current_qemu_machine {
6064     my ($vmid) = @_;
6065
6066     my $cmd = { execute => 'query-machines', arguments => {} };
6067     my $res = vm_qmp_command($vmid, $cmd);
6068
6069     my ($current, $default);
6070     foreach my $e (@$res) {
6071         $default = $e->{name} if $e->{'is-default'};
6072         $current = $e->{name} if $e->{'is-current'};
6073     }
6074
6075     # fallback to the default machine if current is not supported by qemu
6076     return $current || $default || 'pc';
6077 }
6078
6079 sub lspci {
6080
6081     my $devices = {};
6082
6083     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6084             my (undef, $id, $function) = @_;
6085             my $res = { id => $id, function => $function};
6086             push @{$devices->{$id}}, $res;
6087     });
6088
6089     return $devices;
6090 }
6091
6092 1;