PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     onboot => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Specifies whether a VM will be started during system bootup.",
 159         default => 0,
 160     },
 161     autostart => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Automatic restart after crash (currently ignored).",
 165         default => 0,
 166     },
 167     hotplug => {
 168         optional => 1,
 169         type => 'string', format => 'pve-hotplug-features',
 170         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 171         default => 'network,disk,usb',
 172     },
 173     reboot => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 177         default => 1,
 178     },
 179     lock => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Lock/unlock the VM.",
 183         enum => [qw(migrate backup snapshot rollback)],
 184     },
 185     cpulimit => {
 186         optional => 1,
 187         type => 'integer',
 188         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 189         minimum => 0,
 190         default => 0,
 191     },
 192     cpuunits => {
 193         optional => 1,
 194         type => 'integer',
 195         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 196         minimum => 0,
 197         maximum => 500000,
 198         default => 1000,
 199     },
 200     memory => {
 201         optional => 1,
 202         type => 'integer',
 203         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 204         minimum => 16,
 205         default => 512,
 206     },
 207     balloon => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 211         minimum => 0,
 212     },
 213     shares => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 217         minimum => 0,
 218         maximum => 50000,
 219         default => 1000,
 220     },
 221     keyboard => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 225         enum => PVE::Tools::kvmkeymaplist(),
 226         default => 'en-us',
 227     },
 228     name => {
 229         optional => 1,
 230         type => 'string', format => 'dns-name',
 231         description => "Set a name for the VM. Only used on the configuration web interface.",
 232     },
 233     scsihw => {
 234         optional => 1,
 235         type => 'string',
 236         description => "scsi controller model",
 237         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 238         default => 'lsi',
 239     },
 240     description => {
 241         optional => 1,
 242         type => 'string',
 243         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 244     },
 245     ostype => {
 246         optional => 1,
 247         type => 'string',
 248         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 249         description => <<EODESC,
 250 Used to enable special optimization/features for specific
 251 operating systems:
 252
 253 other  => unspecified OS
 254 wxp    => Microsoft Windows XP
 255 w2k    => Microsoft Windows 2000
 256 w2k3   => Microsoft Windows 2003
 257 w2k8   => Microsoft Windows 2008
 258 wvista => Microsoft Windows Vista
 259 win7   => Microsoft Windows 7
 260 win8   => Microsoft Windows 8/2012
 261 l24    => Linux 2.4 Kernel
 262 l26    => Linux 2.6/3.X Kernel
 263 solaris => solaris/opensolaris/openindiania kernel
 264
 265 other|l24|l26|solaris                       ... no special behaviour
 266 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 267 EODESC
 268     },
 269     boot => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 273         pattern => '[acdn]{1,4}',
 274         default => 'cdn',
 275     },
 276     bootdisk => {
 277         optional => 1,
 278         type => 'string', format => 'pve-qm-bootdisk',
 279         description => "Enable booting from specified disk.",
 280         pattern => '(ide|sata|scsi|virtio)\d+',
 281     },
 282     smp => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "The number of CPUs. Please use option -sockets instead.",
 286         minimum => 1,
 287         default => 1,
 288     },
 289     sockets => {
 290         optional => 1,
 291         type => 'integer',
 292         description => "The number of CPU sockets.",
 293         minimum => 1,
 294         default => 1,
 295     },
 296     cores => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of cores per socket.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     numa => {
 304         optional => 1,
 305         type => 'boolean',
 306         description => "Enable/disable Numa.",
 307         default => 0,
 308     },
 309     vcpus => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "Number of hotplugged vcpus.",
 313         minimum => 1,
 314         default => 0,
 315     },
 316     acpi => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable ACPI.",
 320         default => 1,
 321     },
 322     agent => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable Qemu GuestAgent.",
 326         default => 0,
 327     },
 328     kvm => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable KVM hardware virtualization.",
 332         default => 1,
 333     },
 334     tdf => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Enable/disable time drift fix.",
 338         default => 0,
 339     },
 340     localtime => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 344     },
 345     freeze => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 349     },
 350     vga => {
 351         optional => 1,
 352         type => 'string',
 353         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 354         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 355     },
 356     watchdog => {
 357         optional => 1,
 358         type => 'string', format => 'pve-qm-watchdog',
 359         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 360         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 361     },
 362     startdate => {
 363         optional => 1,
 364         type => 'string',
 365         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 366         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 367         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 368         default => 'now',
 369     },
 370     startup => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-startup',
 373         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 374         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 375     },
 376     template => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable Template.",
 380         default => 0,
 381     },
 382     args => {
 383         optional => 1,
 384         type => 'string',
 385         description => <<EODESCR,
 386 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 387
 388 args: -no-reboot -no-hpet
 389 EODESCR
 390     },
 391     tablet => {
 392         optional => 1,
 393         type => 'boolean',
 394         default => 1,
 395         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 396     },
 397     migrate_speed => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 401         minimum => 0,
 402         default => 0,
 403     },
 404     migrate_downtime => {
 405         optional => 1,
 406         type => 'number',
 407         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 408         minimum => 0,
 409         default => 0.1,
 410     },
 411     cdrom => {
 412         optional => 1,
 413         type => 'string', format => 'pve-qm-drive',
 414         typetext => 'volume',
 415         description => "This is an alias for option -ide2",
 416     },
 417     cpu => {
 418         optional => 1,
 419         description => "Emulated CPU type.",
 420         type => 'string',
 421         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 422         default => 'kvm64',
 423     },
 424     parent => get_standard_option('pve-snapshot-name', {
 425         optional => 1,
 426         description => "Parent snapshot name. This is used internally, and should not be modified.",
 427     }),
 428     snaptime => {
 429         optional => 1,
 430         description => "Timestamp for snapshots.",
 431         type => 'integer',
 432         minimum => 0,
 433     },
 434     vmstate => {
 435         optional => 1,
 436         type => 'string', format => 'pve-volume-id',
 437         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 438     },
 439     machine => {
 440         description => "Specific the Qemu machine type.",
 441         type => 'string',
 442         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 443         maxLength => 40,
 444         optional => 1,
 445     },
 446     smbios1 => {
 447         description => "Specify SMBIOS type 1 fields.",
 448         type => 'string', format => 'pve-qm-smbios1',
 449         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 450         maxLength => 256,
 451         optional => 1,
 452     },
 453 };
 454
 455 # what about other qemu settings ?
 456 #cpu => 'string',
 457 #machine => 'string',
 458 #fda => 'file',
 459 #fdb => 'file',
 460 #mtdblock => 'file',
 461 #sd => 'file',
 462 #pflash => 'file',
 463 #snapshot => 'bool',
 464 #bootp => 'file',
 465 ##tftp => 'dir',
 466 ##smb => 'dir',
 467 #kernel => 'file',
 468 #append => 'string',
 469 #initrd => 'file',
 470 ##soundhw => 'string',
 471
 472 while (my ($k, $v) = each %$confdesc) {
 473     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 474 }
 475
 476 my $MAX_IDE_DISKS = 4;
 477 my $MAX_SCSI_DISKS = 14;
 478 my $MAX_VIRTIO_DISKS = 16;
 479 my $MAX_SATA_DISKS = 6;
 480 my $MAX_USB_DEVICES = 5;
 481 my $MAX_NETS = 32;
 482 my $MAX_UNUSED_DISKS = 8;
 483 my $MAX_HOSTPCI_DEVICES = 4;
 484 my $MAX_SERIAL_PORTS = 4;
 485 my $MAX_PARALLEL_PORTS = 3;
 486 my $MAX_NUMA = 8;
 487 my $MAX_MEM = 4194304;
 488 my $STATICMEM = 1024;
 489
 490 my $numadesc = {
 491     optional => 1,
 492     type => 'string', format => 'pve-qm-numanode',
 493     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 494     description => "numa topology",
 495 };
 496 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 497
 498 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 499     $confdesc->{"numa$i"} = $numadesc;
 500 }
 501
 502 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 503                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 504                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 505 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 506
 507 my $netdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-net',
 510     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 511     description => <<EODESCR,
 512 Specify network devices.
 513
 514 MODEL is one of: $nic_model_list_txt
 515
 516 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 517 automatically generated if not specified.
 518
 519 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 520
 521 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 522
 523 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 524
 525 10.0.2.2   Gateway
 526 10.0.2.3   DNS Server
 527 10.0.2.4   SMB Server
 528
 529 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 530
 531 EODESCR
 532 };
 533 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 534
 535 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 536     $confdesc->{"net$i"} = $netdesc;
 537 }
 538
 539 my $drivename_hash;
 540
 541 my $idedesc = {
 542     optional => 1,
 543     type => 'string', format => 'pve-qm-drive',
 544     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 545     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 546 };
 547 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 548
 549 my $scsidesc = {
 550     optional => 1,
 551     type => 'string', format => 'pve-qm-drive',
 552     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 553     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 554 };
 555 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 556
 557 my $satadesc = {
 558     optional => 1,
 559     type => 'string', format => 'pve-qm-drive',
 560     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 561     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 562 };
 563 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 564
 565 my $virtiodesc = {
 566     optional => 1,
 567     type => 'string', format => 'pve-qm-drive',
 568     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 569     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 570 };
 571 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 572
 573 my $usbdesc = {
 574     optional => 1,
 575     type => 'string', format => 'pve-qm-usb-device',
 576     typetext => 'host=HOSTUSBDEVICE|spice',
 577     description => <<EODESCR,
 578 Configure an USB device (n is 0 to 4). This can be used to
 579 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 580
 581 'bus-port(.port)*' (decimal numbers) or
 582 'vendor_id:product_id' (hexadeciaml numbers)
 583
 584 You can use the 'lsusb -t' command to list existing usb devices.
 585
 586 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 587
 588 The value 'spice' can be used to add a usb redirection devices for spice.
 589
 590 EODESCR
 591 };
 592 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 593
 594 my $hostpcidesc = {
 595         optional => 1,
 596         type => 'string', format => 'pve-qm-hostpci',
 597         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 598         description => <<EODESCR,
 599 Map host pci devices. HOSTPCIDEVICE syntax is:
 600
 601 'bus:dev.func' (hexadecimal numbers)
 602
 603 You can us the 'lspci' command to list existing pci devices.
 604
 605 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 606
 607 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 608
 609 Experimental: user reported problems with this option.
 610 EODESCR
 611 };
 612 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 613
 614 my $serialdesc = {
 615         optional => 1,
 616         type => 'string',
 617         pattern => '(/dev/.+|socket)',
 618         description =>  <<EODESCR,
 619 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 620
 621 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 622
 623 Experimental: user reported problems with this option.
 624 EODESCR
 625 };
 626
 627 my $paralleldesc= {
 628         optional => 1,
 629         type => 'string',
 630         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 631         description =>  <<EODESCR,
 632 Map host parallel devices (n is 0 to 2).
 633
 634 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 635
 636 Experimental: user reported problems with this option.
 637 EODESCR
 638 };
 639
 640 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 641     $confdesc->{"parallel$i"} = $paralleldesc;
 642 }
 643
 644 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 645     $confdesc->{"serial$i"} = $serialdesc;
 646 }
 647
 648 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 649     $confdesc->{"hostpci$i"} = $hostpcidesc;
 650 }
 651
 652 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 653     $drivename_hash->{"ide$i"} = 1;
 654     $confdesc->{"ide$i"} = $idedesc;
 655 }
 656
 657 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 658     $drivename_hash->{"sata$i"} = 1;
 659     $confdesc->{"sata$i"} = $satadesc;
 660 }
 661
 662 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 663     $drivename_hash->{"scsi$i"} = 1;
 664     $confdesc->{"scsi$i"} = $scsidesc ;
 665 }
 666
 667 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 668     $drivename_hash->{"virtio$i"} = 1;
 669     $confdesc->{"virtio$i"} = $virtiodesc;
 670 }
 671
 672 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 673     $confdesc->{"usb$i"} = $usbdesc;
 674 }
 675
 676 my $unuseddesc = {
 677     optional => 1,
 678     type => 'string', format => 'pve-volume-id',
 679     description => "Reference to unused volumes.",
 680 };
 681
 682 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 683     $confdesc->{"unused$i"} = $unuseddesc;
 684 }
 685
 686 my $kvm_api_version = 0;
 687
 688 sub kvm_version {
 689
 690     return $kvm_api_version if $kvm_api_version;
 691
 692     my $fh = IO::File->new("</dev/kvm") ||
 693         return 0;
 694
 695     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 696         $kvm_api_version = $v;
 697     }
 698
 699     $fh->close();
 700
 701     return  $kvm_api_version;
 702 }
 703
 704 my $kvm_user_version;
 705
 706 sub kvm_user_version {
 707
 708     return $kvm_user_version if $kvm_user_version;
 709
 710     $kvm_user_version = 'unknown';
 711
 712     my $tmp = `kvm -help 2>/dev/null`;
 713
 714     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 715         $kvm_user_version = $2;
 716     }
 717
 718     return $kvm_user_version;
 719
 720 }
 721
 722 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 723
 724 sub disknames {
 725     # order is important - used to autoselect boot disk
 726     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 727             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 728             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 729             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 730 }
 731
 732 sub valid_drivename {
 733     my $dev = shift;
 734
 735     return defined($drivename_hash->{$dev});
 736 }
 737
 738 sub option_exists {
 739     my $key = shift;
 740     return defined($confdesc->{$key});
 741 }
 742
 743 sub nic_models {
 744     return $nic_model_list;
 745 }
 746
 747 sub os_list_description {
 748
 749     return {
 750         other => 'Other',
 751         wxp => 'Windows XP',
 752         w2k => 'Windows 2000',
 753         w2k3 =>, 'Windows 2003',
 754         w2k8 => 'Windows 2008',
 755         wvista => 'Windows Vista',
 756         win7 => 'Windows 7',
 757         win8 => 'Windows 8/2012',
 758         l24 => 'Linux 2.4',
 759         l26 => 'Linux 2.6',
 760     };
 761 }
 762
 763 my $cdrom_path;
 764
 765 sub get_cdrom_path {
 766
 767     return  $cdrom_path if $cdrom_path;
 768
 769     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 770     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 771     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 772 }
 773
 774 sub get_iso_path {
 775     my ($storecfg, $vmid, $cdrom) = @_;
 776
 777     if ($cdrom eq 'cdrom') {
 778         return get_cdrom_path();
 779     } elsif ($cdrom eq 'none') {
 780         return '';
 781     } elsif ($cdrom =~ m|^/|) {
 782         return $cdrom;
 783     } else {
 784         return PVE::Storage::path($storecfg, $cdrom);
 785     }
 786 }
 787
 788 # try to convert old style file names to volume IDs
 789 sub filename_to_volume_id {
 790     my ($vmid, $file, $media) = @_;
 791
 792     if (!($file eq 'none' || $file eq 'cdrom' ||
 793           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 794
 795         return undef if $file =~ m|/|;
 796
 797         if ($media && $media eq 'cdrom') {
 798             $file = "local:iso/$file";
 799         } else {
 800             $file = "local:$vmid/$file";
 801         }
 802     }
 803
 804     return $file;
 805 }
 806
 807 sub verify_media_type {
 808     my ($opt, $vtype, $media) = @_;
 809
 810     return if !$media;
 811
 812     my $etype;
 813     if ($media eq 'disk') {
 814         $etype = 'images';
 815     } elsif ($media eq 'cdrom') {
 816         $etype = 'iso';
 817     } else {
 818         die "internal error";
 819     }
 820
 821     return if ($vtype eq $etype);
 822
 823     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 824 }
 825
 826 sub cleanup_drive_path {
 827     my ($opt, $storecfg, $drive) = @_;
 828
 829     # try to convert filesystem paths to volume IDs
 830
 831     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 832         ($drive->{file} !~ m|^/dev/.+|) &&
 833         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 834         ($drive->{file} !~ m/^\d+$/)) {
 835         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 836         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 837         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 838         verify_media_type($opt, $vtype, $drive->{media});
 839         $drive->{file} = $volid;
 840     }
 841
 842     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 843 }
 844
 845 sub create_conf_nolock {
 846     my ($vmid, $settings) = @_;
 847
 848     my $filename = config_file($vmid);
 849
 850     die "configuration file '$filename' already exists\n" if -f $filename;
 851
 852     my $defaults = load_defaults();
 853
 854     $settings->{name} = "vm$vmid" if !$settings->{name};
 855     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 856
 857     my $data = '';
 858     foreach my $opt (keys %$settings) {
 859         next if !$confdesc->{$opt};
 860
 861         my $value = $settings->{$opt};
 862         next if !$value;
 863
 864         $data .= "$opt: $value\n";
 865     }
 866
 867     PVE::Tools::file_set_contents($filename, $data);
 868 }
 869
 870 sub parse_hotplug_features {
 871     my ($data) = @_;
 872
 873     my $res = {};
 874
 875     return $res if $data eq '0';
 876
 877     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 878
 879     foreach my $feature (PVE::Tools::split_list($data)) {
 880         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 881             $res->{$1} = 1;
 882         } else {
 883             warn "ignoring unknown hotplug feature '$feature'\n";
 884         }
 885     }
 886     return $res;
 887 }
 888
 889 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 890 sub pve_verify_hotplug_features {
 891     my ($value, $noerr) = @_;
 892
 893     return $value if parse_hotplug_features($value);
 894
 895     return undef if $noerr;
 896
 897     die "unable to parse hotplug option\n";
 898 }
 899
 900 my $parse_size = sub {
 901     my ($value) = @_;
 902
 903     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 904     my ($size, $unit) = ($1, $3);
 905     if ($unit) {
 906         if ($unit eq 'K') {
 907             $size = $size * 1024;
 908         } elsif ($unit eq 'M') {
 909             $size = $size * 1024 * 1024;
 910         } elsif ($unit eq 'G') {
 911             $size = $size * 1024 * 1024 * 1024;
 912         }
 913     }
 914     return int($size);
 915 };
 916
 917 my $format_size = sub {
 918     my ($size) = @_;
 919
 920     $size = int($size);
 921
 922     my $kb = int($size/1024);
 923     return $size if $kb*1024 != $size;
 924
 925     my $mb = int($kb/1024);
 926     return "${kb}K" if $mb*1024 != $kb;
 927
 928     my $gb = int($mb/1024);
 929     return "${mb}M" if $gb*1024 != $mb;
 930
 931     return "${gb}G";
 932 };
 933
 934 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 935 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 936 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 937 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 938
 939 sub parse_drive {
 940     my ($key, $data) = @_;
 941
 942     my $res = {};
 943
 944     # $key may be undefined - used to verify JSON parameters
 945     if (!defined($key)) {
 946         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 947         $res->{index} = 0;
 948     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 949         $res->{interface} = $1;
 950         $res->{index} = $2;
 951     } else {
 952         return undef;
 953     }
 954
 955     foreach my $p (split (/,/, $data)) {
 956         next if $p =~ m/^\s*$/;
 957
 958         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread)=(.+)$/) {
 959             my ($k, $v) = ($1, $2);
 960
 961             $k = 'file' if $k eq 'volume';
 962
 963             return undef if defined $res->{$k};
 964
 965             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 966                 return undef if !$v || $v !~ m/^\d+/;
 967                 $k = "m$k";
 968                 $v = sprintf("%.3f", $v / (1024*1024));
 969             }
 970             $res->{$k} = $v;
 971         } else {
 972             if (!$res->{file} && $p !~ m/=/) {
 973                 $res->{file} = $p;
 974             } else {
 975                 return undef;
 976             }
 977         }
 978     }
 979
 980     return undef if !$res->{file};
 981
 982     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 983         $res->{format} = $1;
 984     }
 985
 986     return undef if $res->{cache} &&
 987         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 988     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 989     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 990     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 991     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 992     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 993     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 994     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 995     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 996     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 997     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 998     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 999     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
1000     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
1001
1002     return undef if $res->{mbps_rd} && $res->{mbps};
1003     return undef if $res->{mbps_wr} && $res->{mbps};
1004
1005     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1006     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1007     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1008     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1009     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1010     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1011
1012     return undef if $res->{iops_rd} && $res->{iops};
1013     return undef if $res->{iops_wr} && $res->{iops};
1014
1015
1016     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1017     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1018     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1019     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1020     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1021     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1022
1023
1024     if ($res->{size}) {
1025         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1026     }
1027
1028     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1029         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1030         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1031         return undef if $res->{interface} eq 'virtio';
1032     }
1033
1034     # rerror does not work with scsi drives
1035     if ($res->{rerror}) {
1036         return undef if $res->{interface} eq 'scsi';
1037     }
1038
1039     return $res;
1040 }
1041
1042 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1043
1044 sub print_drive {
1045     my ($vmid, $drive) = @_;
1046
1047     my $opts = '';
1048     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread') {
1049         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1050     }
1051
1052     if ($drive->{size}) {
1053         $opts .= ",size=" . &$format_size($drive->{size});
1054     }
1055
1056     return "$drive->{file}$opts";
1057 }
1058
1059 sub scsi_inquiry {
1060     my($fh, $noerr) = @_;
1061
1062     my $SG_IO = 0x2285;
1063     my $SG_GET_VERSION_NUM = 0x2282;
1064
1065     my $versionbuf = "\x00" x 8;
1066     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1067     if (!$ret) {
1068         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1069         return undef;
1070     }
1071     my $version = unpack("I", $versionbuf);
1072     if ($version < 30000) {
1073         die "scsi generic interface too old\n"  if !$noerr;
1074         return undef;
1075     }
1076
1077     my $buf = "\x00" x 36;
1078     my $sensebuf = "\x00" x 8;
1079     my $cmd = pack("C x3 C x1", 0x12, 36);
1080
1081     # see /usr/include/scsi/sg.h
1082     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1083
1084     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1085                       length($sensebuf), 0, length($buf), $buf,
1086                       $cmd, $sensebuf, 6000);
1087
1088     $ret = ioctl($fh, $SG_IO, $packet);
1089     if (!$ret) {
1090         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1091         return undef;
1092     }
1093
1094     my @res = unpack($sg_io_hdr_t, $packet);
1095     if ($res[17] || $res[18]) {
1096         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1097         return undef;
1098     }
1099
1100     my $res = {};
1101     (my $byte0, my $byte1, $res->{vendor},
1102      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1103
1104     $res->{removable} = $byte1 & 128 ? 1 : 0;
1105     $res->{type} = $byte0 & 31;
1106
1107     return $res;
1108 }
1109
1110 sub path_is_scsi {
1111     my ($path) = @_;
1112
1113     my $fh = IO::File->new("+<$path") || return undef;
1114     my $res = scsi_inquiry($fh, 1);
1115     close($fh);
1116
1117     return $res;
1118 }
1119
1120 sub machine_type_is_q35 {
1121     my ($conf) = @_;
1122
1123     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1124 }
1125
1126 sub print_tabletdevice_full {
1127     my ($conf) = @_;
1128
1129     my $q35 = machine_type_is_q35($conf);
1130
1131     # we use uhci for old VMs because tablet driver was buggy in older qemu
1132     my $usbbus = $q35 ? "ehci" : "uhci";
1133
1134     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1135 }
1136
1137 sub print_drivedevice_full {
1138     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1139
1140     my $device = '';
1141     my $maxdev = 0;
1142
1143     if ($drive->{interface} eq 'virtio') {
1144         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1145         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1146         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1147     } elsif ($drive->{interface} eq 'scsi') {
1148
1149         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1150         my $unit = $drive->{index} % $maxdev;
1151         my $devicetype = 'hd';
1152         my $path = '';
1153         if (drive_is_cdrom($drive)) {
1154               $devicetype = 'cd';
1155         } else {
1156               if ($drive->{file} =~ m|^/|) {
1157                   $path = $drive->{file};
1158               } else {
1159                   $path = PVE::Storage::path($storecfg, $drive->{file});
1160               }
1161
1162               if($path =~ m/^iscsi\:\/\//){
1163                   $devicetype = 'generic';
1164               } else {
1165                   if (my $info = path_is_scsi($path)) {
1166                       if ($info->{type} == 0) {
1167                           $devicetype = 'block';
1168                       } elsif ($info->{type} == 1) { # tape
1169                           $devicetype = 'generic';
1170                       }
1171                   }
1172               }
1173          }
1174
1175         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1176             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1177         } else {
1178             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1179         }
1180
1181     } elsif ($drive->{interface} eq 'ide'){
1182         $maxdev = 2;
1183         my $controller = int($drive->{index} / $maxdev);
1184         my $unit = $drive->{index} % $maxdev;
1185         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1186
1187         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1188     } elsif ($drive->{interface} eq 'sata'){
1189         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1190         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1191         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1192     } elsif ($drive->{interface} eq 'usb') {
1193         die "implement me";
1194         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1195     } else {
1196         die "unsupported interface type";
1197     }
1198
1199     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1200
1201     return $device;
1202 }
1203
1204 sub get_initiator_name {
1205     my $initiator;
1206
1207     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1208     while (defined(my $line = <$fh>)) {
1209         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1210         $initiator = $1;
1211         last;
1212     }
1213     $fh->close();
1214
1215     return $initiator;
1216 }
1217
1218 sub print_drive_full {
1219     my ($storecfg, $vmid, $drive) = @_;
1220
1221     my $opts = '';
1222     foreach my $o (@qemu_drive_options) {
1223         next if $o eq 'bootindex';
1224         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1225     }
1226
1227     foreach my $o (qw(bps bps_rd bps_wr)) {
1228         my $v = $drive->{"m$o"};
1229         $opts .= ",$o=" . int($v*1024*1024) if $v;
1230     }
1231
1232     # use linux-aio by default (qemu default is threads)
1233     $opts .= ",aio=native" if !$drive->{aio};
1234
1235     my $path;
1236     my $volid = $drive->{file};
1237     if (drive_is_cdrom($drive)) {
1238         $path = get_iso_path($storecfg, $vmid, $volid);
1239     } else {
1240         if ($volid =~ m|^/|) {
1241             $path = $volid;
1242         } else {
1243             $path = PVE::Storage::path($storecfg, $volid);
1244         }
1245     }
1246
1247     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1248
1249     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1250     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1251
1252     my $pathinfo = $path ? "file=$path," : '';
1253
1254     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1255 }
1256
1257 sub print_netdevice_full {
1258     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1259
1260     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1261
1262     my $device = $net->{model};
1263     if ($net->{model} eq 'virtio') {
1264          $device = 'virtio-net-pci';
1265      };
1266
1267     my $pciaddr = print_pci_addr("$netid", $bridges);
1268     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1269     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1270         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1271         my $vectors = $net->{queues} * 2 + 2;
1272         $tmpstr .= ",vectors=$vectors,mq=on";
1273     }
1274     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1275     return $tmpstr;
1276 }
1277
1278 sub print_netdev_full {
1279     my ($vmid, $conf, $net, $netid) = @_;
1280
1281     my $i = '';
1282     if ($netid =~ m/^net(\d+)$/) {
1283         $i = int($1);
1284     }
1285
1286     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1287
1288     my $ifname = "tap${vmid}i$i";
1289
1290     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1291     die "interface name '$ifname' is too long (max 15 character)\n"
1292         if length($ifname) >= 16;
1293
1294     my $vhostparam = '';
1295     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1296
1297     my $vmname = $conf->{name} || "vm$vmid";
1298
1299     my $netdev = "";
1300
1301     if ($net->{bridge}) {
1302         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1303     } else {
1304         $netdev = "type=user,id=$netid,hostname=$vmname";
1305     }
1306
1307     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1308
1309     return $netdev;
1310 }
1311
1312 sub drive_is_cdrom {
1313     my ($drive) = @_;
1314
1315     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1316
1317 }
1318
1319 sub parse_numa {
1320     my ($data) = @_;
1321
1322     my $res = {};
1323
1324     foreach my $kvp (split(/,/, $data)) {
1325
1326         if ($kvp =~ m/^memory=(\S+)$/) {
1327             $res->{memory} = $1;
1328         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1329             $res->{policy} = $1;
1330         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1331             $res->{cpus}->{start} = $1;
1332             $res->{cpus}->{end} = $3;
1333         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1334             $res->{hostnodes}->{start} = $1;
1335             $res->{hostnodes}->{end} = $3;
1336         } else {
1337             return undef;
1338         }
1339     }
1340
1341     return $res;
1342 }
1343
1344 sub parse_hostpci {
1345     my ($value) = @_;
1346
1347     return undef if !$value;
1348
1349
1350     my @list = split(/,/, $value);
1351     my $found;
1352
1353     my $res = {};
1354     foreach my $kv (@list) {
1355
1356         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1357             $found = 1;
1358             if(defined($4)){
1359                 push @{$res->{pciid}}, { id => $2 , function => $4};
1360
1361             }else{
1362                 my $pcidevices = lspci($2);
1363                 $res->{pciid} = $pcidevices->{$2};
1364             }
1365         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1366             $res->{driver} = $1;
1367         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1368             $res->{rombar} = $1;
1369         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1370             $res->{'x-vga'} = $1;
1371         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1372             $res->{pcie} = 1 if $1 == 1;
1373         } else {
1374             warn "unknown hostpci setting '$kv'\n";
1375         }
1376     }
1377
1378     return undef if !$found;
1379
1380     return $res;
1381 }
1382
1383 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1384 sub parse_net {
1385     my ($data) = @_;
1386
1387     my $res = {};
1388
1389     foreach my $kvp (split(/,/, $data)) {
1390
1391         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1392             my $model = lc($1);
1393             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1394             $res->{model} = $model;
1395             $res->{macaddr} = $mac;
1396         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1397             $res->{bridge} = $1;
1398         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1399             $res->{queues} = $1;
1400         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1401             $res->{rate} = $1;
1402         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1403             $res->{tag} = $1;
1404         } elsif ($kvp =~ m/^firewall=([01])$/) {
1405             $res->{firewall} = $1;
1406         } elsif ($kvp =~ m/^link_down=([01])$/) {
1407             $res->{link_down} = $1;
1408         } else {
1409             return undef;
1410         }
1411
1412     }
1413
1414     return undef if !$res->{model};
1415
1416     return $res;
1417 }
1418
1419 sub print_net {
1420     my $net = shift;
1421
1422     my $res = "$net->{model}";
1423     $res .= "=$net->{macaddr}" if $net->{macaddr};
1424     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1425     $res .= ",rate=$net->{rate}" if $net->{rate};
1426     $res .= ",tag=$net->{tag}" if $net->{tag};
1427     $res .= ",firewall=1" if $net->{firewall};
1428     $res .= ",link_down=1" if $net->{link_down};
1429     $res .= ",queues=$net->{queues}" if $net->{queues};
1430
1431     return $res;
1432 }
1433
1434 sub add_random_macs {
1435     my ($settings) = @_;
1436
1437     foreach my $opt (keys %$settings) {
1438         next if $opt !~ m/^net(\d+)$/;
1439         my $net = parse_net($settings->{$opt});
1440         next if !$net;
1441         $settings->{$opt} = print_net($net);
1442     }
1443 }
1444
1445 sub add_unused_volume {
1446     my ($config, $volid) = @_;
1447
1448     my $key;
1449     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1450         my $test = "unused$ind";
1451         if (my $vid = $config->{$test}) {
1452             return if $vid eq $volid; # do not add duplicates
1453         } else {
1454             $key = $test;
1455         }
1456     }
1457
1458     die "To many unused volume - please delete them first.\n" if !$key;
1459
1460     $config->{$key} = $volid;
1461
1462     return $key;
1463 }
1464
1465 sub vm_is_volid_owner {
1466     my ($storecfg, $vmid, $volid) = @_;
1467
1468     if ($volid !~  m|^/|) {
1469         my ($path, $owner);
1470         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1471         if ($owner && ($owner == $vmid)) {
1472             return 1;
1473         }
1474     }
1475
1476     return undef;
1477 }
1478
1479 sub vmconfig_delete_pending_option {
1480     my ($conf, $key) = @_;
1481
1482     delete $conf->{pending}->{$key};
1483     my $pending_delete_hash = { $key => 1 };
1484     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1485         $pending_delete_hash->{$opt} = 1;
1486     }
1487     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1488 }
1489
1490 sub vmconfig_undelete_pending_option {
1491     my ($conf, $key) = @_;
1492
1493     my $pending_delete_hash = {};
1494     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1495         $pending_delete_hash->{$opt} = 1;
1496     }
1497     delete $pending_delete_hash->{$key};
1498
1499     my @keylist = keys %$pending_delete_hash;
1500     if (scalar(@keylist)) {
1501         $conf->{pending}->{delete} = join(',', @keylist);
1502     } else {
1503         delete $conf->{pending}->{delete};
1504     }
1505 }
1506
1507 sub vmconfig_register_unused_drive {
1508     my ($storecfg, $vmid, $conf, $drive) = @_;
1509
1510     if (!drive_is_cdrom($drive)) {
1511         my $volid = $drive->{file};
1512         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1513             add_unused_volume($conf, $volid, $vmid);
1514         }
1515     }
1516 }
1517
1518 sub vmconfig_cleanup_pending {
1519     my ($conf) = @_;
1520
1521     # remove pending changes when nothing changed
1522     my $changes;
1523     foreach my $opt (keys %{$conf->{pending}}) {
1524         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1525             $changes = 1;
1526             delete $conf->{pending}->{$opt};
1527         }
1528     }
1529
1530     # remove delete if option is not set
1531     my $pending_delete_hash = {};
1532     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1533         if (defined($conf->{$opt})) {
1534             $pending_delete_hash->{$opt} = 1;
1535         } else {
1536             $changes = 1;
1537         }
1538     }
1539
1540     my @keylist = keys %$pending_delete_hash;
1541     if (scalar(@keylist)) {
1542         $conf->{pending}->{delete} = join(',', @keylist);
1543     } else {
1544         delete $conf->{pending}->{delete};
1545     }
1546
1547     return $changes;
1548 }
1549
1550 my $valid_smbios1_options = {
1551     manufacturer => '\S+',
1552     product => '\S+',
1553     version => '\S+',
1554     serial => '\S+',
1555     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1556     sku => '\S+',
1557     family => '\S+',
1558 };
1559
1560 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1561 sub parse_smbios1 {
1562     my ($data) = @_;
1563
1564     my $res = {};
1565
1566     foreach my $kvp (split(/,/, $data)) {
1567         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1568         my ($k, $v) = split(/=/, $kvp);
1569         return undef if !defined($k) || !defined($v);
1570         return undef if !$valid_smbios1_options->{$k};
1571         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1572         $res->{$k} = $v;
1573     }
1574
1575     return $res;
1576 }
1577
1578 sub print_smbios1 {
1579     my ($smbios1) = @_;
1580
1581     my $data = '';
1582     foreach my $k (keys %$smbios1) {
1583         next if !defined($smbios1->{$k});
1584         next if !$valid_smbios1_options->{$k};
1585         $data .= ',' if $data;
1586         $data .= "$k=$smbios1->{$k}";
1587     }
1588     return $data;
1589 }
1590
1591 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1592 sub verify_smbios1 {
1593     my ($value, $noerr) = @_;
1594
1595     return $value if parse_smbios1($value);
1596
1597     return undef if $noerr;
1598
1599     die "unable to parse smbios (type 1) options\n";
1600 }
1601
1602 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1603 sub verify_bootdisk {
1604     my ($value, $noerr) = @_;
1605
1606     return $value if valid_drivename($value);
1607
1608     return undef if $noerr;
1609
1610     die "invalid boot disk '$value'\n";
1611 }
1612
1613 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1614 sub verify_numa {
1615     my ($value, $noerr) = @_;
1616
1617     return $value if parse_numa($value);
1618
1619     return undef if $noerr;
1620
1621     die "unable to parse numa options\n";
1622 }
1623
1624 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1625 sub verify_net {
1626     my ($value, $noerr) = @_;
1627
1628     return $value if parse_net($value);
1629
1630     return undef if $noerr;
1631
1632     die "unable to parse network options\n";
1633 }
1634
1635 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1636 sub verify_drive {
1637     my ($value, $noerr) = @_;
1638
1639     return $value if parse_drive(undef, $value);
1640
1641     return undef if $noerr;
1642
1643     die "unable to parse drive options\n";
1644 }
1645
1646 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1647 sub verify_hostpci {
1648     my ($value, $noerr) = @_;
1649
1650     return $value if parse_hostpci($value);
1651
1652     return undef if $noerr;
1653
1654     die "unable to parse pci id\n";
1655 }
1656
1657 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1658 sub verify_watchdog {
1659     my ($value, $noerr) = @_;
1660
1661     return $value if parse_watchdog($value);
1662
1663     return undef if $noerr;
1664
1665     die "unable to parse watchdog options\n";
1666 }
1667
1668 sub parse_watchdog {
1669     my ($value) = @_;
1670
1671     return undef if !$value;
1672
1673     my $res = {};
1674
1675     foreach my $p (split(/,/, $value)) {
1676         next if $p =~ m/^\s*$/;
1677
1678         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1679             $res->{model} = $2;
1680         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1681             $res->{action} = $2;
1682         } else {
1683             return undef;
1684         }
1685     }
1686
1687     return $res;
1688 }
1689
1690 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1691 sub verify_startup {
1692     my ($value, $noerr) = @_;
1693
1694     return $value if parse_startup($value);
1695
1696     return undef if $noerr;
1697
1698     die "unable to parse startup options\n";
1699 }
1700
1701 sub parse_startup {
1702     my ($value) = @_;
1703
1704     return undef if !$value;
1705
1706     my $res = {};
1707
1708     foreach my $p (split(/,/, $value)) {
1709         next if $p =~ m/^\s*$/;
1710
1711         if ($p =~ m/^(order=)?(\d+)$/) {
1712             $res->{order} = $2;
1713         } elsif ($p =~ m/^up=(\d+)$/) {
1714             $res->{up} = $1;
1715         } elsif ($p =~ m/^down=(\d+)$/) {
1716             $res->{down} = $1;
1717         } else {
1718             return undef;
1719         }
1720     }
1721
1722     return $res;
1723 }
1724
1725 sub parse_usb_device {
1726     my ($value) = @_;
1727
1728     return undef if !$value;
1729
1730     my @dl = split(/,/, $value);
1731     my $found;
1732
1733     my $res = {};
1734     foreach my $v (@dl) {
1735         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1736             $found = 1;
1737             $res->{vendorid} = $2;
1738             $res->{productid} = $4;
1739         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1740             $found = 1;
1741             $res->{hostbus} = $1;
1742             $res->{hostport} = $2;
1743         } elsif ($v =~ m/^spice$/) {
1744             $found = 1;
1745             $res->{spice} = 1;
1746         } else {
1747             return undef;
1748         }
1749     }
1750     return undef if !$found;
1751
1752     return $res;
1753 }
1754
1755 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1756 sub verify_usb_device {
1757     my ($value, $noerr) = @_;
1758
1759     return $value if parse_usb_device($value);
1760
1761     return undef if $noerr;
1762
1763     die "unable to parse usb device\n";
1764 }
1765
1766 # add JSON properties for create and set function
1767 sub json_config_properties {
1768     my $prop = shift;
1769
1770     foreach my $opt (keys %$confdesc) {
1771         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1772         $prop->{$opt} = $confdesc->{$opt};
1773     }
1774
1775     return $prop;
1776 }
1777
1778 sub check_type {
1779     my ($key, $value) = @_;
1780
1781     die "unknown setting '$key'\n" if !$confdesc->{$key};
1782
1783     my $type = $confdesc->{$key}->{type};
1784
1785     if (!defined($value)) {
1786         die "got undefined value\n";
1787     }
1788
1789     if ($value =~ m/[\n\r]/) {
1790         die "property contains a line feed\n";
1791     }
1792
1793     if ($type eq 'boolean') {
1794         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1795         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1796         die "type check ('boolean') failed - got '$value'\n";
1797     } elsif ($type eq 'integer') {
1798         return int($1) if $value =~ m/^(\d+)$/;
1799         die "type check ('integer') failed - got '$value'\n";
1800     } elsif ($type eq 'number') {
1801         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1802         die "type check ('number') failed - got '$value'\n";
1803     } elsif ($type eq 'string') {
1804         if (my $fmt = $confdesc->{$key}->{format}) {
1805             if ($fmt eq 'pve-qm-drive') {
1806                 # special case - we need to pass $key to parse_drive()
1807                 my $drive = parse_drive($key, $value);
1808                 return $value if $drive;
1809                 die "unable to parse drive options\n";
1810             }
1811             PVE::JSONSchema::check_format($fmt, $value);
1812             return $value;
1813         }
1814         $value =~ s/^\"(.*)\"$/$1/;
1815         return $value;
1816     } else {
1817         die "internal error"
1818     }
1819 }
1820
1821 sub lock_config_full {
1822     my ($vmid, $timeout, $code, @param) = @_;
1823
1824     my $filename = config_file_lock($vmid);
1825
1826     my $res = lock_file($filename, $timeout, $code, @param);
1827
1828     die $@ if $@;
1829
1830     return $res;
1831 }
1832
1833 sub lock_config_mode {
1834     my ($vmid, $timeout, $shared, $code, @param) = @_;
1835
1836     my $filename = config_file_lock($vmid);
1837
1838     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1839
1840     die $@ if $@;
1841
1842     return $res;
1843 }
1844
1845 sub lock_config {
1846     my ($vmid, $code, @param) = @_;
1847
1848     return lock_config_full($vmid, 10, $code, @param);
1849 }
1850
1851 sub cfs_config_path {
1852     my ($vmid, $node) = @_;
1853
1854     $node = $nodename if !$node;
1855     return "nodes/$node/qemu-server/$vmid.conf";
1856 }
1857
1858 sub check_iommu_support{
1859     #fixme : need to check IOMMU support
1860     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1861
1862     my $iommu=1;
1863     return $iommu;
1864
1865 }
1866
1867 sub config_file {
1868     my ($vmid, $node) = @_;
1869
1870     my $cfspath = cfs_config_path($vmid, $node);
1871     return "/etc/pve/$cfspath";
1872 }
1873
1874 sub config_file_lock {
1875     my ($vmid) = @_;
1876
1877     return "$lock_dir/lock-$vmid.conf";
1878 }
1879
1880 sub touch_config {
1881     my ($vmid) = @_;
1882
1883     my $conf = config_file($vmid);
1884     utime undef, undef, $conf;
1885 }
1886
1887 sub destroy_vm {
1888     my ($storecfg, $vmid, $keep_empty_config) = @_;
1889
1890     my $conffile = config_file($vmid);
1891
1892     my $conf = load_config($vmid);
1893
1894     check_lock($conf);
1895
1896     # only remove disks owned by this VM
1897     foreach_drive($conf, sub {
1898         my ($ds, $drive) = @_;
1899
1900         return if drive_is_cdrom($drive);
1901
1902         my $volid = $drive->{file};
1903
1904         return if !$volid || $volid =~ m|^/|;
1905
1906         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1907         return if !$path || !$owner || ($owner != $vmid);
1908
1909         PVE::Storage::vdisk_free($storecfg, $volid);
1910     });
1911
1912     if ($keep_empty_config) {
1913         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1914     } else {
1915         unlink $conffile;
1916     }
1917
1918     # also remove unused disk
1919     eval {
1920         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1921
1922         eval {
1923             PVE::Storage::foreach_volid($dl, sub {
1924                 my ($volid, $sid, $volname, $d) = @_;
1925                 PVE::Storage::vdisk_free($storecfg, $volid);
1926             });
1927         };
1928         warn $@ if $@;
1929
1930     };
1931     warn $@ if $@;
1932 }
1933
1934 sub load_config {
1935     my ($vmid, $node) = @_;
1936
1937     my $cfspath = cfs_config_path($vmid, $node);
1938
1939     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1940
1941     die "no such VM ('$vmid')\n" if !defined($conf);
1942
1943     return $conf;
1944 }
1945
1946 sub parse_vm_config {
1947     my ($filename, $raw) = @_;
1948
1949     return undef if !defined($raw);
1950
1951     my $res = {
1952         digest => Digest::SHA::sha1_hex($raw),
1953         snapshots => {},
1954         pending => {},
1955     };
1956
1957     $filename =~ m|/qemu-server/(\d+)\.conf$|
1958         || die "got strange filename '$filename'";
1959
1960     my $vmid = $1;
1961
1962     my $conf = $res;
1963     my $descr = '';
1964     my $section = '';
1965
1966     my @lines = split(/\n/, $raw);
1967     foreach my $line (@lines) {
1968         next if $line =~ m/^\s*$/;
1969
1970         if ($line =~ m/^\[PENDING\]\s*$/i) {
1971             $section = 'pending';
1972             $conf->{description} = $descr if $descr;
1973             $descr = '';
1974             $conf = $res->{$section} = {};
1975             next;
1976
1977         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1978             $section = $1;
1979             $conf->{description} = $descr if $descr;
1980             $descr = '';
1981             $conf = $res->{snapshots}->{$section} = {};
1982             next;
1983         }
1984
1985         if ($line =~ m/^\#(.*)\s*$/) {
1986             $descr .= PVE::Tools::decode_text($1) . "\n";
1987             next;
1988         }
1989
1990         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1991             $descr .= PVE::Tools::decode_text($2);
1992         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1993             $conf->{snapstate} = $1;
1994         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1995             my $key = $1;
1996             my $value = $2;
1997             $conf->{$key} = $value;
1998         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1999             my $value = $1;
2000             if ($section eq 'pending') {
2001                 $conf->{delete} = $value; # we parse this later
2002             } else {
2003                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2004             }
2005         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2006             my $key = $1;
2007             my $value = $2;
2008             eval { $value = check_type($key, $value); };
2009             if ($@) {
2010                 warn "vm $vmid - unable to parse value of '$key' - $@";
2011             } else {
2012                 my $fmt = $confdesc->{$key}->{format};
2013                 if ($fmt && $fmt eq 'pve-qm-drive') {
2014                     my $v = parse_drive($key, $value);
2015                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2016                         $v->{file} = $volid;
2017                         $value = print_drive($vmid, $v);
2018                     } else {
2019                         warn "vm $vmid - unable to parse value of '$key'\n";
2020                         next;
2021                     }
2022                 }
2023
2024                 if ($key eq 'cdrom') {
2025                     $conf->{ide2} = $value;
2026                 } else {
2027                     $conf->{$key} = $value;
2028                 }
2029             }
2030         }
2031     }
2032
2033     $conf->{description} = $descr if $descr;
2034
2035     delete $res->{snapstate}; # just to be sure
2036
2037     return $res;
2038 }
2039
2040 sub write_vm_config {
2041     my ($filename, $conf) = @_;
2042
2043     delete $conf->{snapstate}; # just to be sure
2044
2045     if ($conf->{cdrom}) {
2046         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2047         $conf->{ide2} = $conf->{cdrom};
2048         delete $conf->{cdrom};
2049     }
2050
2051     # we do not use 'smp' any longer
2052     if ($conf->{sockets}) {
2053         delete $conf->{smp};
2054     } elsif ($conf->{smp}) {
2055         $conf->{sockets} = $conf->{smp};
2056         delete $conf->{cores};
2057         delete $conf->{smp};
2058     }
2059
2060     my $used_volids = {};
2061
2062     my $cleanup_config = sub {
2063         my ($cref, $pending, $snapname) = @_;
2064
2065         foreach my $key (keys %$cref) {
2066             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2067                 $key eq 'snapstate' || $key eq 'pending';
2068             my $value = $cref->{$key};
2069             if ($key eq 'delete') {
2070                 die "propertry 'delete' is only allowed in [PENDING]\n"
2071                     if !$pending;
2072                 # fixme: check syntax?
2073                 next;
2074             }
2075             eval { $value = check_type($key, $value); };
2076             die "unable to parse value of '$key' - $@" if $@;
2077
2078             $cref->{$key} = $value;
2079
2080             if (!$snapname && valid_drivename($key)) {
2081                 my $drive = parse_drive($key, $value);
2082                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2083             }
2084         }
2085     };
2086
2087     &$cleanup_config($conf);
2088
2089     &$cleanup_config($conf->{pending}, 1);
2090
2091     foreach my $snapname (keys %{$conf->{snapshots}}) {
2092         die "internal error" if $snapname eq 'pending';
2093         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2094     }
2095
2096     # remove 'unusedX' settings if we re-add a volume
2097     foreach my $key (keys %$conf) {
2098         my $value = $conf->{$key};
2099         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2100             delete $conf->{$key};
2101         }
2102     }
2103
2104     my $generate_raw_config = sub {
2105         my ($conf) = @_;
2106
2107         my $raw = '';
2108
2109         # add description as comment to top of file
2110         my $descr = $conf->{description} || '';
2111         foreach my $cl (split(/\n/, $descr)) {
2112             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2113         }
2114
2115         foreach my $key (sort keys %$conf) {
2116             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2117             $raw .= "$key: $conf->{$key}\n";
2118         }
2119         return $raw;
2120     };
2121
2122     my $raw = &$generate_raw_config($conf);
2123
2124     if (scalar(keys %{$conf->{pending}})){
2125         $raw .= "\n[PENDING]\n";
2126         $raw .= &$generate_raw_config($conf->{pending});
2127     }
2128
2129     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2130         $raw .= "\n[$snapname]\n";
2131         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2132     }
2133
2134     return $raw;
2135 }
2136
2137 sub update_config_nolock {
2138     my ($vmid, $conf, $skiplock) = @_;
2139
2140     check_lock($conf) if !$skiplock;
2141
2142     my $cfspath = cfs_config_path($vmid);
2143
2144     PVE::Cluster::cfs_write_file($cfspath, $conf);
2145 }
2146
2147 sub update_config {
2148     my ($vmid, $conf, $skiplock) = @_;
2149
2150     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2151 }
2152
2153 sub load_defaults {
2154
2155     my $res = {};
2156
2157     # we use static defaults from our JSON schema configuration
2158     foreach my $key (keys %$confdesc) {
2159         if (defined(my $default = $confdesc->{$key}->{default})) {
2160             $res->{$key} = $default;
2161         }
2162     }
2163
2164     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2165     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2166
2167     return $res;
2168 }
2169
2170 sub config_list {
2171     my $vmlist = PVE::Cluster::get_vmlist();
2172     my $res = {};
2173     return $res if !$vmlist || !$vmlist->{ids};
2174     my $ids = $vmlist->{ids};
2175
2176     foreach my $vmid (keys %$ids) {
2177         my $d = $ids->{$vmid};
2178         next if !$d->{node} || $d->{node} ne $nodename;
2179         next if !$d->{type} || $d->{type} ne 'qemu';
2180         $res->{$vmid}->{exists} = 1;
2181     }
2182     return $res;
2183 }
2184
2185 # test if VM uses local resources (to prevent migration)
2186 sub check_local_resources {
2187     my ($conf, $noerr) = @_;
2188
2189     my $loc_res = 0;
2190
2191     $loc_res = 1 if $conf->{hostusb}; # old syntax
2192     $loc_res = 1 if $conf->{hostpci}; # old syntax
2193
2194     foreach my $k (keys %$conf) {
2195         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2196         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2197     }
2198
2199     die "VM uses local resources\n" if $loc_res && !$noerr;
2200
2201     return $loc_res;
2202 }
2203
2204 # check if used storages are available on all nodes (use by migrate)
2205 sub check_storage_availability {
2206     my ($storecfg, $conf, $node) = @_;
2207
2208     foreach_drive($conf, sub {
2209         my ($ds, $drive) = @_;
2210
2211         my $volid = $drive->{file};
2212         return if !$volid;
2213
2214         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2215         return if !$sid;
2216
2217         # check if storage is available on both nodes
2218         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2219         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2220    });
2221 }
2222
2223 # list nodes where all VM images are available (used by has_feature API)
2224 sub shared_nodes {
2225     my ($conf, $storecfg) = @_;
2226
2227     my $nodelist = PVE::Cluster::get_nodelist();
2228     my $nodehash = { map { $_ => 1 } @$nodelist };
2229     my $nodename = PVE::INotify::nodename();
2230
2231     foreach_drive($conf, sub {
2232         my ($ds, $drive) = @_;
2233
2234         my $volid = $drive->{file};
2235         return if !$volid;
2236
2237         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2238         if ($storeid) {
2239             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2240             if ($scfg->{disable}) {
2241                 $nodehash = {};
2242             } elsif (my $avail = $scfg->{nodes}) {
2243                 foreach my $node (keys %$nodehash) {
2244                     delete $nodehash->{$node} if !$avail->{$node};
2245                 }
2246             } elsif (!$scfg->{shared}) {
2247                 foreach my $node (keys %$nodehash) {
2248                     delete $nodehash->{$node} if $node ne $nodename
2249                 }
2250             }
2251         }
2252     });
2253
2254     return $nodehash
2255 }
2256
2257 sub check_lock {
2258     my ($conf) = @_;
2259
2260     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2261 }
2262
2263 sub check_cmdline {
2264     my ($pidfile, $pid) = @_;
2265
2266     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2267     if (defined($fh)) {
2268         my $line = <$fh>;
2269         $fh->close;
2270         return undef if !$line;
2271         my @param = split(/\0/, $line);
2272
2273         my $cmd = $param[0];
2274         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2275
2276         for (my $i = 0; $i < scalar (@param); $i++) {
2277             my $p = $param[$i];
2278             next if !$p;
2279             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2280                 my $p = $param[$i+1];
2281                 return 1 if $p && ($p eq $pidfile);
2282                 return undef;
2283             }
2284         }
2285     }
2286     return undef;
2287 }
2288
2289 sub check_running {
2290     my ($vmid, $nocheck, $node) = @_;
2291
2292     my $filename = config_file($vmid, $node);
2293
2294     die "unable to find configuration file for VM $vmid - no such machine\n"
2295         if !$nocheck && ! -f $filename;
2296
2297     my $pidfile = pidfile_name($vmid);
2298
2299     if (my $fd = IO::File->new("<$pidfile")) {
2300         my $st = stat($fd);
2301         my $line = <$fd>;
2302         close($fd);
2303
2304         my $mtime = $st->mtime;
2305         if ($mtime > time()) {
2306             warn "file '$filename' modified in future\n";
2307         }
2308
2309         if ($line =~ m/^(\d+)$/) {
2310             my $pid = $1;
2311             if (check_cmdline($pidfile, $pid)) {
2312                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2313                     return $pid;
2314                 }
2315             }
2316         }
2317     }
2318
2319     return undef;
2320 }
2321
2322 sub vzlist {
2323
2324     my $vzlist = config_list();
2325
2326     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2327
2328     while (defined(my $de = $fd->read)) {
2329         next if $de !~ m/^(\d+)\.pid$/;
2330         my $vmid = $1;
2331         next if !defined($vzlist->{$vmid});
2332         if (my $pid = check_running($vmid)) {
2333             $vzlist->{$vmid}->{pid} = $pid;
2334         }
2335     }
2336
2337     return $vzlist;
2338 }
2339
2340 sub disksize {
2341     my ($storecfg, $conf) = @_;
2342
2343     my $bootdisk = $conf->{bootdisk};
2344     return undef if !$bootdisk;
2345     return undef if !valid_drivename($bootdisk);
2346
2347     return undef if !$conf->{$bootdisk};
2348
2349     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2350     return undef if !defined($drive);
2351
2352     return undef if drive_is_cdrom($drive);
2353
2354     my $volid = $drive->{file};
2355     return undef if !$volid;
2356
2357     return $drive->{size};
2358 }
2359
2360 my $last_proc_pid_stat;
2361
2362 # get VM status information
2363 # This must be fast and should not block ($full == false)
2364 # We only query KVM using QMP if $full == true (this can be slow)
2365 sub vmstatus {
2366     my ($opt_vmid, $full) = @_;
2367
2368     my $res = {};
2369
2370     my $storecfg = PVE::Storage::config();
2371
2372     my $list = vzlist();
2373     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2374
2375     my $cpucount = $cpuinfo->{cpus} || 1;
2376
2377     foreach my $vmid (keys %$list) {
2378         next if $opt_vmid && ($vmid ne $opt_vmid);
2379
2380         my $cfspath = cfs_config_path($vmid);
2381         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2382
2383         my $d = {};
2384         $d->{pid} = $list->{$vmid}->{pid};
2385
2386         # fixme: better status?
2387         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2388
2389         my $size = disksize($storecfg, $conf);
2390         if (defined($size)) {
2391             $d->{disk} = 0; # no info available
2392             $d->{maxdisk} = $size;
2393         } else {
2394             $d->{disk} = 0;
2395             $d->{maxdisk} = 0;
2396         }
2397
2398         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2399         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2400         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2401
2402         $d->{name} = $conf->{name} || "VM $vmid";
2403         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2404
2405         if ($conf->{balloon}) {
2406             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2407             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2408         }
2409
2410         $d->{uptime} = 0;
2411         $d->{cpu} = 0;
2412         $d->{mem} = 0;
2413
2414         $d->{netout} = 0;
2415         $d->{netin} = 0;
2416
2417         $d->{diskread} = 0;
2418         $d->{diskwrite} = 0;
2419
2420         $d->{template} = is_template($conf);
2421
2422         $res->{$vmid} = $d;
2423     }
2424
2425     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2426     foreach my $dev (keys %$netdev) {
2427         next if $dev !~ m/^tap([1-9]\d*)i/;
2428         my $vmid = $1;
2429         my $d = $res->{$vmid};
2430         next if !$d;
2431
2432         $d->{netout} += $netdev->{$dev}->{receive};
2433         $d->{netin} += $netdev->{$dev}->{transmit};
2434     }
2435
2436     my $ctime = gettimeofday;
2437
2438     foreach my $vmid (keys %$list) {
2439
2440         my $d = $res->{$vmid};
2441         my $pid = $d->{pid};
2442         next if !$pid;
2443
2444         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2445         next if !$pstat; # not running
2446
2447         my $used = $pstat->{utime} + $pstat->{stime};
2448
2449         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2450
2451         if ($pstat->{vsize}) {
2452             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2453         }
2454
2455         my $old = $last_proc_pid_stat->{$pid};
2456         if (!$old) {
2457             $last_proc_pid_stat->{$pid} = {
2458                 time => $ctime,
2459                 used => $used,
2460                 cpu => 0,
2461             };
2462             next;
2463         }
2464
2465         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2466
2467         if ($dtime > 1000) {
2468             my $dutime = $used -  $old->{used};
2469
2470             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2471             $last_proc_pid_stat->{$pid} = {
2472                 time => $ctime,
2473                 used => $used,
2474                 cpu => $d->{cpu},
2475             };
2476         } else {
2477             $d->{cpu} = $old->{cpu};
2478         }
2479     }
2480
2481     return $res if !$full;
2482
2483     my $qmpclient = PVE::QMPClient->new();
2484
2485     my $ballooncb = sub {
2486         my ($vmid, $resp) = @_;
2487
2488         my $info = $resp->{'return'};
2489         return if !$info->{max_mem};
2490
2491         my $d = $res->{$vmid};
2492
2493         # use memory assigned to VM
2494         $d->{maxmem} = $info->{max_mem};
2495         $d->{balloon} = $info->{actual};
2496
2497         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2498             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2499             $d->{freemem} = $info->{free_mem};
2500         }
2501
2502     };
2503
2504     my $blockstatscb = sub {
2505         my ($vmid, $resp) = @_;
2506         my $data = $resp->{'return'} || [];
2507         my $totalrdbytes = 0;
2508         my $totalwrbytes = 0;
2509         for my $blockstat (@$data) {
2510             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2511             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2512         }
2513         $res->{$vmid}->{diskread} = $totalrdbytes;
2514         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2515     };
2516
2517     my $statuscb = sub {
2518         my ($vmid, $resp) = @_;
2519
2520         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2521         # this fails if ballon driver is not loaded, so this must be
2522         # the last commnand (following command are aborted if this fails).
2523         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2524
2525         my $status = 'unknown';
2526         if (!defined($status = $resp->{'return'}->{status})) {
2527             warn "unable to get VM status\n";
2528             return;
2529         }
2530
2531         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2532     };
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536         next if !$res->{$vmid}->{pid}; # not running
2537         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2538     }
2539
2540     $qmpclient->queue_execute(undef, 1);
2541
2542     foreach my $vmid (keys %$list) {
2543         next if $opt_vmid && ($vmid ne $opt_vmid);
2544         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2545     }
2546
2547     return $res;
2548 }
2549
2550 sub foreach_dimm {
2551     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2552
2553     my $dimm_id = 0;
2554     my $current_size = 1024;
2555     my $dimm_size = 512;
2556     return if $current_size == $memory;
2557
2558     for (my $j = 0; $j < 8; $j++) {
2559         for (my $i = 0; $i < 32; $i++) {
2560             my $name = "dimm${dimm_id}";
2561             $dimm_id++;
2562             my $numanode = $i % $sockets;
2563             $current_size += $dimm_size;
2564             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2565             return  $current_size if $current_size >= $memory;
2566         }
2567         $dimm_size *= 2;
2568     }
2569 }
2570
2571 sub foreach_drive {
2572     my ($conf, $func) = @_;
2573
2574     foreach my $ds (keys %$conf) {
2575         next if !valid_drivename($ds);
2576
2577         my $drive = parse_drive($ds, $conf->{$ds});
2578         next if !$drive;
2579
2580         &$func($ds, $drive);
2581     }
2582 }
2583
2584 sub foreach_volid {
2585     my ($conf, $func) = @_;
2586
2587     my $volhash = {};
2588
2589     my $test_volid = sub {
2590         my ($volid, $is_cdrom) = @_;
2591
2592         return if !$volid;
2593
2594         $volhash->{$volid} = $is_cdrom || 0;
2595     };
2596
2597     foreach_drive($conf, sub {
2598         my ($ds, $drive) = @_;
2599         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2600     });
2601
2602     foreach my $snapname (keys %{$conf->{snapshots}}) {
2603         my $snap = $conf->{snapshots}->{$snapname};
2604         &$test_volid($snap->{vmstate}, 0);
2605         foreach_drive($snap, sub {
2606             my ($ds, $drive) = @_;
2607             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2608         });
2609     }
2610
2611     foreach my $volid (keys %$volhash) {
2612         &$func($volid, $volhash->{$volid});
2613     }
2614 }
2615
2616 sub vga_conf_has_spice {
2617     my ($vga) = @_;
2618
2619     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2620
2621     return $1 || 1;
2622 }
2623
2624 sub config_to_command {
2625     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2626
2627     my $cmd = [];
2628     my $globalFlags = [];
2629     my $machineFlags = [];
2630     my $rtcFlags = [];
2631     my $cpuFlags = [];
2632     my $devices = [];
2633     my $pciaddr = '';
2634     my $bridges = {};
2635     my $kvmver = kvm_user_version();
2636     my $vernum = 0; # unknown
2637     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2638         $vernum = $1*1000000+$2*1000;
2639     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2640         $vernum = $1*1000000+$2*1000+$3;
2641     }
2642
2643     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2644
2645     my $have_ovz = -f '/proc/vz/vestat';
2646
2647     my $q35 = machine_type_is_q35($conf);
2648     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2649     my $machine_type = $forcemachine || $conf->{machine};
2650
2651     push @$cmd, '/usr/bin/kvm';
2652
2653     push @$cmd, '-id', $vmid;
2654
2655     my $use_virtio = 0;
2656
2657     my $qmpsocket = qmp_socket($vmid);
2658     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2659     push @$cmd, '-mon', "chardev=qmp,mode=control";
2660
2661     my $socket = vnc_socket($vmid);
2662     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2663
2664     push @$cmd, '-pidfile' , pidfile_name($vmid);
2665
2666     push @$cmd, '-daemonize';
2667
2668     if ($conf->{smbios1}) {
2669         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2670     }
2671
2672     if ($q35) {
2673         # the q35 chipset support native usb2, so we enable usb controller
2674         # by default for this machine type
2675         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2676     } else {
2677         $pciaddr = print_pci_addr("piix3", $bridges);
2678         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2679
2680         my $use_usb2 = 0;
2681         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2682             next if !$conf->{"usb$i"};
2683             $use_usb2 = 1;
2684         }
2685         # include usb device config
2686         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2687     }
2688
2689     my $vga = $conf->{vga};
2690
2691     my $qxlnum = vga_conf_has_spice($vga);
2692     $vga = 'qxl' if $qxlnum;
2693
2694     if (!$vga) {
2695         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2696                                 $conf->{ostype} eq 'win7' ||
2697                                 $conf->{ostype} eq 'w2k8')) {
2698             $vga = 'std';
2699         } else {
2700             $vga = 'cirrus';
2701         }
2702     }
2703
2704     # enable absolute mouse coordinates (needed by vnc)
2705     my $tablet;
2706     if (defined($conf->{tablet})) {
2707         $tablet = $conf->{tablet};
2708     } else {
2709         $tablet = $defaults->{tablet};
2710         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2711         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2712     }
2713
2714     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2715
2716     # host pci devices
2717     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2718         my $d = parse_hostpci($conf->{"hostpci$i"});
2719         next if !$d;
2720
2721         my $pcie = $d->{pcie};
2722         if($pcie){
2723             die "q35 machine model is not enabled" if !$q35;
2724             $pciaddr = print_pcie_addr("hostpci$i");
2725         }else{
2726             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2727         }
2728
2729         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2730         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2731         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2732         if ($xvga && $xvga ne '') {
2733             push @$cpuFlags, 'kvm=off';
2734             $vga = 'none';
2735         }
2736         $driver = "vfio-pci" if $xvga ne '';
2737         my $pcidevices = $d->{pciid};
2738         my $multifunction = 1 if @$pcidevices > 1;
2739
2740         my $j=0;
2741         foreach my $pcidevice (@$pcidevices) {
2742
2743             my $id = "hostpci$i";
2744             $id .= ".$j" if $multifunction;
2745             my $addr = $pciaddr;
2746             $addr .= ".$j" if $multifunction;
2747             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2748
2749             if($j == 0){
2750                 $devicestr .= "$rombar$xvga";
2751                 $devicestr .= ",multifunction=on" if $multifunction;
2752             }
2753
2754             push @$devices, '-device', $devicestr;
2755             $j++;
2756         }
2757     }
2758
2759     # usb devices
2760     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2761         my $d = parse_usb_device($conf->{"usb$i"});
2762         next if !$d;
2763         if ($d->{vendorid} && $d->{productid}) {
2764             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2765         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2766             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2767         } elsif ($d->{spice}) {
2768             # usb redir support for spice
2769             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2770             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2771         }
2772     }
2773
2774     # serial devices
2775     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2776         if (my $path = $conf->{"serial$i"}) {
2777             if ($path eq 'socket') {
2778                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2779                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2780                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2781             } else {
2782                 die "no such serial device\n" if ! -c $path;
2783                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2784                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2785             }
2786         }
2787     }
2788
2789     # parallel devices
2790     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2791         if (my $path = $conf->{"parallel$i"}) {
2792             die "no such parallel device\n" if ! -c $path;
2793             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2794             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2795             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2796         }
2797     }
2798
2799     my $vmname = $conf->{name} || "vm$vmid";
2800
2801     push @$cmd, '-name', $vmname;
2802
2803     my $sockets = 1;
2804     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2805     $sockets = $conf->{sockets} if  $conf->{sockets};
2806
2807     my $cores = $conf->{cores} || 1;
2808
2809     my $maxcpus = $sockets * $cores;
2810
2811     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2812
2813     my $allowed_vcpus = $cpuinfo->{cpus};
2814
2815     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2816         if ($allowed_vcpus < $maxcpus);
2817
2818     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2819
2820     push @$cmd, '-nodefaults';
2821
2822     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2823
2824     my $bootindex_hash = {};
2825     my $i = 1;
2826     foreach my $o (split(//, $bootorder)) {
2827         $bootindex_hash->{$o} = $i*100;
2828         $i++;
2829     }
2830
2831     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2832
2833     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2834
2835     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2836
2837     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2838
2839     # time drift fix
2840     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2841
2842     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2843     my $useLocaltime = $conf->{localtime};
2844
2845     if (my $ost = $conf->{ostype}) {
2846         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2847
2848         if ($ost =~ m/^w/) { # windows
2849             $useLocaltime = 1 if !defined($conf->{localtime});
2850
2851             # use time drift fix when acpi is enabled
2852             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2853                 $tdf = 1 if !defined($conf->{tdf});
2854             }
2855         }
2856
2857         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2858             $ost eq 'wvista') {
2859             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2860             push @$cmd, '-no-hpet';
2861             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2862                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2863                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2864                 push @$cpuFlags , 'hv_time' if !$nokvm;
2865
2866             } else {
2867                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2868             }
2869         }
2870
2871         if ($ost eq 'win7' || $ost eq 'win8') {
2872             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2873         }
2874     }
2875
2876     push @$rtcFlags, 'driftfix=slew' if $tdf;
2877
2878     if ($nokvm) {
2879         push @$machineFlags, 'accel=tcg';
2880     } else {
2881         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2882     }
2883
2884     if ($machine_type) {
2885         push @$machineFlags, "type=${machine_type}";
2886     }
2887
2888     if ($conf->{startdate}) {
2889         push @$rtcFlags, "base=$conf->{startdate}";
2890     } elsif ($useLocaltime) {
2891         push @$rtcFlags, 'base=localtime';
2892     }
2893
2894     my $cpu = $nokvm ? "qemu64" : "kvm64";
2895     $cpu = $conf->{cpu} if $conf->{cpu};
2896
2897     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2898
2899     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2900
2901     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2902
2903     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2904
2905     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2906
2907         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2908         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2909     }
2910
2911     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2912
2913     push @$cmd, '-cpu', "$cpu,enforce";
2914
2915     my $memory = $conf->{memory} || $defaults->{memory};
2916     my $static_memory = 0;
2917     my $dimm_memory = 0;
2918
2919     if ($hotplug_features->{memory}) {
2920         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2921         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2922         $static_memory = $STATICMEM;
2923         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2924         $dimm_memory = $memory - $static_memory;
2925         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2926
2927     } else {
2928
2929         $static_memory = $memory;
2930         push @$cmd, '-m', $static_memory;
2931     }
2932
2933     if ($conf->{numa}) {
2934
2935         my $numa_totalmemory = undef;
2936         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2937             next if !$conf->{"numa$i"};
2938             my $numa = parse_numa($conf->{"numa$i"});
2939             next if !$numa;
2940             # memory
2941             die "missing numa node$i memory value\n" if !$numa->{memory};
2942             my $numa_memory = $numa->{memory};
2943             $numa_totalmemory += $numa_memory;
2944             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2945
2946             # cpus
2947             my $cpus_start = $numa->{cpus}->{start};
2948             die "missing numa node$i cpus\n" if !defined($cpus_start);
2949             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2950             my $cpus = $cpus_start;
2951             if (defined($cpus_end)) {
2952                 $cpus .= "-$cpus_end";
2953                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2954             }
2955
2956             # hostnodes
2957             my $hostnodes_start = $numa->{hostnodes}->{start};
2958             if (defined($hostnodes_start)) {
2959                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2960                 my $hostnodes = $hostnodes_start;
2961                 if (defined($hostnodes_end)) {
2962                     $hostnodes .= "-$hostnodes_end";
2963                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2964                 }
2965
2966                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2967                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2968                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2969                 }
2970
2971                 # policy
2972                 my $policy = $numa->{policy};
2973                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2974                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2975             }
2976
2977             push @$cmd, '-object', $numa_object;
2978             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2979         }
2980
2981         die "total memory for NUMA nodes must be equal to vm static memory\n"
2982             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2983
2984         #if no custom tology, we split memory and cores across numa nodes
2985         if(!$numa_totalmemory) {
2986
2987             my $numa_memory = ($static_memory / $sockets) . "M";
2988
2989             for (my $i = 0; $i < $sockets; $i++)  {
2990
2991                 my $cpustart = ($cores * $i);
2992                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2993                 my $cpus = $cpustart;
2994                 $cpus .= "-$cpuend" if $cpuend;
2995
2996                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2997                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2998             }
2999         }
3000     }
3001
3002     if ($hotplug_features->{memory}) {
3003         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3004             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3005             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3006             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3007
3008             #if dimm_memory is not aligned to dimm map
3009             if($current_size > $memory) {
3010                  $conf->{memory} = $current_size;
3011                  update_config_nolock($vmid, $conf, 1);
3012             }
3013         });
3014     }
3015
3016     push @$cmd, '-S' if $conf->{freeze};
3017
3018     # set keyboard layout
3019     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3020     push @$cmd, '-k', $kb if $kb;
3021
3022     # enable sound
3023     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3024     #push @$cmd, '-soundhw', 'es1370';
3025     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3026
3027     if($conf->{agent}) {
3028         my $qgasocket = qmp_socket($vmid, 1);
3029         my $pciaddr = print_pci_addr("qga0", $bridges);
3030         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3031         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3032         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3033     }
3034
3035     my $spice_port;
3036
3037     if ($qxlnum) {
3038         if ($qxlnum > 1) {
3039             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3040                 for(my $i = 1; $i < $qxlnum; $i++){
3041                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3042                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3043                 }
3044             } else {
3045                 # assume other OS works like Linux
3046                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3047                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3048             }
3049         }
3050
3051         my $pciaddr = print_pci_addr("spice", $bridges);
3052
3053         $spice_port = PVE::Tools::next_spice_port();
3054
3055         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3056
3057         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3058         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3059         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3060     }
3061
3062     # enable balloon by default, unless explicitly disabled
3063     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3064         $pciaddr = print_pci_addr("balloon0", $bridges);
3065         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3066     }
3067
3068     if ($conf->{watchdog}) {
3069         my $wdopts = parse_watchdog($conf->{watchdog});
3070         $pciaddr = print_pci_addr("watchdog", $bridges);
3071         my $watchdog = $wdopts->{model} || 'i6300esb';
3072         push @$devices, '-device', "$watchdog$pciaddr";
3073         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3074     }
3075
3076     my $vollist = [];
3077     my $scsicontroller = {};
3078     my $ahcicontroller = {};
3079     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3080
3081     # Add iscsi initiator name if available
3082     if (my $initiator = get_initiator_name()) {
3083         push @$devices, '-iscsi', "initiator-name=$initiator";
3084     }
3085
3086     foreach_drive($conf, sub {
3087         my ($ds, $drive) = @_;
3088
3089         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3090             push @$vollist, $drive->{file};
3091         }
3092
3093         $use_virtio = 1 if $ds =~ m/^virtio/;
3094
3095         if (drive_is_cdrom ($drive)) {
3096             if ($bootindex_hash->{d}) {
3097                 $drive->{bootindex} = $bootindex_hash->{d};
3098                 $bootindex_hash->{d} += 1;
3099             }
3100         } else {
3101             if ($bootindex_hash->{c}) {
3102                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3103                 $bootindex_hash->{c} += 1;
3104             }
3105         }
3106
3107         if($drive->{interface} eq 'virtio'){
3108            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3109         }
3110
3111         if ($drive->{interface} eq 'scsi') {
3112
3113             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3114
3115             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3116             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3117             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr" if !$scsicontroller->{$controller};
3118             $scsicontroller->{$controller}=1;
3119         }
3120
3121         if ($drive->{interface} eq 'sata') {
3122            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3123            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3124            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3125            $ahcicontroller->{$controller}=1;
3126         }
3127
3128         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3129         push @$devices, '-drive',$drive_cmd;
3130         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3131     });
3132
3133     for (my $i = 0; $i < $MAX_NETS; $i++) {
3134          next if !$conf->{"net$i"};
3135          my $d = parse_net($conf->{"net$i"});
3136          next if !$d;
3137
3138          $use_virtio = 1 if $d->{model} eq 'virtio';
3139
3140          if ($bootindex_hash->{n}) {
3141             $d->{bootindex} = $bootindex_hash->{n};
3142             $bootindex_hash->{n} += 1;
3143          }
3144
3145          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3146          push @$devices, '-netdev', $netdevfull;
3147
3148          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3149          push @$devices, '-device', $netdevicefull;
3150     }
3151
3152     if (!$q35) {
3153         # add pci bridges
3154         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3155            $bridges->{1} = 1;
3156            $bridges->{2} = 1;
3157         }
3158
3159         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3160
3161         while (my ($k, $v) = each %$bridges) {
3162             $pciaddr = print_pci_addr("pci.$k");
3163             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3164         }
3165     }
3166
3167     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3168     # when the VM uses virtio devices.
3169     if (!$use_virtio && $have_ovz) {
3170
3171         my $cpuunits = defined($conf->{cpuunits}) ?
3172             $conf->{cpuunits} : $defaults->{cpuunits};
3173
3174         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3175
3176         # fixme: cpulimit is currently ignored
3177         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3178     }
3179
3180     # add custom args
3181     if ($conf->{args}) {
3182         my $aa = PVE::Tools::split_args($conf->{args});
3183         push @$cmd, @$aa;
3184     }
3185
3186     push @$cmd, @$devices;
3187     push @$cmd, '-rtc', join(',', @$rtcFlags)
3188         if scalar(@$rtcFlags);
3189     push @$cmd, '-machine', join(',', @$machineFlags)
3190         if scalar(@$machineFlags);
3191     push @$cmd, '-global', join(',', @$globalFlags)
3192         if scalar(@$globalFlags);
3193
3194     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3195 }
3196
3197 sub vnc_socket {
3198     my ($vmid) = @_;
3199     return "${var_run_tmpdir}/$vmid.vnc";
3200 }
3201
3202 sub spice_port {
3203     my ($vmid) = @_;
3204
3205     my $res = vm_mon_cmd($vmid, 'query-spice');
3206
3207     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3208 }
3209
3210 sub qmp_socket {
3211     my ($vmid, $qga) = @_;
3212     my $sockettype = $qga ? 'qga' : 'qmp';
3213     return "${var_run_tmpdir}/$vmid.$sockettype";
3214 }
3215
3216 sub pidfile_name {
3217     my ($vmid) = @_;
3218     return "${var_run_tmpdir}/$vmid.pid";
3219 }
3220
3221 sub vm_devices_list {
3222     my ($vmid) = @_;
3223
3224     my $res = vm_mon_cmd($vmid, 'query-pci');
3225     my $devices = {};
3226     foreach my $pcibus (@$res) {
3227         foreach my $device (@{$pcibus->{devices}}) {
3228             next if !$device->{'qdev_id'};
3229             if ($device->{'pci_bridge'}) {
3230                 $devices->{$device->{'qdev_id'}} = 1;
3231                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3232                     next if !$bridge_device->{'qdev_id'};
3233                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3234                     $devices->{$device->{'qdev_id'}}++;
3235                 }
3236             } else {
3237                 $devices->{$device->{'qdev_id'}} = 1;
3238             }
3239         }
3240     }
3241
3242     my $resblock = vm_mon_cmd($vmid, 'query-block');
3243     foreach my $block (@$resblock) {
3244         if($block->{device} =~ m/^drive-(\S+)/){
3245                 $devices->{$1} = 1;
3246         }
3247     }
3248
3249     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3250     foreach my $mice (@$resmice) {
3251         if ($mice->{name} eq 'QEMU HID Tablet') {
3252             $devices->{tablet} = 1;
3253             last;
3254         }
3255     }
3256
3257     return $devices;
3258 }
3259
3260 sub vm_deviceplug {
3261     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3262
3263     my $q35 = machine_type_is_q35($conf);
3264
3265     my $devices_list = vm_devices_list($vmid);
3266     return 1 if defined($devices_list->{$deviceid});
3267
3268     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3269
3270     if ($deviceid eq 'tablet') {
3271
3272         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3273
3274     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3275
3276         qemu_iothread_add($vmid, $deviceid, $device);
3277
3278         qemu_driveadd($storecfg, $vmid, $device);
3279         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3280
3281         qemu_deviceadd($vmid, $devicefull);
3282         eval { qemu_deviceaddverify($vmid, $deviceid); };
3283         if (my $err = $@) {
3284             eval { qemu_drivedel($vmid, $deviceid); };
3285             warn $@ if $@;
3286             die $err;
3287         }
3288
3289     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3290
3291         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3292         my $pciaddr = print_pci_addr($deviceid);
3293         my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3294
3295         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3296
3297         qemu_deviceadd($vmid, $devicefull);
3298         qemu_deviceaddverify($vmid, $deviceid);
3299
3300     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3301
3302         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3303         qemu_driveadd($storecfg, $vmid, $device);
3304
3305         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3306         eval { qemu_deviceadd($vmid, $devicefull); };
3307         if (my $err = $@) {
3308             eval { qemu_drivedel($vmid, $deviceid); };
3309             warn $@ if $@;
3310             die $err;
3311         }
3312
3313     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3314
3315         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3316         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3317         qemu_deviceadd($vmid, $netdevicefull);
3318         eval { qemu_deviceaddverify($vmid, $deviceid); };
3319         if (my $err = $@) {
3320             eval { qemu_netdevdel($vmid, $deviceid); };
3321             warn $@ if $@;
3322             die $err;
3323         }
3324
3325     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3326
3327         my $bridgeid = $2;
3328         my $pciaddr = print_pci_addr($deviceid);
3329         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3330
3331         qemu_deviceadd($vmid, $devicefull);
3332         qemu_deviceaddverify($vmid, $deviceid);
3333
3334     } else {
3335         die "can't hotplug device '$deviceid'\n";
3336     }
3337
3338     return 1;
3339 }
3340
3341 # fixme: this should raise exceptions on error!
3342 sub vm_deviceunplug {
3343     my ($vmid, $conf, $deviceid) = @_;
3344
3345     my $devices_list = vm_devices_list($vmid);
3346     return 1 if !defined($devices_list->{$deviceid});
3347
3348     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3349
3350     if ($deviceid eq 'tablet') {
3351
3352         qemu_devicedel($vmid, $deviceid);
3353
3354     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3355
3356         qemu_devicedel($vmid, $deviceid);
3357         qemu_devicedelverify($vmid, $deviceid);
3358         qemu_drivedel($vmid, $deviceid);
3359         qemu_iothread_del($conf, $vmid, $deviceid);
3360
3361     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3362
3363         qemu_devicedel($vmid, $deviceid);
3364         qemu_devicedelverify($vmid, $deviceid);
3365
3366     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3367
3368         qemu_devicedel($vmid, $deviceid);
3369         qemu_drivedel($vmid, $deviceid);
3370         qemu_deletescsihw($conf, $vmid, $deviceid);
3371
3372     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3373
3374         qemu_devicedel($vmid, $deviceid);
3375         qemu_devicedelverify($vmid, $deviceid);
3376         qemu_netdevdel($vmid, $deviceid);
3377
3378     } else {
3379         die "can't unplug device '$deviceid'\n";
3380     }
3381
3382     return 1;
3383 }
3384
3385 sub qemu_deviceadd {
3386     my ($vmid, $devicefull) = @_;
3387
3388     $devicefull = "driver=".$devicefull;
3389     my %options =  split(/[=,]/, $devicefull);
3390
3391     vm_mon_cmd($vmid, "device_add" , %options);
3392 }
3393
3394 sub qemu_devicedel {
3395     my ($vmid, $deviceid) = @_;
3396
3397     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3398 }
3399
3400 sub qemu_iothread_add {
3401     my($vmid, $deviceid, $device) = @_;
3402
3403     if ($device->{iothread}) {
3404         my $iothreads = vm_iothreads_list($vmid);
3405         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3406     }
3407 }
3408
3409 sub qemu_iothread_del {
3410     my($conf, $vmid, $deviceid) = @_;
3411
3412     my $device = parse_drive($deviceid, $conf->{$deviceid});
3413     if ($device->{iothread}) {
3414         my $iothreads = vm_iothreads_list($vmid);
3415         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3416     }
3417 }
3418
3419 sub qemu_objectadd {
3420     my($vmid, $objectid, $qomtype) = @_;
3421
3422     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3423
3424     return 1;
3425 }
3426
3427 sub qemu_objectdel {
3428     my($vmid, $objectid) = @_;
3429
3430     vm_mon_cmd($vmid, "object-del", id => $objectid);
3431
3432     return 1;
3433 }
3434
3435 sub qemu_driveadd {
3436     my ($storecfg, $vmid, $device) = @_;
3437
3438     my $drive = print_drive_full($storecfg, $vmid, $device);
3439     $drive =~ s/\\/\\\\/g;
3440     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3441
3442     # If the command succeeds qemu prints: "OK"
3443     return 1 if $ret =~ m/OK/s;
3444
3445     die "adding drive failed: $ret\n";
3446 }
3447
3448 sub qemu_drivedel {
3449     my($vmid, $deviceid) = @_;
3450
3451     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3452     $ret =~ s/^\s+//;
3453
3454     return 1 if $ret eq "";
3455
3456     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3457     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3458
3459     die "deleting drive $deviceid failed : $ret\n";
3460 }
3461
3462 sub qemu_deviceaddverify {
3463     my ($vmid, $deviceid) = @_;
3464
3465     for (my $i = 0; $i <= 5; $i++) {
3466          my $devices_list = vm_devices_list($vmid);
3467          return 1 if defined($devices_list->{$deviceid});
3468          sleep 1;
3469     }
3470
3471     die "error on hotplug device '$deviceid'\n";
3472 }
3473
3474
3475 sub qemu_devicedelverify {
3476     my ($vmid, $deviceid) = @_;
3477
3478     # need to verify that the device is correctly removed as device_del
3479     # is async and empty return is not reliable
3480
3481     for (my $i = 0; $i <= 5; $i++) {
3482          my $devices_list = vm_devices_list($vmid);
3483          return 1 if !defined($devices_list->{$deviceid});
3484          sleep 1;
3485     }
3486
3487     die "error on hot-unplugging device '$deviceid'\n";
3488 }
3489
3490 sub qemu_findorcreatescsihw {
3491     my ($storecfg, $conf, $vmid, $device) = @_;
3492
3493     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3494
3495     my $scsihwid="$controller_prefix$controller";
3496     my $devices_list = vm_devices_list($vmid);
3497
3498     if(!defined($devices_list->{$scsihwid})) {
3499         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
3500     }
3501
3502     return 1;
3503 }
3504
3505 sub qemu_deletescsihw {
3506     my ($conf, $vmid, $opt) = @_;
3507
3508     my $device = parse_drive($opt, $conf->{$opt});
3509
3510     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^virtio-scsi-single/)) {
3511         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3512         return 1;
3513     }
3514
3515     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3516
3517     my $devices_list = vm_devices_list($vmid);
3518     foreach my $opt (keys %{$devices_list}) {
3519         if (PVE::QemuServer::valid_drivename($opt)) {
3520             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3521             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3522                 return 1;
3523             }
3524         }
3525     }
3526
3527     my $scsihwid="scsihw$controller";
3528
3529     vm_deviceunplug($vmid, $conf, $scsihwid);
3530
3531     return 1;
3532 }
3533
3534 sub qemu_add_pci_bridge {
3535     my ($storecfg, $conf, $vmid, $device) = @_;
3536
3537     my $bridges = {};
3538
3539     my $bridgeid;
3540
3541     print_pci_addr($device, $bridges);
3542
3543     while (my ($k, $v) = each %$bridges) {
3544         $bridgeid = $k;
3545     }
3546     return 1 if !defined($bridgeid) || $bridgeid < 1;
3547
3548     my $bridge = "pci.$bridgeid";
3549     my $devices_list = vm_devices_list($vmid);
3550
3551     if (!defined($devices_list->{$bridge})) {
3552         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3553     }
3554
3555     return 1;
3556 }
3557
3558 sub qemu_set_link_status {
3559     my ($vmid, $device, $up) = @_;
3560
3561     vm_mon_cmd($vmid, "set_link", name => $device,
3562                up => $up ? JSON::true : JSON::false);
3563 }
3564
3565 sub qemu_netdevadd {
3566     my ($vmid, $conf, $device, $deviceid) = @_;
3567
3568     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3569     my %options =  split(/[=,]/, $netdev);
3570
3571     vm_mon_cmd($vmid, "netdev_add",  %options);
3572     return 1;
3573 }
3574
3575 sub qemu_netdevdel {
3576     my ($vmid, $deviceid) = @_;
3577
3578     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3579 }
3580
3581 sub qemu_cpu_hotplug {
3582     my ($vmid, $conf, $vcpus) = @_;
3583
3584     my $sockets = 1;
3585     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3586     $sockets = $conf->{sockets} if  $conf->{sockets};
3587     my $cores = $conf->{cores} || 1;
3588     my $maxcpus = $sockets * $cores;
3589
3590     $vcpus = $maxcpus if !$vcpus;
3591
3592     die "you can't add more vcpus than maxcpus\n"
3593         if $vcpus > $maxcpus;
3594
3595     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3596     die "online cpu unplug is not yet possible\n"
3597         if $vcpus < $currentvcpus;
3598
3599     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3600     die "vcpus in running vm is different than configuration\n"
3601         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3602
3603     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3604         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3605     }
3606 }
3607
3608 sub qemu_memory_hotplug {
3609     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3610
3611     return $value if !check_running($vmid);
3612
3613     my $memory = $conf->{memory} || $defaults->{memory};
3614     $value = $defaults->{memory} if !$value;
3615     return $value if $value == $memory;
3616
3617     my $static_memory = $STATICMEM;
3618     my $dimm_memory = $memory - $static_memory;
3619
3620     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3621     die "memory unplug is not yet available" if $value < $memory;
3622     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3623
3624
3625     my $sockets = 1;
3626     $sockets = $conf->{sockets} if $conf->{sockets};
3627
3628     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3629         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3630
3631             return if $current_size <= $conf->{memory};
3632
3633             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3634             if (my $err = $@) {
3635                 eval { qemu_objectdel($vmid, "mem-$name"); };
3636                 die $err;
3637             }
3638
3639             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3640             if (my $err = $@) {
3641                 eval { qemu_objectdel($vmid, "mem-$name"); };
3642                 die $err;
3643             }
3644             #update conf after each succesful module hotplug
3645             $conf->{memory} = $current_size;
3646             update_config_nolock($vmid, $conf, 1);
3647     });
3648 }
3649
3650 sub qemu_block_set_io_throttle {
3651     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3652
3653     return if !check_running($vmid) ;
3654
3655     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3656
3657 }
3658
3659 # old code, only used to shutdown old VM after update
3660 sub __read_avail {
3661     my ($fh, $timeout) = @_;
3662
3663     my $sel = new IO::Select;
3664     $sel->add($fh);
3665
3666     my $res = '';
3667     my $buf;
3668
3669     my @ready;
3670     while (scalar (@ready = $sel->can_read($timeout))) {
3671         my $count;
3672         if ($count = $fh->sysread($buf, 8192)) {
3673             if ($buf =~ /^(.*)\(qemu\) $/s) {
3674                 $res .= $1;
3675                 last;
3676             } else {
3677                 $res .= $buf;
3678             }
3679         } else {
3680             if (!defined($count)) {
3681                 die "$!\n";
3682             }
3683             last;
3684         }
3685     }
3686
3687     die "monitor read timeout\n" if !scalar(@ready);
3688
3689     return $res;
3690 }
3691
3692 # old code, only used to shutdown old VM after update
3693 sub vm_monitor_command {
3694     my ($vmid, $cmdstr, $nocheck) = @_;
3695
3696     my $res;
3697
3698     eval {
3699         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3700
3701         my $sname = "${var_run_tmpdir}/$vmid.mon";
3702
3703         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3704             die "unable to connect to VM $vmid socket - $!\n";
3705
3706         my $timeout = 3;
3707
3708         # hack: migrate sometime blocks the monitor (when migrate_downtime
3709         # is set)
3710         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3711             $timeout = 60*60; # 1 hour
3712         }
3713
3714         # read banner;
3715         my $data = __read_avail($sock, $timeout);
3716
3717         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3718             die "got unexpected qemu monitor banner\n";
3719         }
3720
3721         my $sel = new IO::Select;
3722         $sel->add($sock);
3723
3724         if (!scalar(my @ready = $sel->can_write($timeout))) {
3725             die "monitor write error - timeout";
3726         }
3727
3728         my $fullcmd = "$cmdstr\r";
3729
3730         # syslog('info', "VM $vmid monitor command: $cmdstr");
3731
3732         my $b;
3733         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3734             die "monitor write error - $!";
3735         }
3736
3737         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3738
3739         $timeout = 20;
3740
3741         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3742             $timeout = 60*60; # 1 hour
3743         } elsif ($cmdstr =~ m/^(eject|change)/) {
3744             $timeout = 60; # note: cdrom mount command is slow
3745         }
3746         if ($res = __read_avail($sock, $timeout)) {
3747
3748             my @lines = split("\r?\n", $res);
3749
3750             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3751
3752             $res = join("\n", @lines);
3753             $res .= "\n";
3754         }
3755     };
3756
3757     my $err = $@;
3758
3759     if ($err) {
3760         syslog("err", "VM $vmid monitor command failed - $err");
3761         die $err;
3762     }
3763
3764     return $res;
3765 }
3766
3767 sub qemu_block_resize {
3768     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3769
3770     my $running = check_running($vmid);
3771
3772     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3773
3774     return if !$running;
3775
3776     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3777
3778 }
3779
3780 sub qemu_volume_snapshot {
3781     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3782
3783     my $running = check_running($vmid);
3784
3785     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3786
3787     return if !$running;
3788
3789     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3790
3791 }
3792
3793 sub qemu_volume_snapshot_delete {
3794     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3795
3796     my $running = check_running($vmid);
3797
3798     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3799
3800     return if !$running;
3801
3802     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3803 }
3804
3805 sub set_migration_caps {
3806     my ($vmid) = @_;
3807
3808     my $cap_ref = [];
3809
3810     my $enabled_cap = {
3811         "auto-converge" => 1,
3812         "xbzrle" => 0,
3813         "x-rdma-pin-all" => 0,
3814         "zero-blocks" => 0,
3815     };
3816
3817     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3818
3819     for my $supported_capability (@$supported_capabilities) {
3820         push @$cap_ref, {
3821             capability => $supported_capability->{capability},
3822             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3823         };
3824     }
3825
3826     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3827 }
3828
3829 my $fast_plug_option = {
3830     'lock' => 1,
3831     'name' => 1,
3832     'onboot' => 1,
3833     'shares' => 1,
3834     'startup' => 1,
3835 };
3836
3837 # hotplug changes in [PENDING]
3838 # $selection hash can be used to only apply specified options, for
3839 # example: { cores => 1 } (only apply changed 'cores')
3840 # $errors ref is used to return error messages
3841 sub vmconfig_hotplug_pending {
3842     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3843
3844     my $defaults = load_defaults();
3845
3846     # commit values which do not have any impact on running VM first
3847     # Note: those option cannot raise errors, we we do not care about
3848     # $selection and always apply them.
3849
3850     my $add_error = sub {
3851         my ($opt, $msg) = @_;
3852         $errors->{$opt} = "hotplug problem - $msg";
3853     };
3854
3855     my $changes = 0;
3856     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3857         if ($fast_plug_option->{$opt}) {
3858             $conf->{$opt} = $conf->{pending}->{$opt};
3859             delete $conf->{pending}->{$opt};
3860             $changes = 1;
3861         }
3862     }
3863
3864     if ($changes) {
3865         update_config_nolock($vmid, $conf, 1);
3866         $conf = load_config($vmid); # update/reload
3867     }
3868
3869     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3870
3871     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3872     foreach my $opt (@delete) {
3873         next if $selection && !$selection->{$opt};
3874         eval {
3875             if ($opt eq 'hotplug') {
3876                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3877             } elsif ($opt eq 'tablet') {
3878                 die "skip\n" if !$hotplug_features->{usb};
3879                 if ($defaults->{tablet}) {
3880                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3881                 } else {
3882                     vm_deviceunplug($vmid, $conf, $opt);
3883                 }
3884             } elsif ($opt eq 'vcpus') {
3885                 die "skip\n" if !$hotplug_features->{cpu};
3886                 qemu_cpu_hotplug($vmid, $conf, undef);
3887             } elsif ($opt eq 'balloon') {
3888                 # enable balloon device is not hotpluggable
3889                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3890             } elsif ($fast_plug_option->{$opt}) {
3891                 # do nothing
3892             } elsif ($opt =~ m/^net(\d+)$/) {
3893                 die "skip\n" if !$hotplug_features->{network};
3894                 vm_deviceunplug($vmid, $conf, $opt);
3895             } elsif (valid_drivename($opt)) {
3896                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3897                 vm_deviceunplug($vmid, $conf, $opt);
3898                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3899             } elsif ($opt =~ m/^memory$/) {
3900                 die "skip\n" if !$hotplug_features->{memory};
3901                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3902             } else {
3903                 die "skip\n";
3904             }
3905         };
3906         if (my $err = $@) {
3907             &$add_error($opt, $err) if $err ne "skip\n";
3908         } else {
3909             # save new config if hotplug was successful
3910             delete $conf->{$opt};
3911             vmconfig_undelete_pending_option($conf, $opt);
3912             update_config_nolock($vmid, $conf, 1);
3913             $conf = load_config($vmid); # update/reload
3914         }
3915     }
3916
3917     foreach my $opt (keys %{$conf->{pending}}) {
3918         next if $selection && !$selection->{$opt};
3919         my $value = $conf->{pending}->{$opt};
3920         eval {
3921             if ($opt eq 'hotplug') {
3922                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3923             } elsif ($opt eq 'tablet') {
3924                 die "skip\n" if !$hotplug_features->{usb};
3925                 if ($value == 1) {
3926                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3927                 } elsif ($value == 0) {
3928                     vm_deviceunplug($vmid, $conf, $opt);
3929                 }
3930             } elsif ($opt eq 'vcpus') {
3931                 die "skip\n" if !$hotplug_features->{cpu};
3932                 qemu_cpu_hotplug($vmid, $conf, $value);
3933             } elsif ($opt eq 'balloon') {
3934                 # enable/disable balloning device is not hotpluggable
3935                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3936                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3937                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3938
3939                 # allow manual ballooning if shares is set to zero
3940                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3941                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3942                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3943                 }
3944             } elsif ($opt =~ m/^net(\d+)$/) {
3945                 # some changes can be done without hotplug
3946                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3947                                     $vmid, $opt, $value);
3948             } elsif (valid_drivename($opt)) {
3949                 # some changes can be done without hotplug
3950                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3951                                      $vmid, $opt, $value, 1);
3952             } elsif ($opt =~ m/^memory$/) { #dimms
3953                 die "skip\n" if !$hotplug_features->{memory};
3954                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3955             } else {
3956                 die "skip\n";  # skip non-hot-pluggable options
3957             }
3958         };
3959         if (my $err = $@) {
3960             &$add_error($opt, $err) if $err ne "skip\n";
3961         } else {
3962             # save new config if hotplug was successful
3963             $conf->{$opt} = $value;
3964             delete $conf->{pending}->{$opt};
3965             update_config_nolock($vmid, $conf, 1);
3966             $conf = load_config($vmid); # update/reload
3967         }
3968     }
3969 }
3970
3971 sub vmconfig_apply_pending {
3972     my ($vmid, $conf, $storecfg) = @_;
3973
3974     # cold plug
3975
3976     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3977     foreach my $opt (@delete) { # delete
3978         die "internal error" if $opt =~ m/^unused/;
3979         $conf = load_config($vmid); # update/reload
3980         if (!defined($conf->{$opt})) {
3981             vmconfig_undelete_pending_option($conf, $opt);
3982             update_config_nolock($vmid, $conf, 1);
3983         } elsif (valid_drivename($opt)) {
3984             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3985             vmconfig_undelete_pending_option($conf, $opt);
3986             delete $conf->{$opt};
3987             update_config_nolock($vmid, $conf, 1);
3988         } else {
3989             vmconfig_undelete_pending_option($conf, $opt);
3990             delete $conf->{$opt};
3991             update_config_nolock($vmid, $conf, 1);
3992         }
3993     }
3994
3995     $conf = load_config($vmid); # update/reload
3996
3997     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3998         $conf = load_config($vmid); # update/reload
3999
4000         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4001             # skip if nothing changed
4002         } elsif (valid_drivename($opt)) {
4003             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4004                 if defined($conf->{$opt});
4005             $conf->{$opt} = $conf->{pending}->{$opt};
4006         } else {
4007             $conf->{$opt} = $conf->{pending}->{$opt};
4008         }
4009
4010         delete $conf->{pending}->{$opt};
4011         update_config_nolock($vmid, $conf, 1);
4012     }
4013 }
4014
4015 my $safe_num_ne = sub {
4016     my ($a, $b) = @_;
4017
4018     return 0 if !defined($a) && !defined($b);
4019     return 1 if !defined($a);
4020     return 1 if !defined($b);
4021
4022     return $a != $b;
4023 };
4024
4025 my $safe_string_ne = sub {
4026     my ($a, $b) = @_;
4027
4028     return 0 if !defined($a) && !defined($b);
4029     return 1 if !defined($a);
4030     return 1 if !defined($b);
4031
4032     return $a ne $b;
4033 };
4034
4035 sub vmconfig_update_net {
4036     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4037
4038     my $newnet = parse_net($value);
4039
4040     if ($conf->{$opt}) {
4041         my $oldnet = parse_net($conf->{$opt});
4042
4043         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4044             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4045             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4046             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4047
4048             # for non online change, we try to hot-unplug
4049             die "skip\n" if !$hotplug;
4050             vm_deviceunplug($vmid, $conf, $opt);
4051         } else {
4052
4053             die "internal error" if $opt !~ m/net(\d+)/;
4054             my $iface = "tap${vmid}i$1";
4055
4056             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4057                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4058             }
4059
4060             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4061                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4062                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4063                 PVE::Network::tap_unplug($iface);
4064                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4065             }
4066
4067             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4068                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4069             }
4070
4071             return 1;
4072         }
4073     }
4074
4075     if ($hotplug) {
4076         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4077     } else {
4078         die "skip\n";
4079     }
4080 }
4081
4082 sub vmconfig_update_disk {
4083     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4084
4085     # fixme: do we need force?
4086
4087     my $drive = parse_drive($opt, $value);
4088
4089     if ($conf->{$opt}) {
4090
4091         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4092
4093             my $media = $drive->{media} || 'disk';
4094             my $oldmedia = $old_drive->{media} || 'disk';
4095             die "unable to change media type\n" if $media ne $oldmedia;
4096
4097             if (!drive_is_cdrom($old_drive)) {
4098
4099                 if ($drive->{file} ne $old_drive->{file}) {
4100
4101                     die "skip\n" if !$hotplug;
4102
4103                     # unplug and register as unused
4104                     vm_deviceunplug($vmid, $conf, $opt);
4105                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4106
4107                 } else {
4108                     # update existing disk
4109
4110                     # skip non hotpluggable value
4111                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4112                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4113                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4114                         die "skip\n";
4115                     }
4116
4117                     # apply throttle
4118                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4119                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4120                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4121                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4122                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4123                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4124                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4125                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4126                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4127                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4128                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4129                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4130
4131                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4132                                                    ($drive->{mbps} || 0)*1024*1024,
4133                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4134                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4135                                                    $drive->{iops} || 0,
4136                                                    $drive->{iops_rd} || 0,
4137                                                    $drive->{iops_wr} || 0,
4138                                                    ($drive->{mbps_max} || 0)*1024*1024,
4139                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4140                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4141                                                    $drive->{iops_max} || 0,
4142                                                    $drive->{iops_rd_max} || 0,
4143                                                    $drive->{iops_wr_max} || 0);
4144
4145                     }
4146
4147                     return 1;
4148                 }
4149
4150             } else { # cdrom
4151
4152                 if ($drive->{file} eq 'none') {
4153                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4154                 } else {
4155                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4156                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4157                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4158                 }
4159
4160                 return 1;
4161             }
4162         }
4163     }
4164
4165     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4166     # hotplug new disks
4167     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4168 }
4169
4170 sub vm_start {
4171     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4172
4173     lock_config($vmid, sub {
4174         my $conf = load_config($vmid, $migratedfrom);
4175
4176         die "you can't start a vm if it's a template\n" if is_template($conf);
4177
4178         check_lock($conf) if !$skiplock;
4179
4180         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4181
4182         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4183             vmconfig_apply_pending($vmid, $conf, $storecfg);
4184             $conf = load_config($vmid); # update/reload
4185         }
4186
4187         my $defaults = load_defaults();
4188
4189         # set environment variable useful inside network script
4190         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4191
4192         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4193
4194         my $migrate_port = 0;
4195         my $migrate_uri;
4196         if ($statefile) {
4197             if ($statefile eq 'tcp') {
4198                 my $localip = "localhost";
4199                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4200                 if ($datacenterconf->{migration_unsecure}) {
4201                         my $nodename = PVE::INotify::nodename();
4202                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4203                 }
4204                 $migrate_port = PVE::Tools::next_migrate_port();
4205                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4206                 push @$cmd, '-incoming', $migrate_uri;
4207                 push @$cmd, '-S';
4208             } else {
4209                 push @$cmd, '-loadstate', $statefile;
4210             }
4211         } elsif ($paused) {
4212             push @$cmd, '-S';
4213         }
4214
4215         # host pci devices
4216         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4217           my $d = parse_hostpci($conf->{"hostpci$i"});
4218           next if !$d;
4219           my $pcidevices = $d->{pciid};
4220           foreach my $pcidevice (@$pcidevices) {
4221                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4222
4223                 my $info = pci_device_info("0000:$pciid");
4224                 die "IOMMU not present\n" if !check_iommu_support();
4225                 die "no pci device info for device '$pciid'\n" if !$info;
4226
4227                 if ($d->{driver} && $d->{driver} eq "vfio") {
4228                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4229                 } else {
4230                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4231                 }
4232
4233                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4234           }
4235         }
4236
4237         PVE::Storage::activate_volumes($storecfg, $vollist);
4238
4239         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4240                     umask => 0077); };
4241         my $err = $@;
4242         die "start failed: $err" if $err;
4243
4244         print "migration listens on $migrate_uri\n" if $migrate_uri;
4245
4246         if ($statefile && $statefile ne 'tcp')  {
4247             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4248             warn $@ if $@;
4249         }
4250
4251         if ($migratedfrom) {
4252
4253             eval {
4254                 set_migration_caps($vmid);
4255             };
4256             warn $@ if $@;
4257
4258             if ($spice_port) {
4259                 print "spice listens on port $spice_port\n";
4260                 if ($spice_ticket) {
4261                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4262                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4263                 }
4264             }
4265
4266         } else {
4267
4268             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4269                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4270                     if $conf->{balloon};
4271             }
4272
4273             foreach my $opt (keys %$conf) {
4274                 next if $opt !~  m/^net\d+$/;
4275                 my $nicconf = parse_net($conf->{$opt});
4276                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4277             }
4278         }
4279
4280         vm_mon_cmd_nocheck($vmid, 'qom-set',
4281                     path => "machine/peripheral/balloon0",
4282                     property => "guest-stats-polling-interval",
4283                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4284
4285     });
4286 }
4287
4288 sub vm_mon_cmd {
4289     my ($vmid, $execute, %params) = @_;
4290
4291     my $cmd = { execute => $execute, arguments => \%params };
4292     vm_qmp_command($vmid, $cmd);
4293 }
4294
4295 sub vm_mon_cmd_nocheck {
4296     my ($vmid, $execute, %params) = @_;
4297
4298     my $cmd = { execute => $execute, arguments => \%params };
4299     vm_qmp_command($vmid, $cmd, 1);
4300 }
4301
4302 sub vm_qmp_command {
4303     my ($vmid, $cmd, $nocheck) = @_;
4304
4305     my $res;
4306
4307     my $timeout;
4308     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4309         $timeout = $cmd->{arguments}->{timeout};
4310         delete $cmd->{arguments}->{timeout};
4311     }
4312
4313     eval {
4314         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4315         my $sname = qmp_socket($vmid);
4316         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4317             my $qmpclient = PVE::QMPClient->new();
4318
4319             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4320         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4321             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4322                 if scalar(%{$cmd->{arguments}});
4323             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4324         } else {
4325             die "unable to open monitor socket\n";
4326         }
4327     };
4328     if (my $err = $@) {
4329         syslog("err", "VM $vmid qmp command failed - $err");
4330         die $err;
4331     }
4332
4333     return $res;
4334 }
4335
4336 sub vm_human_monitor_command {
4337     my ($vmid, $cmdline) = @_;
4338
4339     my $res;
4340
4341     my $cmd = {
4342         execute => 'human-monitor-command',
4343         arguments => { 'command-line' => $cmdline},
4344     };
4345
4346     return vm_qmp_command($vmid, $cmd);
4347 }
4348
4349 sub vm_commandline {
4350     my ($storecfg, $vmid) = @_;
4351
4352     my $conf = load_config($vmid);
4353
4354     my $defaults = load_defaults();
4355
4356     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4357
4358     return join(' ', @$cmd);
4359 }
4360
4361 sub vm_reset {
4362     my ($vmid, $skiplock) = @_;
4363
4364     lock_config($vmid, sub {
4365
4366         my $conf = load_config($vmid);
4367
4368         check_lock($conf) if !$skiplock;
4369
4370         vm_mon_cmd($vmid, "system_reset");
4371     });
4372 }
4373
4374 sub get_vm_volumes {
4375     my ($conf) = @_;
4376
4377     my $vollist = [];
4378     foreach_volid($conf, sub {
4379         my ($volid, $is_cdrom) = @_;
4380
4381         return if $volid =~ m|^/|;
4382
4383         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4384         return if !$sid;
4385
4386         push @$vollist, $volid;
4387     });
4388
4389     return $vollist;
4390 }
4391
4392 sub vm_stop_cleanup {
4393     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4394
4395     eval {
4396         fairsched_rmnod($vmid); # try to destroy group
4397
4398         if (!$keepActive) {
4399             my $vollist = get_vm_volumes($conf);
4400             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4401         }
4402
4403         foreach my $ext (qw(mon qmp pid vnc qga)) {
4404             unlink "/var/run/qemu-server/${vmid}.$ext";
4405         }
4406
4407         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4408     };
4409     warn $@ if $@; # avoid errors - just warn
4410 }
4411
4412 # Note: use $nockeck to skip tests if VM configuration file exists.
4413 # We need that when migration VMs to other nodes (files already moved)
4414 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4415 sub vm_stop {
4416     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4417
4418     $force = 1 if !defined($force) && !$shutdown;
4419
4420     if ($migratedfrom){
4421         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4422         kill 15, $pid if $pid;
4423         my $conf = load_config($vmid, $migratedfrom);
4424         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4425         return;
4426     }
4427
4428     lock_config($vmid, sub {
4429
4430         my $pid = check_running($vmid, $nocheck);
4431         return if !$pid;
4432
4433         my $conf;
4434         if (!$nocheck) {
4435             $conf = load_config($vmid);
4436             check_lock($conf) if !$skiplock;
4437             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4438                 my $opts = parse_startup($conf->{startup});
4439                 $timeout = $opts->{down} if $opts->{down};
4440             }
4441         }
4442
4443         $timeout = 60 if !defined($timeout);
4444
4445         eval {
4446             if ($shutdown) {
4447                 if (defined($conf) && $conf->{agent}) {
4448                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4449                 } else {
4450                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4451                 }
4452             } else {
4453                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4454             }
4455         };
4456         my $err = $@;
4457
4458         if (!$err) {
4459             my $count = 0;
4460             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4461                 $count++;
4462                 sleep 1;
4463             }
4464
4465             if ($count >= $timeout) {
4466                 if ($force) {
4467                     warn "VM still running - terminating now with SIGTERM\n";
4468                     kill 15, $pid;
4469                 } else {
4470                     die "VM quit/powerdown failed - got timeout\n";
4471                 }
4472             } else {
4473                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4474                 return;
4475             }
4476         } else {
4477             if ($force) {
4478                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4479                 kill 15, $pid;
4480             } else {
4481                 die "VM quit/powerdown failed\n";
4482             }
4483         }
4484
4485         # wait again
4486         $timeout = 10;
4487
4488         my $count = 0;
4489         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4490             $count++;
4491             sleep 1;
4492         }
4493
4494         if ($count >= $timeout) {
4495             warn "VM still running - terminating now with SIGKILL\n";
4496             kill 9, $pid;
4497             sleep 1;
4498         }
4499
4500         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4501    });
4502 }
4503
4504 sub vm_suspend {
4505     my ($vmid, $skiplock) = @_;
4506
4507     lock_config($vmid, sub {
4508
4509         my $conf = load_config($vmid);
4510
4511         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4512
4513         vm_mon_cmd($vmid, "stop");
4514     });
4515 }
4516
4517 sub vm_resume {
4518     my ($vmid, $skiplock) = @_;
4519
4520     lock_config($vmid, sub {
4521
4522         my $conf = load_config($vmid);
4523
4524         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4525
4526         vm_mon_cmd($vmid, "cont");
4527     });
4528 }
4529
4530 sub vm_sendkey {
4531     my ($vmid, $skiplock, $key) = @_;
4532
4533     lock_config($vmid, sub {
4534
4535         my $conf = load_config($vmid);
4536
4537         # there is no qmp command, so we use the human monitor command
4538         vm_human_monitor_command($vmid, "sendkey $key");
4539     });
4540 }
4541
4542 sub vm_destroy {
4543     my ($storecfg, $vmid, $skiplock) = @_;
4544
4545     lock_config($vmid, sub {
4546
4547         my $conf = load_config($vmid);
4548
4549         check_lock($conf) if !$skiplock;
4550
4551         if (!check_running($vmid)) {
4552             fairsched_rmnod($vmid); # try to destroy group
4553             destroy_vm($storecfg, $vmid);
4554         } else {
4555             die "VM $vmid is running - destroy failed\n";
4556         }
4557     });
4558 }
4559
4560 # pci helpers
4561
4562 sub file_write {
4563     my ($filename, $buf) = @_;
4564
4565     my $fh = IO::File->new($filename, "w");
4566     return undef if !$fh;
4567
4568     my $res = print $fh $buf;
4569
4570     $fh->close();
4571
4572     return $res;
4573 }
4574
4575 sub pci_device_info {
4576     my ($name) = @_;
4577
4578     my $res;
4579
4580     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4581     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4582
4583     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4584     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4585
4586     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4587     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4588
4589     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4590     return undef if !defined($product) || $product !~ s/^0x//;
4591
4592     $res = {
4593         name => $name,
4594         vendor => $vendor,
4595         product => $product,
4596         domain => $domain,
4597         bus => $bus,
4598         slot => $slot,
4599         func => $func,
4600         irq => $irq,
4601         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4602     };
4603
4604     return $res;
4605 }
4606
4607 sub pci_dev_reset {
4608     my ($dev) = @_;
4609
4610     my $name = $dev->{name};
4611
4612     my $fn = "$pcisysfs/devices/$name/reset";
4613
4614     return file_write($fn, "1");
4615 }
4616
4617 sub pci_dev_bind_to_stub {
4618     my ($dev) = @_;
4619
4620     my $name = $dev->{name};
4621
4622     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4623     return 1 if -d $testdir;
4624
4625     my $data = "$dev->{vendor} $dev->{product}";
4626     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4627
4628     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4629     if (!file_write($fn, $name)) {
4630         return undef if -f $fn;
4631     }
4632
4633     $fn = "$pcisysfs/drivers/pci-stub/bind";
4634     if (! -d $testdir) {
4635         return undef if !file_write($fn, $name);
4636     }
4637
4638     return -d $testdir;
4639 }
4640
4641 sub pci_dev_bind_to_vfio {
4642     my ($dev) = @_;
4643
4644     my $name = $dev->{name};
4645
4646     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4647
4648     if (!-d $vfio_basedir) {
4649         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4650     }
4651     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4652
4653     my $testdir = "$vfio_basedir/$name";
4654     return 1 if -d $testdir;
4655
4656     my $data = "$dev->{vendor} $dev->{product}";
4657     return undef if !file_write("$vfio_basedir/new_id", $data);
4658
4659     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4660     if (!file_write($fn, $name)) {
4661         return undef if -f $fn;
4662     }
4663
4664     $fn = "$vfio_basedir/bind";
4665     if (! -d $testdir) {
4666         return undef if !file_write($fn, $name);
4667     }
4668
4669     return -d $testdir;
4670 }
4671
4672 sub pci_dev_group_bind_to_vfio {
4673     my ($pciid) = @_;
4674
4675     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4676
4677     if (!-d $vfio_basedir) {
4678         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4679     }
4680     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4681
4682     # get IOMMU group devices
4683     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4684       my @devs = grep /^0000:/, readdir($D);
4685     closedir($D);
4686
4687     foreach my $pciid (@devs) {
4688         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4689
4690         # pci bridges, switches or root ports are not supported
4691         # they have a pci_bus subdirectory so skip them
4692         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4693
4694         my $info = pci_device_info($1);
4695         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4696     }
4697
4698     return 1;
4699 }
4700
4701 sub print_pci_addr {
4702     my ($id, $bridges) = @_;
4703
4704     my $res = '';
4705     my $devices = {
4706         piix3 => { bus => 0, addr => 1 },
4707         #addr2 : first videocard
4708         balloon0 => { bus => 0, addr => 3 },
4709         watchdog => { bus => 0, addr => 4 },
4710         scsihw0 => { bus => 0, addr => 5 },
4711         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4712         scsihw1 => { bus => 0, addr => 6 },
4713         ahci0 => { bus => 0, addr => 7 },
4714         qga0 => { bus => 0, addr => 8 },
4715         spice => { bus => 0, addr => 9 },
4716         virtio0 => { bus => 0, addr => 10 },
4717         virtio1 => { bus => 0, addr => 11 },
4718         virtio2 => { bus => 0, addr => 12 },
4719         virtio3 => { bus => 0, addr => 13 },
4720         virtio4 => { bus => 0, addr => 14 },
4721         virtio5 => { bus => 0, addr => 15 },
4722         hostpci0 => { bus => 0, addr => 16 },
4723         hostpci1 => { bus => 0, addr => 17 },
4724         net0 => { bus => 0, addr => 18 },
4725         net1 => { bus => 0, addr => 19 },
4726         net2 => { bus => 0, addr => 20 },
4727         net3 => { bus => 0, addr => 21 },
4728         net4 => { bus => 0, addr => 22 },
4729         net5 => { bus => 0, addr => 23 },
4730         vga1 => { bus => 0, addr => 24 },
4731         vga2 => { bus => 0, addr => 25 },
4732         vga3 => { bus => 0, addr => 26 },
4733         hostpci2 => { bus => 0, addr => 27 },
4734         hostpci3 => { bus => 0, addr => 28 },
4735         #addr29 : usb-host (pve-usb.cfg)
4736         'pci.1' => { bus => 0, addr => 30 },
4737         'pci.2' => { bus => 0, addr => 31 },
4738         'net6' => { bus => 1, addr => 1 },
4739         'net7' => { bus => 1, addr => 2 },
4740         'net8' => { bus => 1, addr => 3 },
4741         'net9' => { bus => 1, addr => 4 },
4742         'net10' => { bus => 1, addr => 5 },
4743         'net11' => { bus => 1, addr => 6 },
4744         'net12' => { bus => 1, addr => 7 },
4745         'net13' => { bus => 1, addr => 8 },
4746         'net14' => { bus => 1, addr => 9 },
4747         'net15' => { bus => 1, addr => 10 },
4748         'net16' => { bus => 1, addr => 11 },
4749         'net17' => { bus => 1, addr => 12 },
4750         'net18' => { bus => 1, addr => 13 },
4751         'net19' => { bus => 1, addr => 14 },
4752         'net20' => { bus => 1, addr => 15 },
4753         'net21' => { bus => 1, addr => 16 },
4754         'net22' => { bus => 1, addr => 17 },
4755         'net23' => { bus => 1, addr => 18 },
4756         'net24' => { bus => 1, addr => 19 },
4757         'net25' => { bus => 1, addr => 20 },
4758         'net26' => { bus => 1, addr => 21 },
4759         'net27' => { bus => 1, addr => 22 },
4760         'net28' => { bus => 1, addr => 23 },
4761         'net29' => { bus => 1, addr => 24 },
4762         'net30' => { bus => 1, addr => 25 },
4763         'net31' => { bus => 1, addr => 26 },
4764         'virtio6' => { bus => 2, addr => 1 },
4765         'virtio7' => { bus => 2, addr => 2 },
4766         'virtio8' => { bus => 2, addr => 3 },
4767         'virtio9' => { bus => 2, addr => 4 },
4768         'virtio10' => { bus => 2, addr => 5 },
4769         'virtio11' => { bus => 2, addr => 6 },
4770         'virtio12' => { bus => 2, addr => 7 },
4771         'virtio13' => { bus => 2, addr => 8 },
4772         'virtio14' => { bus => 2, addr => 9 },
4773         'virtio15' => { bus => 2, addr => 10 },
4774         'virtioscsi0' => { bus => 3, addr => 1 },
4775         'virtioscsi1' => { bus => 3, addr => 2 },
4776         'virtioscsi2' => { bus => 3, addr => 3 },
4777         'virtioscsi3' => { bus => 3, addr => 4 },
4778         'virtioscsi4' => { bus => 3, addr => 5 },
4779         'virtioscsi5' => { bus => 3, addr => 6 },
4780         'virtioscsi6' => { bus => 3, addr => 7 },
4781         'virtioscsi7' => { bus => 3, addr => 8 },
4782         'virtioscsi8' => { bus => 3, addr => 9 },
4783         'virtioscsi9' => { bus => 3, addr => 10 },
4784         'virtioscsi10' => { bus => 3, addr => 11 },
4785         'virtioscsi11' => { bus => 3, addr => 12 },
4786         'virtioscsi12' => { bus => 3, addr => 13 },
4787         'virtioscsi13' => { bus => 3, addr => 14 },
4788         'virtioscsi14' => { bus => 3, addr => 15 },
4789         'virtioscsi15' => { bus => 3, addr => 16 },
4790         'virtioscsi16' => { bus => 3, addr => 17 },
4791         'virtioscsi17' => { bus => 3, addr => 18 },
4792         'virtioscsi18' => { bus => 3, addr => 19 },
4793         'virtioscsi19' => { bus => 3, addr => 20 },
4794         'virtioscsi20' => { bus => 3, addr => 21 },
4795         'virtioscsi21' => { bus => 3, addr => 22 },
4796         'virtioscsi22' => { bus => 3, addr => 23 },
4797         'virtioscsi23' => { bus => 3, addr => 24 },
4798         'virtioscsi24' => { bus => 3, addr => 25 },
4799         'virtioscsi25' => { bus => 3, addr => 26 },
4800         'virtioscsi26' => { bus => 3, addr => 27 },
4801         'virtioscsi27' => { bus => 3, addr => 28 },
4802         'virtioscsi28' => { bus => 3, addr => 29 },
4803         'virtioscsi29' => { bus => 3, addr => 30 },
4804         'virtioscsi30' => { bus => 3, addr => 31 },
4805
4806     };
4807
4808     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4809            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4810            my $bus = $devices->{$id}->{bus};
4811            $res = ",bus=pci.$bus,addr=$addr";
4812            $bridges->{$bus} = 1 if $bridges;
4813     }
4814     return $res;
4815
4816 }
4817
4818 sub print_pcie_addr {
4819     my ($id) = @_;
4820
4821     my $res = '';
4822     my $devices = {
4823         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4824         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4825         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4826         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4827     };
4828
4829     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4830            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4831            my $bus = $devices->{$id}->{bus};
4832            $res = ",bus=$bus,addr=$addr";
4833     }
4834     return $res;
4835
4836 }
4837
4838 # vzdump restore implementaion
4839
4840 sub tar_archive_read_firstfile {
4841     my $archive = shift;
4842
4843     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4844
4845     # try to detect archive type first
4846     my $pid = open (TMP, "tar tf '$archive'|") ||
4847         die "unable to open file '$archive'\n";
4848     my $firstfile = <TMP>;
4849     kill 15, $pid;
4850     close TMP;
4851
4852     die "ERROR: archive contaions no data\n" if !$firstfile;
4853     chomp $firstfile;
4854
4855     return $firstfile;
4856 }
4857
4858 sub tar_restore_cleanup {
4859     my ($storecfg, $statfile) = @_;
4860
4861     print STDERR "starting cleanup\n";
4862
4863     if (my $fd = IO::File->new($statfile, "r")) {
4864         while (defined(my $line = <$fd>)) {
4865             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4866                 my $volid = $2;
4867                 eval {
4868                     if ($volid =~ m|^/|) {
4869                         unlink $volid || die 'unlink failed\n';
4870                     } else {
4871                         PVE::Storage::vdisk_free($storecfg, $volid);
4872                     }
4873                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4874                 };
4875                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4876             } else {
4877                 print STDERR "unable to parse line in statfile - $line";
4878             }
4879         }
4880         $fd->close();
4881     }
4882 }
4883
4884 sub restore_archive {
4885     my ($archive, $vmid, $user, $opts) = @_;
4886
4887     my $format = $opts->{format};
4888     my $comp;
4889
4890     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4891         $format = 'tar' if !$format;
4892         $comp = 'gzip';
4893     } elsif ($archive =~ m/\.tar$/) {
4894         $format = 'tar' if !$format;
4895     } elsif ($archive =~ m/.tar.lzo$/) {
4896         $format = 'tar' if !$format;
4897         $comp = 'lzop';
4898     } elsif ($archive =~ m/\.vma$/) {
4899         $format = 'vma' if !$format;
4900     } elsif ($archive =~ m/\.vma\.gz$/) {
4901         $format = 'vma' if !$format;
4902         $comp = 'gzip';
4903     } elsif ($archive =~ m/\.vma\.lzo$/) {
4904         $format = 'vma' if !$format;
4905         $comp = 'lzop';
4906     } else {
4907         $format = 'vma' if !$format; # default
4908     }
4909
4910     # try to detect archive format
4911     if ($format eq 'tar') {
4912         return restore_tar_archive($archive, $vmid, $user, $opts);
4913     } else {
4914         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4915     }
4916 }
4917
4918 sub restore_update_config_line {
4919     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4920
4921     return if $line =~ m/^\#qmdump\#/;
4922     return if $line =~ m/^\#vzdump\#/;
4923     return if $line =~ m/^lock:/;
4924     return if $line =~ m/^unused\d+:/;
4925     return if $line =~ m/^parent:/;
4926     return if $line =~ m/^template:/; # restored VM is never a template
4927
4928     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4929         # try to convert old 1.X settings
4930         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4931         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4932             my ($model, $macaddr) = split(/\=/, $devconfig);
4933             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4934             my $net = {
4935                 model => $model,
4936                 bridge => "vmbr$ind",
4937                 macaddr => $macaddr,
4938             };
4939             my $netstr = print_net($net);
4940
4941             print $outfd "net$cookie->{netcount}: $netstr\n";
4942             $cookie->{netcount}++;
4943         }
4944     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4945         my ($id, $netstr) = ($1, $2);
4946         my $net = parse_net($netstr);
4947         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4948         $netstr = print_net($net);
4949         print $outfd "$id: $netstr\n";
4950     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4951         my $virtdev = $1;
4952         my $value = $3;
4953         if ($line =~ m/backup=no/) {
4954             print $outfd "#$line";
4955         } elsif ($virtdev && $map->{$virtdev}) {
4956             my $di = parse_drive($virtdev, $value);
4957             delete $di->{format}; # format can change on restore
4958             $di->{file} = $map->{$virtdev};
4959             $value = print_drive($vmid, $di);
4960             print $outfd "$virtdev: $value\n";
4961         } else {
4962             print $outfd $line;
4963         }
4964     } else {
4965         print $outfd $line;
4966     }
4967 }
4968
4969 sub scan_volids {
4970     my ($cfg, $vmid) = @_;
4971
4972     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4973
4974     my $volid_hash = {};
4975     foreach my $storeid (keys %$info) {
4976         foreach my $item (@{$info->{$storeid}}) {
4977             next if !($item->{volid} && $item->{size});
4978             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4979             $volid_hash->{$item->{volid}} = $item;
4980         }
4981     }
4982
4983     return $volid_hash;
4984 }
4985
4986 sub get_used_paths {
4987     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4988
4989     my $used_path = {};
4990
4991     my $scan_config = sub {
4992         my ($cref, $snapname) = @_;
4993
4994         foreach my $key (keys %$cref) {
4995             my $value = $cref->{$key};
4996             if (valid_drivename($key)) {
4997                 next if $skip_drive && $key eq $skip_drive;
4998                 my $drive = parse_drive($key, $value);
4999                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5000                 if ($drive->{file} =~ m!^/!) {
5001                     $used_path->{$drive->{file}}++; # = 1;
5002                 } else {
5003                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5004                     next if !$storeid;
5005                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5006                     next if !$scfg;
5007                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5008                     $used_path->{$path}++; # = 1;
5009                 }
5010             }
5011         }
5012     };
5013
5014     &$scan_config($conf);
5015
5016     undef $skip_drive;
5017
5018     if ($scan_snapshots) {
5019         foreach my $snapname (keys %{$conf->{snapshots}}) {
5020             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5021         }
5022     }
5023
5024     return $used_path;
5025 }
5026
5027 sub update_disksize {
5028     my ($vmid, $conf, $volid_hash) = @_;
5029
5030     my $changes;
5031
5032     my $used = {};
5033
5034     # Note: it is allowed to define multiple storages with same path (alias), so
5035     # we need to check both 'volid' and real 'path' (two different volid can point
5036     # to the same path).
5037
5038     my $usedpath = {};
5039
5040     # update size info
5041     foreach my $opt (keys %$conf) {
5042         if (valid_drivename($opt)) {
5043             my $drive = parse_drive($opt, $conf->{$opt});
5044             my $volid = $drive->{file};
5045             next if !$volid;
5046
5047             $used->{$volid} = 1;
5048             if ($volid_hash->{$volid} &&
5049                 (my $path = $volid_hash->{$volid}->{path})) {
5050                 $usedpath->{$path} = 1;
5051             }
5052
5053             next if drive_is_cdrom($drive);
5054             next if !$volid_hash->{$volid};
5055
5056             $drive->{size} = $volid_hash->{$volid}->{size};
5057             my $new = print_drive($vmid, $drive);
5058             if ($new ne $conf->{$opt}) {
5059                 $changes = 1;
5060                 $conf->{$opt} = $new;
5061             }
5062         }
5063     }
5064
5065     # remove 'unusedX' entry if volume is used
5066     foreach my $opt (keys %$conf) {
5067         next if $opt !~ m/^unused\d+$/;
5068         my $volid = $conf->{$opt};
5069         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5070         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5071             $changes = 1;
5072             delete $conf->{$opt};
5073         }
5074     }
5075
5076     foreach my $volid (sort keys %$volid_hash) {
5077         next if $volid =~ m/vm-$vmid-state-/;
5078         next if $used->{$volid};
5079         my $path = $volid_hash->{$volid}->{path};
5080         next if !$path; # just to be sure
5081         next if $usedpath->{$path};
5082         $changes = 1;
5083         add_unused_volume($conf, $volid);
5084         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5085     }
5086
5087     return $changes;
5088 }
5089
5090 sub rescan {
5091     my ($vmid, $nolock) = @_;
5092
5093     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5094
5095     my $volid_hash = scan_volids($cfg, $vmid);
5096
5097     my $updatefn =  sub {
5098         my ($vmid) = @_;
5099
5100         my $conf = load_config($vmid);
5101
5102         check_lock($conf);
5103
5104         my $vm_volids = {};
5105         foreach my $volid (keys %$volid_hash) {
5106             my $info = $volid_hash->{$volid};
5107             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5108         }
5109
5110         my $changes = update_disksize($vmid, $conf, $vm_volids);
5111
5112         update_config_nolock($vmid, $conf, 1) if $changes;
5113     };
5114
5115     if (defined($vmid)) {
5116         if ($nolock) {
5117             &$updatefn($vmid);
5118         } else {
5119             lock_config($vmid, $updatefn, $vmid);
5120         }
5121     } else {
5122         my $vmlist = config_list();
5123         foreach my $vmid (keys %$vmlist) {
5124             if ($nolock) {
5125                 &$updatefn($vmid);
5126             } else {
5127                 lock_config($vmid, $updatefn, $vmid);
5128             }
5129         }
5130     }
5131 }
5132
5133 sub restore_vma_archive {
5134     my ($archive, $vmid, $user, $opts, $comp) = @_;
5135
5136     my $input = $archive eq '-' ? "<&STDIN" : undef;
5137     my $readfrom = $archive;
5138
5139     my $uncomp = '';
5140     if ($comp) {
5141         $readfrom = '-';
5142         my $qarchive = PVE::Tools::shellquote($archive);
5143         if ($comp eq 'gzip') {
5144             $uncomp = "zcat $qarchive|";
5145         } elsif ($comp eq 'lzop') {
5146             $uncomp = "lzop -d -c $qarchive|";
5147         } else {
5148             die "unknown compression method '$comp'\n";
5149         }
5150
5151     }
5152
5153     my $tmpdir = "/var/tmp/vzdumptmp$$";
5154     rmtree $tmpdir;
5155
5156     # disable interrupts (always do cleanups)
5157     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5158         warn "got interrupt - ignored\n";
5159     };
5160
5161     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5162     POSIX::mkfifo($mapfifo, 0600);
5163     my $fifofh;
5164
5165     my $openfifo = sub {
5166         open($fifofh, '>', $mapfifo) || die $!;
5167     };
5168
5169     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5170
5171     my $oldtimeout;
5172     my $timeout = 5;
5173
5174     my $devinfo = {};
5175
5176     my $rpcenv = PVE::RPCEnvironment::get();
5177
5178     my $conffile = config_file($vmid);
5179     my $tmpfn = "$conffile.$$.tmp";
5180
5181     # Note: $oldconf is undef if VM does not exists
5182     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5183
5184     my $print_devmap = sub {
5185         my $virtdev_hash = {};
5186
5187         my $cfgfn = "$tmpdir/qemu-server.conf";
5188
5189         # we can read the config - that is already extracted
5190         my $fh = IO::File->new($cfgfn, "r") ||
5191             "unable to read qemu-server.conf - $!\n";
5192
5193         while (defined(my $line = <$fh>)) {
5194             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5195                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5196                 die "archive does not contain data for drive '$virtdev'\n"
5197                     if !$devinfo->{$devname};
5198                 if (defined($opts->{storage})) {
5199                     $storeid = $opts->{storage} || 'local';
5200                 } elsif (!$storeid) {
5201                     $storeid = 'local';
5202                 }
5203                 $format = 'raw' if !$format;
5204                 $devinfo->{$devname}->{devname} = $devname;
5205                 $devinfo->{$devname}->{virtdev} = $virtdev;
5206                 $devinfo->{$devname}->{format} = $format;
5207                 $devinfo->{$devname}->{storeid} = $storeid;
5208
5209                 # check permission on storage
5210                 my $pool = $opts->{pool}; # todo: do we need that?
5211                 if ($user ne 'root@pam') {
5212                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5213                 }
5214
5215                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5216             }
5217         }
5218
5219         foreach my $devname (keys %$devinfo) {
5220             die "found no device mapping information for device '$devname'\n"
5221                 if !$devinfo->{$devname}->{virtdev};
5222         }
5223
5224         my $cfg = cfs_read_file('storage.cfg');
5225
5226         # create empty/temp config
5227         if ($oldconf) {
5228             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5229             foreach_drive($oldconf, sub {
5230                 my ($ds, $drive) = @_;
5231
5232                 return if drive_is_cdrom($drive);
5233
5234                 my $volid = $drive->{file};
5235
5236                 return if !$volid || $volid =~ m|^/|;
5237
5238                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5239                 return if !$path || !$owner || ($owner != $vmid);
5240
5241                 # Note: only delete disk we want to restore
5242                 # other volumes will become unused
5243                 if ($virtdev_hash->{$ds}) {
5244                     PVE::Storage::vdisk_free($cfg, $volid);
5245                 }
5246             });
5247         }
5248
5249         my $map = {};
5250         foreach my $virtdev (sort keys %$virtdev_hash) {
5251             my $d = $virtdev_hash->{$virtdev};
5252             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5253             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5254
5255             # test if requested format is supported
5256             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5257             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5258             $d->{format} = $defFormat if !$supported;
5259
5260             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5261                                                   $d->{format}, undef, $alloc_size);
5262             print STDERR "new volume ID is '$volid'\n";
5263             $d->{volid} = $volid;
5264             my $path = PVE::Storage::path($cfg, $volid);
5265
5266             my $write_zeros = 1;
5267             # fixme: what other storages types initialize volumes with zero?
5268             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5269                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5270                 $write_zeros = 0;
5271             }
5272
5273             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5274
5275             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5276             $map->{$virtdev} = $volid;
5277         }
5278
5279         $fh->seek(0, 0) || die "seek failed - $!\n";
5280
5281         my $outfd = new IO::File ($tmpfn, "w") ||
5282             die "unable to write config for VM $vmid\n";
5283
5284         my $cookie = { netcount => 0 };
5285         while (defined(my $line = <$fh>)) {
5286             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5287         }
5288
5289         $fh->close();
5290         $outfd->close();
5291     };
5292
5293     eval {
5294         # enable interrupts
5295         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5296             die "interrupted by signal\n";
5297         };
5298         local $SIG{ALRM} = sub { die "got timeout\n"; };
5299
5300         $oldtimeout = alarm($timeout);
5301
5302         my $parser = sub {
5303             my $line = shift;
5304
5305             print "$line\n";
5306
5307             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5308                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5309                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5310             } elsif ($line =~ m/^CTIME: /) {
5311                 # we correctly received the vma config, so we can disable
5312                 # the timeout now for disk allocation (set to 10 minutes, so
5313                 # that we always timeout if something goes wrong)
5314                 alarm(600);
5315                 &$print_devmap();
5316                 print $fifofh "done\n";
5317                 my $tmp = $oldtimeout || 0;
5318                 $oldtimeout = undef;
5319                 alarm($tmp);
5320                 close($fifofh);
5321             }
5322         };
5323
5324         print "restore vma archive: $cmd\n";
5325         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5326     };
5327     my $err = $@;
5328
5329     alarm($oldtimeout) if $oldtimeout;
5330
5331     unlink $mapfifo;
5332
5333     if ($err) {
5334         rmtree $tmpdir;
5335         unlink $tmpfn;
5336
5337         my $cfg = cfs_read_file('storage.cfg');
5338         foreach my $devname (keys %$devinfo) {
5339             my $volid = $devinfo->{$devname}->{volid};
5340             next if !$volid;
5341             eval {
5342                 if ($volid =~ m|^/|) {
5343                     unlink $volid || die 'unlink failed\n';
5344                 } else {
5345                     PVE::Storage::vdisk_free($cfg, $volid);
5346                 }
5347                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5348             };
5349             print STDERR "unable to cleanup '$volid' - $@" if $@;
5350         }
5351         die $err;
5352     }
5353
5354     rmtree $tmpdir;
5355
5356     rename($tmpfn, $conffile) ||
5357         die "unable to commit configuration file '$conffile'\n";
5358
5359     PVE::Cluster::cfs_update(); # make sure we read new file
5360
5361     eval { rescan($vmid, 1); };
5362     warn $@ if $@;
5363 }
5364
5365 sub restore_tar_archive {
5366     my ($archive, $vmid, $user, $opts) = @_;
5367
5368     if ($archive ne '-') {
5369         my $firstfile = tar_archive_read_firstfile($archive);
5370         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5371             if $firstfile ne 'qemu-server.conf';
5372     }
5373
5374     my $storecfg = cfs_read_file('storage.cfg');
5375
5376     # destroy existing data - keep empty config
5377     my $vmcfgfn = config_file($vmid);
5378     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5379
5380     my $tocmd = "/usr/lib/qemu-server/qmextract";
5381
5382     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5383     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5384     $tocmd .= ' --prealloc' if $opts->{prealloc};
5385     $tocmd .= ' --info' if $opts->{info};
5386
5387     # tar option "xf" does not autodetect compression when read from STDIN,
5388     # so we pipe to zcat
5389     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5390         PVE::Tools::shellquote("--to-command=$tocmd");
5391
5392     my $tmpdir = "/var/tmp/vzdumptmp$$";
5393     mkpath $tmpdir;
5394
5395     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5396     local $ENV{VZDUMP_VMID} = $vmid;
5397     local $ENV{VZDUMP_USER} = $user;
5398
5399     my $conffile = config_file($vmid);
5400     my $tmpfn = "$conffile.$$.tmp";
5401
5402     # disable interrupts (always do cleanups)
5403     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5404         print STDERR "got interrupt - ignored\n";
5405     };
5406
5407     eval {
5408         # enable interrupts
5409         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5410             die "interrupted by signal\n";
5411         };
5412
5413         if ($archive eq '-') {
5414             print "extracting archive from STDIN\n";
5415             run_command($cmd, input => "<&STDIN");
5416         } else {
5417             print "extracting archive '$archive'\n";
5418             run_command($cmd);
5419         }
5420
5421         return if $opts->{info};
5422
5423         # read new mapping
5424         my $map = {};
5425         my $statfile = "$tmpdir/qmrestore.stat";
5426         if (my $fd = IO::File->new($statfile, "r")) {
5427             while (defined (my $line = <$fd>)) {
5428                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5429                     $map->{$1} = $2 if $1;
5430                 } else {
5431                     print STDERR "unable to parse line in statfile - $line\n";
5432                 }
5433             }
5434             $fd->close();
5435         }
5436
5437         my $confsrc = "$tmpdir/qemu-server.conf";
5438
5439         my $srcfd = new IO::File($confsrc, "r") ||
5440             die "unable to open file '$confsrc'\n";
5441
5442         my $outfd = new IO::File ($tmpfn, "w") ||
5443             die "unable to write config for VM $vmid\n";
5444
5445         my $cookie = { netcount => 0 };
5446         while (defined (my $line = <$srcfd>)) {
5447             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5448         }
5449
5450         $srcfd->close();
5451         $outfd->close();
5452     };
5453     my $err = $@;
5454
5455     if ($err) {
5456
5457         unlink $tmpfn;
5458
5459         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5460
5461         die $err;
5462     }
5463
5464     rmtree $tmpdir;
5465
5466     rename $tmpfn, $conffile ||
5467         die "unable to commit configuration file '$conffile'\n";
5468
5469     PVE::Cluster::cfs_update(); # make sure we read new file
5470
5471     eval { rescan($vmid, 1); };
5472     warn $@ if $@;
5473 };
5474
5475
5476 # Internal snapshots
5477
5478 # NOTE: Snapshot create/delete involves several non-atomic
5479 # action, and can take a long time.
5480 # So we try to avoid locking the file and use 'lock' variable
5481 # inside the config file instead.
5482
5483 my $snapshot_copy_config = sub {
5484     my ($source, $dest) = @_;
5485
5486     foreach my $k (keys %$source) {
5487         next if $k eq 'snapshots';
5488         next if $k eq 'snapstate';
5489         next if $k eq 'snaptime';
5490         next if $k eq 'vmstate';
5491         next if $k eq 'lock';
5492         next if $k eq 'digest';
5493         next if $k eq 'description';
5494         next if $k =~ m/^unused\d+$/;
5495
5496         $dest->{$k} = $source->{$k};
5497     }
5498 };
5499
5500 my $snapshot_apply_config = sub {
5501     my ($conf, $snap) = @_;
5502
5503     # copy snapshot list
5504     my $newconf = {
5505         snapshots => $conf->{snapshots},
5506     };
5507
5508     # keep description and list of unused disks
5509     foreach my $k (keys %$conf) {
5510         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5511         $newconf->{$k} = $conf->{$k};
5512     }
5513
5514     &$snapshot_copy_config($snap, $newconf);
5515
5516     return $newconf;
5517 };
5518
5519 sub foreach_writable_storage {
5520     my ($conf, $func) = @_;
5521
5522     my $sidhash = {};
5523
5524     foreach my $ds (keys %$conf) {
5525         next if !valid_drivename($ds);
5526
5527         my $drive = parse_drive($ds, $conf->{$ds});
5528         next if !$drive;
5529         next if drive_is_cdrom($drive);
5530
5531         my $volid = $drive->{file};
5532
5533         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5534         $sidhash->{$sid} = $sid if $sid;
5535     }
5536
5537     foreach my $sid (sort keys %$sidhash) {
5538         &$func($sid);
5539     }
5540 }
5541
5542 my $alloc_vmstate_volid = sub {
5543     my ($storecfg, $vmid, $conf, $snapname) = @_;
5544
5545     # Note: we try to be smart when selecting a $target storage
5546
5547     my $target;
5548
5549     # search shared storage first
5550     foreach_writable_storage($conf, sub {
5551         my ($sid) = @_;
5552         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5553         return if !$scfg->{shared};
5554
5555         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5556     });
5557
5558     if (!$target) {
5559         # now search local storage
5560         foreach_writable_storage($conf, sub {
5561             my ($sid) = @_;
5562             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5563             return if $scfg->{shared};
5564
5565             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5566         });
5567     }
5568
5569     $target = 'local' if !$target;
5570
5571     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5572     # we abort live save after $conf->{memory}, so we need at max twice that space
5573     my $size = $conf->{memory}*2 + $driver_state_size;
5574
5575     my $name = "vm-$vmid-state-$snapname";
5576     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5577     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5578     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5579
5580     return $volid;
5581 };
5582
5583 my $snapshot_prepare = sub {
5584     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5585
5586     my $snap;
5587
5588     my $updatefn =  sub {
5589
5590         my $conf = load_config($vmid);
5591
5592         die "you can't take a snapshot if it's a template\n"
5593             if is_template($conf);
5594
5595         check_lock($conf);
5596
5597         $conf->{lock} = 'snapshot';
5598
5599         die "snapshot name '$snapname' already used\n"
5600             if defined($conf->{snapshots}->{$snapname});
5601
5602         my $storecfg = PVE::Storage::config();
5603         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5604
5605         $snap = $conf->{snapshots}->{$snapname} = {};
5606
5607         if ($save_vmstate && check_running($vmid)) {
5608             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5609         }
5610
5611         &$snapshot_copy_config($conf, $snap);
5612
5613         $snap->{snapstate} = "prepare";
5614         $snap->{snaptime} = time();
5615         $snap->{description} = $comment if $comment;
5616
5617         # always overwrite machine if we save vmstate. This makes sure we
5618         # can restore it later using correct machine type
5619         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5620
5621         update_config_nolock($vmid, $conf, 1);
5622     };
5623
5624     lock_config($vmid, $updatefn);
5625
5626     return $snap;
5627 };
5628
5629 my $snapshot_commit = sub {
5630     my ($vmid, $snapname) = @_;
5631
5632     my $updatefn = sub {
5633
5634         my $conf = load_config($vmid);
5635
5636         die "missing snapshot lock\n"
5637             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5638
5639         my $has_machine_config = defined($conf->{machine});
5640
5641         my $snap = $conf->{snapshots}->{$snapname};
5642
5643         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5644
5645         die "wrong snapshot state\n"
5646             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5647
5648         delete $snap->{snapstate};
5649         delete $conf->{lock};
5650
5651         my $newconf = &$snapshot_apply_config($conf, $snap);
5652
5653         delete $newconf->{machine} if !$has_machine_config;
5654
5655         $newconf->{parent} = $snapname;
5656
5657         update_config_nolock($vmid, $newconf, 1);
5658     };
5659
5660     lock_config($vmid, $updatefn);
5661 };
5662
5663 sub snapshot_rollback {
5664     my ($vmid, $snapname) = @_;
5665
5666     my $prepare = 1;
5667
5668     my $storecfg = PVE::Storage::config();
5669
5670     my $conf = load_config($vmid);
5671
5672     my $get_snapshot_config = sub {
5673
5674         die "you can't rollback if vm is a template\n" if is_template($conf);
5675
5676         my $res = $conf->{snapshots}->{$snapname};
5677
5678         die "snapshot '$snapname' does not exist\n" if !defined($res);
5679
5680         return $res;
5681     };
5682
5683     my $snap = &$get_snapshot_config();
5684
5685     foreach_drive($snap, sub {
5686         my ($ds, $drive) = @_;
5687
5688         return if drive_is_cdrom($drive);
5689
5690         my $volid = $drive->{file};
5691
5692         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5693     });
5694
5695     my $updatefn = sub {
5696
5697         $conf = load_config($vmid);
5698
5699         $snap = &$get_snapshot_config();
5700
5701         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5702             if $snap->{snapstate};
5703
5704         if ($prepare) {
5705             check_lock($conf);
5706             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5707         }
5708
5709         die "unable to rollback vm $vmid: vm is running\n"
5710             if check_running($vmid);
5711
5712         if ($prepare) {
5713             $conf->{lock} = 'rollback';
5714         } else {
5715             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5716             delete $conf->{lock};
5717         }
5718
5719         my $forcemachine;
5720
5721         if (!$prepare) {
5722             my $has_machine_config = defined($conf->{machine});
5723
5724             # copy snapshot config to current config
5725             $conf = &$snapshot_apply_config($conf, $snap);
5726             $conf->{parent} = $snapname;
5727
5728             # Note: old code did not store 'machine', so we try to be smart
5729             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5730             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5731             # we remove the 'machine' configuration if not explicitly specified
5732             # in the original config.
5733             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5734         }
5735
5736         update_config_nolock($vmid, $conf, 1);
5737
5738         if (!$prepare && $snap->{vmstate}) {
5739             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5740             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5741         }
5742     };
5743
5744     lock_config($vmid, $updatefn);
5745
5746     foreach_drive($snap, sub {
5747         my ($ds, $drive) = @_;
5748
5749         return if drive_is_cdrom($drive);
5750
5751         my $volid = $drive->{file};
5752         my $device = "drive-$ds";
5753
5754         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5755     });
5756
5757     $prepare = 0;
5758     lock_config($vmid, $updatefn);
5759 }
5760
5761 my $savevm_wait = sub {
5762     my ($vmid) = @_;
5763
5764     for(;;) {
5765         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5766         if (!$stat->{status}) {
5767             die "savevm not active\n";
5768         } elsif ($stat->{status} eq 'active') {
5769             sleep(1);
5770             next;
5771         } elsif ($stat->{status} eq 'completed') {
5772             last;
5773         } else {
5774             die "query-savevm returned status '$stat->{status}'\n";
5775         }
5776     }
5777 };
5778
5779 sub snapshot_create {
5780     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5781
5782     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5783
5784     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5785
5786     my $config = load_config($vmid);
5787
5788     my $running = check_running($vmid);
5789
5790     my $freezefs = $running && $config->{agent};
5791     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5792
5793     my $drivehash = {};
5794
5795     if ($freezefs) {
5796         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5797         warn "guest-fsfreeze-freeze problems - $@" if $@;
5798     }
5799
5800     eval {
5801         # create internal snapshots of all drives
5802
5803         my $storecfg = PVE::Storage::config();
5804
5805         if ($running) {
5806             if ($snap->{vmstate}) {
5807                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5808                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5809                 &$savevm_wait($vmid);
5810             } else {
5811                 vm_mon_cmd($vmid, "savevm-start");
5812             }
5813         };
5814
5815         foreach_drive($snap, sub {
5816             my ($ds, $drive) = @_;
5817
5818             return if drive_is_cdrom($drive);
5819
5820             my $volid = $drive->{file};
5821             my $device = "drive-$ds";
5822
5823             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5824             $drivehash->{$ds} = 1;
5825        });
5826     };
5827     my $err = $@;
5828
5829     if ($running) {
5830         eval { vm_mon_cmd($vmid, "savevm-end")  };
5831         warn $@ if $@;
5832
5833         if ($freezefs) {
5834             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5835             warn "guest-fsfreeze-thaw problems - $@" if $@;
5836         }
5837
5838         # savevm-end is async, we need to wait
5839         for (;;) {
5840             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5841             if (!$stat->{bytes}) {
5842                 last;
5843             } else {
5844                 print "savevm not yet finished\n";
5845                 sleep(1);
5846                 next;
5847             }
5848         }
5849     }
5850
5851     if ($err) {
5852         warn "snapshot create failed: starting cleanup\n";
5853         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5854         warn $@ if $@;
5855         die $err;
5856     }
5857
5858     &$snapshot_commit($vmid, $snapname);
5859 }
5860
5861 # Note: $drivehash is only set when called from snapshot_create.
5862 sub snapshot_delete {
5863     my ($vmid, $snapname, $force, $drivehash) = @_;
5864
5865     my $prepare = 1;
5866
5867     my $snap;
5868     my $unused = [];
5869
5870     my $unlink_parent = sub {
5871         my ($confref, $new_parent) = @_;
5872
5873         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5874             if ($new_parent) {
5875                 $confref->{parent} = $new_parent;
5876             } else {
5877                 delete $confref->{parent};
5878             }
5879         }
5880     };
5881
5882     my $updatefn =  sub {
5883         my ($remove_drive) = @_;
5884
5885         my $conf = load_config($vmid);
5886
5887         if (!$drivehash) {
5888             check_lock($conf);
5889             die "you can't delete a snapshot if vm is a template\n"
5890                 if is_template($conf);
5891         }
5892
5893         $snap = $conf->{snapshots}->{$snapname};
5894
5895         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5896
5897         # remove parent refs
5898         if (!$prepare) {
5899             &$unlink_parent($conf, $snap->{parent});
5900             foreach my $sn (keys %{$conf->{snapshots}}) {
5901                 next if $sn eq $snapname;
5902                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5903             }
5904         }
5905
5906         if ($remove_drive) {
5907             if ($remove_drive eq 'vmstate') {
5908                 delete $snap->{$remove_drive};
5909             } else {
5910                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5911                 my $volid = $drive->{file};
5912                 delete $snap->{$remove_drive};
5913                 add_unused_volume($conf, $volid);
5914             }
5915         }
5916
5917         if ($prepare) {
5918             $snap->{snapstate} = 'delete';
5919         } else {
5920             delete $conf->{snapshots}->{$snapname};
5921             delete $conf->{lock} if $drivehash;
5922             foreach my $volid (@$unused) {
5923                 add_unused_volume($conf, $volid);
5924             }
5925         }
5926
5927         update_config_nolock($vmid, $conf, 1);
5928     };
5929
5930     lock_config($vmid, $updatefn);
5931
5932     # now remove vmstate file
5933
5934     my $storecfg = PVE::Storage::config();
5935
5936     if ($snap->{vmstate}) {
5937         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5938         if (my $err = $@) {
5939             die $err if !$force;
5940             warn $err;
5941         }
5942         # save changes (remove vmstate from snapshot)
5943         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5944     };
5945
5946     # now remove all internal snapshots
5947     foreach_drive($snap, sub {
5948         my ($ds, $drive) = @_;
5949
5950         return if drive_is_cdrom($drive);
5951
5952         my $volid = $drive->{file};
5953         my $device = "drive-$ds";
5954
5955         if (!$drivehash || $drivehash->{$ds}) {
5956             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5957             if (my $err = $@) {
5958                 die $err if !$force;
5959                 warn $err;
5960             }
5961         }
5962
5963         # save changes (remove drive fron snapshot)
5964         lock_config($vmid, $updatefn, $ds) if !$force;
5965         push @$unused, $volid;
5966     });
5967
5968     # now cleanup config
5969     $prepare = 0;
5970     lock_config($vmid, $updatefn);
5971 }
5972
5973 sub has_feature {
5974     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5975
5976     my $err;
5977     foreach_drive($conf, sub {
5978         my ($ds, $drive) = @_;
5979
5980         return if drive_is_cdrom($drive);
5981         my $volid = $drive->{file};
5982         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5983     });
5984
5985     return $err ? 0 : 1;
5986 }
5987
5988 sub template_create {
5989     my ($vmid, $conf, $disk) = @_;
5990
5991     my $storecfg = PVE::Storage::config();
5992
5993     foreach_drive($conf, sub {
5994         my ($ds, $drive) = @_;
5995
5996         return if drive_is_cdrom($drive);
5997         return if $disk && $ds ne $disk;
5998
5999         my $volid = $drive->{file};
6000         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6001
6002         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6003         $drive->{file} = $voliddst;
6004         $conf->{$ds} = print_drive($vmid, $drive);
6005         update_config_nolock($vmid, $conf, 1);
6006     });
6007 }
6008
6009 sub is_template {
6010     my ($conf) = @_;
6011
6012     return 1 if defined $conf->{template} && $conf->{template} == 1;
6013 }
6014
6015 sub qemu_img_convert {
6016     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6017
6018     my $storecfg = PVE::Storage::config();
6019     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6020     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6021
6022     if ($src_storeid && $dst_storeid) {
6023         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6024         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6025
6026         my $src_format = qemu_img_format($src_scfg, $src_volname);
6027         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6028
6029         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6030         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6031
6032         my $cmd = [];
6033         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6034         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6035         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6036
6037         my $parser = sub {
6038             my $line = shift;
6039             if($line =~ m/\((\S+)\/100\%\)/){
6040                 my $percent = $1;
6041                 my $transferred = int($size * $percent / 100);
6042                 my $remaining = $size - $transferred;
6043
6044                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6045             }
6046
6047         };
6048
6049         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6050         my $err = $@;
6051         die "copy failed: $err" if $err;
6052     }
6053 }
6054
6055 sub qemu_img_format {
6056     my ($scfg, $volname) = @_;
6057
6058     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6059         return $1;
6060     } elsif ($scfg->{type} eq 'iscsi') {
6061         return "host_device";
6062     } else {
6063         return "raw";
6064     }
6065 }
6066
6067 sub qemu_drive_mirror {
6068     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6069
6070     my $count = 0;
6071     my $old_len = 0;
6072     my $frozen = undef;
6073     my $maxwait = 120;
6074
6075     my $storecfg = PVE::Storage::config();
6076     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6077
6078     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6079
6080     my $format;
6081     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6082         $format = $1;
6083     }
6084
6085     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6086
6087     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6088     $opts->{format} = $format if $format;
6089
6090     #fixme : sometime drive-mirror timeout, but works fine after.
6091     # (I have see the problem with big volume > 200GB), so we need to eval
6092     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6093     # ignore errors here
6094
6095     eval {
6096         while (1) {
6097             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6098             my $stat = @$stats[0];
6099             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6100             die "error job is not mirroring" if $stat->{type} ne "mirror";
6101
6102             my $busy = $stat->{busy};
6103
6104             if (my $total = $stat->{len}) {
6105                 my $transferred = $stat->{offset} || 0;
6106                 my $remaining = $total - $transferred;
6107                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6108
6109                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy\n";
6110             }
6111
6112             if ($stat->{len} == $stat->{offset}) {
6113                 if ($busy eq 'false') {
6114
6115                     last if $vmiddst != $vmid;
6116
6117                     # try to switch the disk if source and destination are on the same guest
6118                     eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6119                     last if !$@;
6120                     die $@ if $@ !~ m/cannot be completed/;
6121                 }
6122
6123                 if ($count > $maxwait) {
6124                     # if too much writes to disk occurs at the end of migration
6125                     #the disk needs to be freezed to be able to complete the migration
6126                     vm_suspend($vmid,1);
6127                     $frozen = 1;
6128                 }
6129                 $count ++
6130             }
6131             $old_len = $stat->{offset};
6132             sleep 1;
6133         }
6134
6135         vm_resume($vmid, 1) if $frozen;
6136
6137     };
6138     my $err = $@;
6139
6140     my $cancel_job = sub {
6141         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6142         while (1) {
6143             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6144             my $stat = @$stats[0];
6145             last if !$stat;
6146             sleep 1;
6147         }
6148     };
6149
6150     if ($err) {
6151         eval { &$cancel_job(); };
6152         die "mirroring error: $err";
6153     }
6154
6155     if ($vmiddst != $vmid) {
6156         # if we clone a disk for a new target vm, we don't switch the disk
6157         &$cancel_job(); # so we call block-job-cancel
6158     }
6159 }
6160
6161 sub clone_disk {
6162     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6163         $newvmid, $storage, $format, $full, $newvollist) = @_;
6164
6165     my $newvolid;
6166
6167     if (!$full) {
6168         print "create linked clone of drive $drivename ($drive->{file})\n";
6169         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6170         push @$newvollist, $newvolid;
6171     } else {
6172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6173         $storeid = $storage if $storage;
6174
6175         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6176         if (!$format) {
6177             $format = $drive->{format} || $defFormat;
6178         }
6179
6180         # test if requested format is supported - else use default
6181         my $supported = grep { $_ eq $format } @$validFormats;
6182         $format = $defFormat if !$supported;
6183
6184         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6185
6186         print "create full clone of drive $drivename ($drive->{file})\n";
6187         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6188         push @$newvollist, $newvolid;
6189
6190         if (!$running || $snapname) {
6191             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6192         } else {
6193             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6194         }
6195     }
6196
6197     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6198
6199     my $disk = $drive;
6200     $disk->{format} = undef;
6201     $disk->{file} = $newvolid;
6202     $disk->{size} = $size;
6203
6204     return $disk;
6205 }
6206
6207 # this only works if VM is running
6208 sub get_current_qemu_machine {
6209     my ($vmid) = @_;
6210
6211     my $cmd = { execute => 'query-machines', arguments => {} };
6212     my $res = vm_qmp_command($vmid, $cmd);
6213
6214     my ($current, $default);
6215     foreach my $e (@$res) {
6216         $default = $e->{name} if $e->{'is-default'};
6217         $current = $e->{name} if $e->{'is-current'};
6218     }
6219
6220     # fallback to the default machine if current is not supported by qemu
6221     return $current || $default || 'pc';
6222 }
6223
6224 sub qemu_machine_feature_enabled {
6225     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6226
6227     my $current_major;
6228     my $current_minor;
6229
6230     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6231
6232         $current_major = $3;
6233         $current_minor = $4;
6234
6235     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6236
6237         $current_major = $1;
6238         $current_minor = $2;
6239     }
6240
6241     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6242
6243
6244 }
6245
6246 sub lspci {
6247
6248     my $devices = {};
6249
6250     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6251             my (undef, $id, $function) = @_;
6252             my $res = { id => $id, function => $function};
6253             push @{$devices->{$id}}, $res;
6254     });
6255
6256     return $devices;
6257 }
6258
6259 sub vm_iothreads_list {
6260     my ($vmid) = @_;
6261
6262     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6263
6264     my $iothreads = {};
6265     foreach my $iothread (@$res) {
6266         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6267     }
6268
6269     return $iothreads;
6270 }
6271
6272 sub scsihw_infos {
6273     my ($conf, $drive) = @_;
6274
6275     my $maxdev = 0;
6276
6277     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6278         $maxdev = 7;
6279     } elsif ($conf->{scsihw} && ($conf->{scsihw} =~ m/^virtio-scsi-single/)) {
6280         $maxdev = 1;
6281     } else {
6282         $maxdev = 256;
6283     }
6284
6285     my $controller = int($drive->{index} / $maxdev);
6286     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} =~ m/^virtio-scsi-single/) ? "virtioscsi" : "scsihw";
6287
6288     return ($maxdev, $controller, $controller_prefix);
6289 }
6290 1;