PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     onboot => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Specifies whether a VM will be started during system bootup.",
 159         default => 0,
 160     },
 161     autostart => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Automatic restart after crash (currently ignored).",
 165         default => 0,
 166     },
 167     hotplug => {
 168         optional => 1,
 169         type => 'string', format => 'pve-hotplug-features',
 170         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 171         default => 'network,disk,usb',
 172     },
 173     reboot => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 177         default => 1,
 178     },
 179     lock => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Lock/unlock the VM.",
 183         enum => [qw(migrate backup snapshot rollback)],
 184     },
 185     cpulimit => {
 186         optional => 1,
 187         type => 'integer',
 188         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 189         minimum => 0,
 190         default => 0,
 191     },
 192     cpuunits => {
 193         optional => 1,
 194         type => 'integer',
 195         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 196         minimum => 0,
 197         maximum => 500000,
 198         default => 1000,
 199     },
 200     memory => {
 201         optional => 1,
 202         type => 'integer',
 203         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 204         minimum => 16,
 205         default => 512,
 206     },
 207     balloon => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 211         minimum => 0,
 212     },
 213     shares => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 217         minimum => 0,
 218         maximum => 50000,
 219         default => 1000,
 220     },
 221     keyboard => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 225         enum => PVE::Tools::kvmkeymaplist(),
 226         default => 'en-us',
 227     },
 228     name => {
 229         optional => 1,
 230         type => 'string', format => 'dns-name',
 231         description => "Set a name for the VM. Only used on the configuration web interface.",
 232     },
 233     scsihw => {
 234         optional => 1,
 235         type => 'string',
 236         description => "scsi controller model",
 237         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 238         default => 'lsi',
 239     },
 240     description => {
 241         optional => 1,
 242         type => 'string',
 243         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 244     },
 245     ostype => {
 246         optional => 1,
 247         type => 'string',
 248         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 249         description => <<EODESC,
 250 Used to enable special optimization/features for specific
 251 operating systems:
 252
 253 other  => unspecified OS
 254 wxp    => Microsoft Windows XP
 255 w2k    => Microsoft Windows 2000
 256 w2k3   => Microsoft Windows 2003
 257 w2k8   => Microsoft Windows 2008
 258 wvista => Microsoft Windows Vista
 259 win7   => Microsoft Windows 7
 260 win8   => Microsoft Windows 8/2012
 261 l24    => Linux 2.4 Kernel
 262 l26    => Linux 2.6/3.X Kernel
 263 solaris => solaris/opensolaris/openindiania kernel
 264
 265 other|l24|l26|solaris                       ... no special behaviour
 266 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 267 EODESC
 268     },
 269     boot => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 273         pattern => '[acdn]{1,4}',
 274         default => 'cdn',
 275     },
 276     bootdisk => {
 277         optional => 1,
 278         type => 'string', format => 'pve-qm-bootdisk',
 279         description => "Enable booting from specified disk.",
 280         pattern => '(ide|sata|scsi|virtio)\d+',
 281     },
 282     smp => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "The number of CPUs. Please use option -sockets instead.",
 286         minimum => 1,
 287         default => 1,
 288     },
 289     sockets => {
 290         optional => 1,
 291         type => 'integer',
 292         description => "The number of CPU sockets.",
 293         minimum => 1,
 294         default => 1,
 295     },
 296     cores => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of cores per socket.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     numa => {
 304         optional => 1,
 305         type => 'boolean',
 306         description => "Enable/disable Numa.",
 307         default => 0,
 308     },
 309     vcpus => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "Number of hotplugged vcpus.",
 313         minimum => 1,
 314         default => 0,
 315     },
 316     acpi => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable ACPI.",
 320         default => 1,
 321     },
 322     agent => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable Qemu GuestAgent.",
 326         default => 0,
 327     },
 328     kvm => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable KVM hardware virtualization.",
 332         default => 1,
 333     },
 334     tdf => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Enable/disable time drift fix.",
 338         default => 0,
 339     },
 340     localtime => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 344     },
 345     freeze => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 349     },
 350     vga => {
 351         optional => 1,
 352         type => 'string',
 353         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 354         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 355     },
 356     watchdog => {
 357         optional => 1,
 358         type => 'string', format => 'pve-qm-watchdog',
 359         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 360         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 361     },
 362     startdate => {
 363         optional => 1,
 364         type => 'string',
 365         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 366         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 367         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 368         default => 'now',
 369     },
 370     startup => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-startup',
 373         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 374         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 375     },
 376     template => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable Template.",
 380         default => 0,
 381     },
 382     args => {
 383         optional => 1,
 384         type => 'string',
 385         description => <<EODESCR,
 386 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 387
 388 args: -no-reboot -no-hpet
 389 EODESCR
 390     },
 391     tablet => {
 392         optional => 1,
 393         type => 'boolean',
 394         default => 1,
 395         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 396     },
 397     migrate_speed => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 401         minimum => 0,
 402         default => 0,
 403     },
 404     migrate_downtime => {
 405         optional => 1,
 406         type => 'number',
 407         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 408         minimum => 0,
 409         default => 0.1,
 410     },
 411     cdrom => {
 412         optional => 1,
 413         type => 'string', format => 'pve-qm-drive',
 414         typetext => 'volume',
 415         description => "This is an alias for option -ide2",
 416     },
 417     cpu => {
 418         optional => 1,
 419         description => "Emulated CPU type.",
 420         type => 'string',
 421         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 422         default => 'kvm64',
 423     },
 424     parent => get_standard_option('pve-snapshot-name', {
 425         optional => 1,
 426         description => "Parent snapshot name. This is used internally, and should not be modified.",
 427     }),
 428     snaptime => {
 429         optional => 1,
 430         description => "Timestamp for snapshots.",
 431         type => 'integer',
 432         minimum => 0,
 433     },
 434     vmstate => {
 435         optional => 1,
 436         type => 'string', format => 'pve-volume-id',
 437         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 438     },
 439     machine => {
 440         description => "Specific the Qemu machine type.",
 441         type => 'string',
 442         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 443         maxLength => 40,
 444         optional => 1,
 445     },
 446     smbios1 => {
 447         description => "Specify SMBIOS type 1 fields.",
 448         type => 'string', format => 'pve-qm-smbios1',
 449         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 450         maxLength => 256,
 451         optional => 1,
 452     },
 453 };
 454
 455 # what about other qemu settings ?
 456 #cpu => 'string',
 457 #machine => 'string',
 458 #fda => 'file',
 459 #fdb => 'file',
 460 #mtdblock => 'file',
 461 #sd => 'file',
 462 #pflash => 'file',
 463 #snapshot => 'bool',
 464 #bootp => 'file',
 465 ##tftp => 'dir',
 466 ##smb => 'dir',
 467 #kernel => 'file',
 468 #append => 'string',
 469 #initrd => 'file',
 470 ##soundhw => 'string',
 471
 472 while (my ($k, $v) = each %$confdesc) {
 473     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 474 }
 475
 476 my $MAX_IDE_DISKS = 4;
 477 my $MAX_SCSI_DISKS = 14;
 478 my $MAX_VIRTIO_DISKS = 16;
 479 my $MAX_SATA_DISKS = 6;
 480 my $MAX_USB_DEVICES = 5;
 481 my $MAX_NETS = 32;
 482 my $MAX_UNUSED_DISKS = 8;
 483 my $MAX_HOSTPCI_DEVICES = 4;
 484 my $MAX_SERIAL_PORTS = 4;
 485 my $MAX_PARALLEL_PORTS = 3;
 486 my $MAX_NUMA = 8;
 487 my $MAX_MEM = 4194304;
 488 my $STATICMEM = 1024;
 489
 490 my $numadesc = {
 491     optional => 1,
 492     type => 'string', format => 'pve-qm-numanode',
 493     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 494     description => "numa topology",
 495 };
 496 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 497
 498 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 499     $confdesc->{"numa$i"} = $numadesc;
 500 }
 501
 502 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 503                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 504                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 505 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 506
 507 my $netdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-net',
 510     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 511     description => <<EODESCR,
 512 Specify network devices.
 513
 514 MODEL is one of: $nic_model_list_txt
 515
 516 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 517 automatically generated if not specified.
 518
 519 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 520
 521 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 522
 523 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 524
 525 10.0.2.2   Gateway
 526 10.0.2.3   DNS Server
 527 10.0.2.4   SMB Server
 528
 529 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 530
 531 EODESCR
 532 };
 533 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 534
 535 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 536     $confdesc->{"net$i"} = $netdesc;
 537 }
 538
 539 my $drivename_hash;
 540
 541 my $idedesc = {
 542     optional => 1,
 543     type => 'string', format => 'pve-qm-drive',
 544     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 545     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 546 };
 547 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 548
 549 my $scsidesc = {
 550     optional => 1,
 551     type => 'string', format => 'pve-qm-drive',
 552     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on]',
 553     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 554 };
 555 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 556
 557 my $satadesc = {
 558     optional => 1,
 559     type => 'string', format => 'pve-qm-drive',
 560     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 561     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 562 };
 563 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 564
 565 my $virtiodesc = {
 566     optional => 1,
 567     type => 'string', format => 'pve-qm-drive',
 568     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 569     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 570 };
 571 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 572
 573 my $usbdesc = {
 574     optional => 1,
 575     type => 'string', format => 'pve-qm-usb-device',
 576     typetext => 'host=HOSTUSBDEVICE|spice',
 577     description => <<EODESCR,
 578 Configure an USB device (n is 0 to 4). This can be used to
 579 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 580
 581 'bus-port(.port)*' (decimal numbers) or
 582 'vendor_id:product_id' (hexadeciaml numbers)
 583
 584 You can use the 'lsusb -t' command to list existing usb devices.
 585
 586 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 587
 588 The value 'spice' can be used to add a usb redirection devices for spice.
 589
 590 EODESCR
 591 };
 592 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 593
 594 my $hostpcidesc = {
 595         optional => 1,
 596         type => 'string', format => 'pve-qm-hostpci',
 597         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 598         description => <<EODESCR,
 599 Map host pci devices. HOSTPCIDEVICE syntax is:
 600
 601 'bus:dev.func' (hexadecimal numbers)
 602
 603 You can us the 'lspci' command to list existing pci devices.
 604
 605 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 606
 607 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 608
 609 Experimental: user reported problems with this option.
 610 EODESCR
 611 };
 612 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 613
 614 my $serialdesc = {
 615         optional => 1,
 616         type => 'string',
 617         pattern => '(/dev/.+|socket)',
 618         description =>  <<EODESCR,
 619 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 620
 621 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 622
 623 Experimental: user reported problems with this option.
 624 EODESCR
 625 };
 626
 627 my $paralleldesc= {
 628         optional => 1,
 629         type => 'string',
 630         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 631         description =>  <<EODESCR,
 632 Map host parallel devices (n is 0 to 2).
 633
 634 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 635
 636 Experimental: user reported problems with this option.
 637 EODESCR
 638 };
 639
 640 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 641     $confdesc->{"parallel$i"} = $paralleldesc;
 642 }
 643
 644 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 645     $confdesc->{"serial$i"} = $serialdesc;
 646 }
 647
 648 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 649     $confdesc->{"hostpci$i"} = $hostpcidesc;
 650 }
 651
 652 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 653     $drivename_hash->{"ide$i"} = 1;
 654     $confdesc->{"ide$i"} = $idedesc;
 655 }
 656
 657 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 658     $drivename_hash->{"sata$i"} = 1;
 659     $confdesc->{"sata$i"} = $satadesc;
 660 }
 661
 662 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 663     $drivename_hash->{"scsi$i"} = 1;
 664     $confdesc->{"scsi$i"} = $scsidesc ;
 665 }
 666
 667 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 668     $drivename_hash->{"virtio$i"} = 1;
 669     $confdesc->{"virtio$i"} = $virtiodesc;
 670 }
 671
 672 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 673     $confdesc->{"usb$i"} = $usbdesc;
 674 }
 675
 676 my $unuseddesc = {
 677     optional => 1,
 678     type => 'string', format => 'pve-volume-id',
 679     description => "Reference to unused volumes.",
 680 };
 681
 682 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 683     $confdesc->{"unused$i"} = $unuseddesc;
 684 }
 685
 686 my $kvm_api_version = 0;
 687
 688 sub kvm_version {
 689
 690     return $kvm_api_version if $kvm_api_version;
 691
 692     my $fh = IO::File->new("</dev/kvm") ||
 693         return 0;
 694
 695     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 696         $kvm_api_version = $v;
 697     }
 698
 699     $fh->close();
 700
 701     return  $kvm_api_version;
 702 }
 703
 704 my $kvm_user_version;
 705
 706 sub kvm_user_version {
 707
 708     return $kvm_user_version if $kvm_user_version;
 709
 710     $kvm_user_version = 'unknown';
 711
 712     my $tmp = `kvm -help 2>/dev/null`;
 713
 714     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 715         $kvm_user_version = $2;
 716     }
 717
 718     return $kvm_user_version;
 719
 720 }
 721
 722 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 723
 724 sub disknames {
 725     # order is important - used to autoselect boot disk
 726     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 727             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 728             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 729             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 730 }
 731
 732 sub valid_drivename {
 733     my $dev = shift;
 734
 735     return defined($drivename_hash->{$dev});
 736 }
 737
 738 sub option_exists {
 739     my $key = shift;
 740     return defined($confdesc->{$key});
 741 }
 742
 743 sub nic_models {
 744     return $nic_model_list;
 745 }
 746
 747 sub os_list_description {
 748
 749     return {
 750         other => 'Other',
 751         wxp => 'Windows XP',
 752         w2k => 'Windows 2000',
 753         w2k3 =>, 'Windows 2003',
 754         w2k8 => 'Windows 2008',
 755         wvista => 'Windows Vista',
 756         win7 => 'Windows 7',
 757         win8 => 'Windows 8/2012',
 758         l24 => 'Linux 2.4',
 759         l26 => 'Linux 2.6',
 760     };
 761 }
 762
 763 my $cdrom_path;
 764
 765 sub get_cdrom_path {
 766
 767     return  $cdrom_path if $cdrom_path;
 768
 769     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 770     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 771     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 772 }
 773
 774 sub get_iso_path {
 775     my ($storecfg, $vmid, $cdrom) = @_;
 776
 777     if ($cdrom eq 'cdrom') {
 778         return get_cdrom_path();
 779     } elsif ($cdrom eq 'none') {
 780         return '';
 781     } elsif ($cdrom =~ m|^/|) {
 782         return $cdrom;
 783     } else {
 784         return PVE::Storage::path($storecfg, $cdrom);
 785     }
 786 }
 787
 788 # try to convert old style file names to volume IDs
 789 sub filename_to_volume_id {
 790     my ($vmid, $file, $media) = @_;
 791
 792     if (!($file eq 'none' || $file eq 'cdrom' ||
 793           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 794
 795         return undef if $file =~ m|/|;
 796
 797         if ($media && $media eq 'cdrom') {
 798             $file = "local:iso/$file";
 799         } else {
 800             $file = "local:$vmid/$file";
 801         }
 802     }
 803
 804     return $file;
 805 }
 806
 807 sub verify_media_type {
 808     my ($opt, $vtype, $media) = @_;
 809
 810     return if !$media;
 811
 812     my $etype;
 813     if ($media eq 'disk') {
 814         $etype = 'images';
 815     } elsif ($media eq 'cdrom') {
 816         $etype = 'iso';
 817     } else {
 818         die "internal error";
 819     }
 820
 821     return if ($vtype eq $etype);
 822
 823     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 824 }
 825
 826 sub cleanup_drive_path {
 827     my ($opt, $storecfg, $drive) = @_;
 828
 829     # try to convert filesystem paths to volume IDs
 830
 831     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 832         ($drive->{file} !~ m|^/dev/.+|) &&
 833         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 834         ($drive->{file} !~ m/^\d+$/)) {
 835         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 836         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 837         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 838         verify_media_type($opt, $vtype, $drive->{media});
 839         $drive->{file} = $volid;
 840     }
 841
 842     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 843 }
 844
 845 sub create_conf_nolock {
 846     my ($vmid, $settings) = @_;
 847
 848     my $filename = config_file($vmid);
 849
 850     die "configuration file '$filename' already exists\n" if -f $filename;
 851
 852     my $defaults = load_defaults();
 853
 854     $settings->{name} = "vm$vmid" if !$settings->{name};
 855     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 856
 857     my $data = '';
 858     foreach my $opt (keys %$settings) {
 859         next if !$confdesc->{$opt};
 860
 861         my $value = $settings->{$opt};
 862         next if !$value;
 863
 864         $data .= "$opt: $value\n";
 865     }
 866
 867     PVE::Tools::file_set_contents($filename, $data);
 868 }
 869
 870 sub parse_hotplug_features {
 871     my ($data) = @_;
 872
 873     my $res = {};
 874
 875     return $res if $data eq '0';
 876
 877     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 878
 879     foreach my $feature (PVE::Tools::split_list($data)) {
 880         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 881             $res->{$1} = 1;
 882         } else {
 883             warn "ignoring unknown hotplug feature '$feature'\n";
 884         }
 885     }
 886     return $res;
 887 }
 888
 889 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 890 sub pve_verify_hotplug_features {
 891     my ($value, $noerr) = @_;
 892
 893     return $value if parse_hotplug_features($value);
 894
 895     return undef if $noerr;
 896
 897     die "unable to parse hotplug option\n";
 898 }
 899
 900 my $parse_size = sub {
 901     my ($value) = @_;
 902
 903     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 904     my ($size, $unit) = ($1, $3);
 905     if ($unit) {
 906         if ($unit eq 'K') {
 907             $size = $size * 1024;
 908         } elsif ($unit eq 'M') {
 909             $size = $size * 1024 * 1024;
 910         } elsif ($unit eq 'G') {
 911             $size = $size * 1024 * 1024 * 1024;
 912         }
 913     }
 914     return int($size);
 915 };
 916
 917 my $format_size = sub {
 918     my ($size) = @_;
 919
 920     $size = int($size);
 921
 922     my $kb = int($size/1024);
 923     return $size if $kb*1024 != $size;
 924
 925     my $mb = int($kb/1024);
 926     return "${kb}K" if $mb*1024 != $kb;
 927
 928     my $gb = int($mb/1024);
 929     return "${mb}M" if $gb*1024 != $mb;
 930
 931     return "${gb}G";
 932 };
 933
 934 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 935 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 936 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 937 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 938
 939 sub parse_drive {
 940     my ($key, $data) = @_;
 941
 942     my $res = {};
 943
 944     # $key may be undefined - used to verify JSON parameters
 945     if (!defined($key)) {
 946         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 947         $res->{index} = 0;
 948     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 949         $res->{interface} = $1;
 950         $res->{index} = $2;
 951     } else {
 952         return undef;
 953     }
 954
 955     foreach my $p (split (/,/, $data)) {
 956         next if $p =~ m/^\s*$/;
 957
 958         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread)=(.+)$/) {
 959             my ($k, $v) = ($1, $2);
 960
 961             $k = 'file' if $k eq 'volume';
 962
 963             return undef if defined $res->{$k};
 964
 965             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 966                 return undef if !$v || $v !~ m/^\d+/;
 967                 $k = "m$k";
 968                 $v = sprintf("%.3f", $v / (1024*1024));
 969             }
 970             $res->{$k} = $v;
 971         } else {
 972             if (!$res->{file} && $p !~ m/=/) {
 973                 $res->{file} = $p;
 974             } else {
 975                 return undef;
 976             }
 977         }
 978     }
 979
 980     return undef if !$res->{file};
 981
 982     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 983         $res->{format} = $1;
 984     }
 985
 986     return undef if $res->{cache} &&
 987         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 988     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 989     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 990     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 991     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 992     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 993     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 994     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 995     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 996     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 997     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 998     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 999     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
1000     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
1001
1002     return undef if $res->{mbps_rd} && $res->{mbps};
1003     return undef if $res->{mbps_wr} && $res->{mbps};
1004
1005     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1006     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1007     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1008     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1009     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1010     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1011
1012     return undef if $res->{iops_rd} && $res->{iops};
1013     return undef if $res->{iops_wr} && $res->{iops};
1014
1015
1016     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1017     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1018     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1019     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1020     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1021     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1022
1023
1024     if ($res->{size}) {
1025         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1026     }
1027
1028     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1029         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1030         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1031         return undef if $res->{interface} eq 'virtio';
1032     }
1033
1034     # rerror does not work with scsi drives
1035     if ($res->{rerror}) {
1036         return undef if $res->{interface} eq 'scsi';
1037     }
1038
1039     return $res;
1040 }
1041
1042 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1043
1044 sub print_drive {
1045     my ($vmid, $drive) = @_;
1046
1047     my $opts = '';
1048     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread') {
1049         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1050     }
1051
1052     if ($drive->{size}) {
1053         $opts .= ",size=" . &$format_size($drive->{size});
1054     }
1055
1056     return "$drive->{file}$opts";
1057 }
1058
1059 sub scsi_inquiry {
1060     my($fh, $noerr) = @_;
1061
1062     my $SG_IO = 0x2285;
1063     my $SG_GET_VERSION_NUM = 0x2282;
1064
1065     my $versionbuf = "\x00" x 8;
1066     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1067     if (!$ret) {
1068         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1069         return undef;
1070     }
1071     my $version = unpack("I", $versionbuf);
1072     if ($version < 30000) {
1073         die "scsi generic interface too old\n"  if !$noerr;
1074         return undef;
1075     }
1076
1077     my $buf = "\x00" x 36;
1078     my $sensebuf = "\x00" x 8;
1079     my $cmd = pack("C x3 C x1", 0x12, 36);
1080
1081     # see /usr/include/scsi/sg.h
1082     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1083
1084     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1085                       length($sensebuf), 0, length($buf), $buf,
1086                       $cmd, $sensebuf, 6000);
1087
1088     $ret = ioctl($fh, $SG_IO, $packet);
1089     if (!$ret) {
1090         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1091         return undef;
1092     }
1093
1094     my @res = unpack($sg_io_hdr_t, $packet);
1095     if ($res[17] || $res[18]) {
1096         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1097         return undef;
1098     }
1099
1100     my $res = {};
1101     (my $byte0, my $byte1, $res->{vendor},
1102      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1103
1104     $res->{removable} = $byte1 & 128 ? 1 : 0;
1105     $res->{type} = $byte0 & 31;
1106
1107     return $res;
1108 }
1109
1110 sub path_is_scsi {
1111     my ($path) = @_;
1112
1113     my $fh = IO::File->new("+<$path") || return undef;
1114     my $res = scsi_inquiry($fh, 1);
1115     close($fh);
1116
1117     return $res;
1118 }
1119
1120 sub machine_type_is_q35 {
1121     my ($conf) = @_;
1122
1123     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1124 }
1125
1126 sub print_tabletdevice_full {
1127     my ($conf) = @_;
1128
1129     my $q35 = machine_type_is_q35($conf);
1130
1131     # we use uhci for old VMs because tablet driver was buggy in older qemu
1132     my $usbbus = $q35 ? "ehci" : "uhci";
1133
1134     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1135 }
1136
1137 sub print_drivedevice_full {
1138     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1139
1140     my $device = '';
1141     my $maxdev = 0;
1142
1143     if ($drive->{interface} eq 'virtio') {
1144         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1145         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1146         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1147     } elsif ($drive->{interface} eq 'scsi') {
1148
1149         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1150         my $unit = $drive->{index} % $maxdev;
1151         my $devicetype = 'hd';
1152         my $path = '';
1153         if (drive_is_cdrom($drive)) {
1154               $devicetype = 'cd';
1155         } else {
1156               if ($drive->{file} =~ m|^/|) {
1157                   $path = $drive->{file};
1158               } else {
1159                   $path = PVE::Storage::path($storecfg, $drive->{file});
1160               }
1161
1162               if($path =~ m/^iscsi\:\/\//){
1163                   $devicetype = 'generic';
1164               } else {
1165                   if (my $info = path_is_scsi($path)) {
1166                       if ($info->{type} == 0) {
1167                           $devicetype = 'block';
1168                       } elsif ($info->{type} == 1) { # tape
1169                           $devicetype = 'generic';
1170                       }
1171                   }
1172               }
1173          }
1174
1175         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1176             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1177         } else {
1178             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1179         }
1180
1181     } elsif ($drive->{interface} eq 'ide'){
1182         $maxdev = 2;
1183         my $controller = int($drive->{index} / $maxdev);
1184         my $unit = $drive->{index} % $maxdev;
1185         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1186
1187         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1188     } elsif ($drive->{interface} eq 'sata'){
1189         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1190         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1191         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1192     } elsif ($drive->{interface} eq 'usb') {
1193         die "implement me";
1194         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1195     } else {
1196         die "unsupported interface type";
1197     }
1198
1199     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1200
1201     return $device;
1202 }
1203
1204 sub get_initiator_name {
1205     my $initiator;
1206
1207     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1208     while (defined(my $line = <$fh>)) {
1209         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1210         $initiator = $1;
1211         last;
1212     }
1213     $fh->close();
1214
1215     return $initiator;
1216 }
1217
1218 sub print_drive_full {
1219     my ($storecfg, $vmid, $drive) = @_;
1220
1221     my $opts = '';
1222     foreach my $o (@qemu_drive_options) {
1223         next if $o eq 'bootindex';
1224         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1225     }
1226
1227     foreach my $o (qw(bps bps_rd bps_wr)) {
1228         my $v = $drive->{"m$o"};
1229         $opts .= ",$o=" . int($v*1024*1024) if $v;
1230     }
1231
1232     # aio native works only with O_DIRECT
1233     if (!$drive->{aio}) {
1234         if(!$drive->{cache} || $drive->{cache} eq 'none' || $drive->{cache} eq 'directsync') {
1235             $opts .= ",aio=native";
1236         } else {
1237             $opts .= ",aio=threads";
1238         }
1239     }
1240
1241
1242     my $path;
1243     my $volid = $drive->{file};
1244     if (drive_is_cdrom($drive)) {
1245         $path = get_iso_path($storecfg, $vmid, $volid);
1246     } else {
1247         if ($volid =~ m|^/|) {
1248             $path = $volid;
1249         } else {
1250             $path = PVE::Storage::path($storecfg, $volid);
1251         }
1252     }
1253
1254     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1255
1256     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1257     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1258
1259     my $pathinfo = $path ? "file=$path," : '';
1260
1261     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1262 }
1263
1264 sub print_netdevice_full {
1265     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1266
1267     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1268
1269     my $device = $net->{model};
1270     if ($net->{model} eq 'virtio') {
1271          $device = 'virtio-net-pci';
1272      };
1273
1274     my $pciaddr = print_pci_addr("$netid", $bridges);
1275     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1276     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1277         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1278         my $vectors = $net->{queues} * 2 + 2;
1279         $tmpstr .= ",vectors=$vectors,mq=on";
1280     }
1281     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1282     return $tmpstr;
1283 }
1284
1285 sub print_netdev_full {
1286     my ($vmid, $conf, $net, $netid) = @_;
1287
1288     my $i = '';
1289     if ($netid =~ m/^net(\d+)$/) {
1290         $i = int($1);
1291     }
1292
1293     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1294
1295     my $ifname = "tap${vmid}i$i";
1296
1297     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1298     die "interface name '$ifname' is too long (max 15 character)\n"
1299         if length($ifname) >= 16;
1300
1301     my $vhostparam = '';
1302     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1303
1304     my $vmname = $conf->{name} || "vm$vmid";
1305
1306     my $netdev = "";
1307
1308     if ($net->{bridge}) {
1309         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1310     } else {
1311         $netdev = "type=user,id=$netid,hostname=$vmname";
1312     }
1313
1314     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1315
1316     return $netdev;
1317 }
1318
1319 sub drive_is_cdrom {
1320     my ($drive) = @_;
1321
1322     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1323
1324 }
1325
1326 sub parse_numa {
1327     my ($data) = @_;
1328
1329     my $res = {};
1330
1331     foreach my $kvp (split(/,/, $data)) {
1332
1333         if ($kvp =~ m/^memory=(\S+)$/) {
1334             $res->{memory} = $1;
1335         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1336             $res->{policy} = $1;
1337         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1338             $res->{cpus}->{start} = $1;
1339             $res->{cpus}->{end} = $3;
1340         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1341             $res->{hostnodes}->{start} = $1;
1342             $res->{hostnodes}->{end} = $3;
1343         } else {
1344             return undef;
1345         }
1346     }
1347
1348     return $res;
1349 }
1350
1351 sub parse_hostpci {
1352     my ($value) = @_;
1353
1354     return undef if !$value;
1355
1356
1357     my @list = split(/,/, $value);
1358     my $found;
1359
1360     my $res = {};
1361     foreach my $kv (@list) {
1362
1363         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1364             $found = 1;
1365             if(defined($4)){
1366                 push @{$res->{pciid}}, { id => $2 , function => $4};
1367
1368             }else{
1369                 my $pcidevices = lspci($2);
1370                 $res->{pciid} = $pcidevices->{$2};
1371             }
1372         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1373             $res->{driver} = $1;
1374         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1375             $res->{rombar} = $1;
1376         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1377             $res->{'x-vga'} = $1;
1378         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1379             $res->{pcie} = 1 if $1 == 1;
1380         } else {
1381             warn "unknown hostpci setting '$kv'\n";
1382         }
1383     }
1384
1385     return undef if !$found;
1386
1387     return $res;
1388 }
1389
1390 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1391 sub parse_net {
1392     my ($data) = @_;
1393
1394     my $res = {};
1395
1396     foreach my $kvp (split(/,/, $data)) {
1397
1398         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1399             my $model = lc($1);
1400             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1401             $res->{model} = $model;
1402             $res->{macaddr} = $mac;
1403         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1404             $res->{bridge} = $1;
1405         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1406             $res->{queues} = $1;
1407         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1408             $res->{rate} = $1;
1409         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1410             $res->{tag} = $1;
1411         } elsif ($kvp =~ m/^firewall=([01])$/) {
1412             $res->{firewall} = $1;
1413         } elsif ($kvp =~ m/^link_down=([01])$/) {
1414             $res->{link_down} = $1;
1415         } else {
1416             return undef;
1417         }
1418
1419     }
1420
1421     return undef if !$res->{model};
1422
1423     return $res;
1424 }
1425
1426 sub print_net {
1427     my $net = shift;
1428
1429     my $res = "$net->{model}";
1430     $res .= "=$net->{macaddr}" if $net->{macaddr};
1431     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1432     $res .= ",rate=$net->{rate}" if $net->{rate};
1433     $res .= ",tag=$net->{tag}" if $net->{tag};
1434     $res .= ",firewall=1" if $net->{firewall};
1435     $res .= ",link_down=1" if $net->{link_down};
1436     $res .= ",queues=$net->{queues}" if $net->{queues};
1437
1438     return $res;
1439 }
1440
1441 sub add_random_macs {
1442     my ($settings) = @_;
1443
1444     foreach my $opt (keys %$settings) {
1445         next if $opt !~ m/^net(\d+)$/;
1446         my $net = parse_net($settings->{$opt});
1447         next if !$net;
1448         $settings->{$opt} = print_net($net);
1449     }
1450 }
1451
1452 sub add_unused_volume {
1453     my ($config, $volid) = @_;
1454
1455     my $key;
1456     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1457         my $test = "unused$ind";
1458         if (my $vid = $config->{$test}) {
1459             return if $vid eq $volid; # do not add duplicates
1460         } else {
1461             $key = $test;
1462         }
1463     }
1464
1465     die "To many unused volume - please delete them first.\n" if !$key;
1466
1467     $config->{$key} = $volid;
1468
1469     return $key;
1470 }
1471
1472 sub vm_is_volid_owner {
1473     my ($storecfg, $vmid, $volid) = @_;
1474
1475     if ($volid !~  m|^/|) {
1476         my ($path, $owner);
1477         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1478         if ($owner && ($owner == $vmid)) {
1479             return 1;
1480         }
1481     }
1482
1483     return undef;
1484 }
1485
1486 sub vmconfig_delete_pending_option {
1487     my ($conf, $key) = @_;
1488
1489     delete $conf->{pending}->{$key};
1490     my $pending_delete_hash = { $key => 1 };
1491     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1492         $pending_delete_hash->{$opt} = 1;
1493     }
1494     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1495 }
1496
1497 sub vmconfig_undelete_pending_option {
1498     my ($conf, $key) = @_;
1499
1500     my $pending_delete_hash = {};
1501     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1502         $pending_delete_hash->{$opt} = 1;
1503     }
1504     delete $pending_delete_hash->{$key};
1505
1506     my @keylist = keys %$pending_delete_hash;
1507     if (scalar(@keylist)) {
1508         $conf->{pending}->{delete} = join(',', @keylist);
1509     } else {
1510         delete $conf->{pending}->{delete};
1511     }
1512 }
1513
1514 sub vmconfig_register_unused_drive {
1515     my ($storecfg, $vmid, $conf, $drive) = @_;
1516
1517     if (!drive_is_cdrom($drive)) {
1518         my $volid = $drive->{file};
1519         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1520             add_unused_volume($conf, $volid, $vmid);
1521         }
1522     }
1523 }
1524
1525 sub vmconfig_cleanup_pending {
1526     my ($conf) = @_;
1527
1528     # remove pending changes when nothing changed
1529     my $changes;
1530     foreach my $opt (keys %{$conf->{pending}}) {
1531         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1532             $changes = 1;
1533             delete $conf->{pending}->{$opt};
1534         }
1535     }
1536
1537     # remove delete if option is not set
1538     my $pending_delete_hash = {};
1539     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1540         if (defined($conf->{$opt})) {
1541             $pending_delete_hash->{$opt} = 1;
1542         } else {
1543             $changes = 1;
1544         }
1545     }
1546
1547     my @keylist = keys %$pending_delete_hash;
1548     if (scalar(@keylist)) {
1549         $conf->{pending}->{delete} = join(',', @keylist);
1550     } else {
1551         delete $conf->{pending}->{delete};
1552     }
1553
1554     return $changes;
1555 }
1556
1557 my $valid_smbios1_options = {
1558     manufacturer => '\S+',
1559     product => '\S+',
1560     version => '\S+',
1561     serial => '\S+',
1562     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1563     sku => '\S+',
1564     family => '\S+',
1565 };
1566
1567 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1568 sub parse_smbios1 {
1569     my ($data) = @_;
1570
1571     my $res = {};
1572
1573     foreach my $kvp (split(/,/, $data)) {
1574         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1575         my ($k, $v) = split(/=/, $kvp);
1576         return undef if !defined($k) || !defined($v);
1577         return undef if !$valid_smbios1_options->{$k};
1578         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1579         $res->{$k} = $v;
1580     }
1581
1582     return $res;
1583 }
1584
1585 sub print_smbios1 {
1586     my ($smbios1) = @_;
1587
1588     my $data = '';
1589     foreach my $k (keys %$smbios1) {
1590         next if !defined($smbios1->{$k});
1591         next if !$valid_smbios1_options->{$k};
1592         $data .= ',' if $data;
1593         $data .= "$k=$smbios1->{$k}";
1594     }
1595     return $data;
1596 }
1597
1598 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1599 sub verify_smbios1 {
1600     my ($value, $noerr) = @_;
1601
1602     return $value if parse_smbios1($value);
1603
1604     return undef if $noerr;
1605
1606     die "unable to parse smbios (type 1) options\n";
1607 }
1608
1609 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1610 sub verify_bootdisk {
1611     my ($value, $noerr) = @_;
1612
1613     return $value if valid_drivename($value);
1614
1615     return undef if $noerr;
1616
1617     die "invalid boot disk '$value'\n";
1618 }
1619
1620 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1621 sub verify_numa {
1622     my ($value, $noerr) = @_;
1623
1624     return $value if parse_numa($value);
1625
1626     return undef if $noerr;
1627
1628     die "unable to parse numa options\n";
1629 }
1630
1631 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1632 sub verify_net {
1633     my ($value, $noerr) = @_;
1634
1635     return $value if parse_net($value);
1636
1637     return undef if $noerr;
1638
1639     die "unable to parse network options\n";
1640 }
1641
1642 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1643 sub verify_drive {
1644     my ($value, $noerr) = @_;
1645
1646     return $value if parse_drive(undef, $value);
1647
1648     return undef if $noerr;
1649
1650     die "unable to parse drive options\n";
1651 }
1652
1653 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1654 sub verify_hostpci {
1655     my ($value, $noerr) = @_;
1656
1657     return $value if parse_hostpci($value);
1658
1659     return undef if $noerr;
1660
1661     die "unable to parse pci id\n";
1662 }
1663
1664 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1665 sub verify_watchdog {
1666     my ($value, $noerr) = @_;
1667
1668     return $value if parse_watchdog($value);
1669
1670     return undef if $noerr;
1671
1672     die "unable to parse watchdog options\n";
1673 }
1674
1675 sub parse_watchdog {
1676     my ($value) = @_;
1677
1678     return undef if !$value;
1679
1680     my $res = {};
1681
1682     foreach my $p (split(/,/, $value)) {
1683         next if $p =~ m/^\s*$/;
1684
1685         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1686             $res->{model} = $2;
1687         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1688             $res->{action} = $2;
1689         } else {
1690             return undef;
1691         }
1692     }
1693
1694     return $res;
1695 }
1696
1697 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1698 sub verify_startup {
1699     my ($value, $noerr) = @_;
1700
1701     return $value if parse_startup($value);
1702
1703     return undef if $noerr;
1704
1705     die "unable to parse startup options\n";
1706 }
1707
1708 sub parse_startup {
1709     my ($value) = @_;
1710
1711     return undef if !$value;
1712
1713     my $res = {};
1714
1715     foreach my $p (split(/,/, $value)) {
1716         next if $p =~ m/^\s*$/;
1717
1718         if ($p =~ m/^(order=)?(\d+)$/) {
1719             $res->{order} = $2;
1720         } elsif ($p =~ m/^up=(\d+)$/) {
1721             $res->{up} = $1;
1722         } elsif ($p =~ m/^down=(\d+)$/) {
1723             $res->{down} = $1;
1724         } else {
1725             return undef;
1726         }
1727     }
1728
1729     return $res;
1730 }
1731
1732 sub parse_usb_device {
1733     my ($value) = @_;
1734
1735     return undef if !$value;
1736
1737     my @dl = split(/,/, $value);
1738     my $found;
1739
1740     my $res = {};
1741     foreach my $v (@dl) {
1742         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1743             $found = 1;
1744             $res->{vendorid} = $2;
1745             $res->{productid} = $4;
1746         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1747             $found = 1;
1748             $res->{hostbus} = $1;
1749             $res->{hostport} = $2;
1750         } elsif ($v =~ m/^spice$/) {
1751             $found = 1;
1752             $res->{spice} = 1;
1753         } else {
1754             return undef;
1755         }
1756     }
1757     return undef if !$found;
1758
1759     return $res;
1760 }
1761
1762 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1763 sub verify_usb_device {
1764     my ($value, $noerr) = @_;
1765
1766     return $value if parse_usb_device($value);
1767
1768     return undef if $noerr;
1769
1770     die "unable to parse usb device\n";
1771 }
1772
1773 # add JSON properties for create and set function
1774 sub json_config_properties {
1775     my $prop = shift;
1776
1777     foreach my $opt (keys %$confdesc) {
1778         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1779         $prop->{$opt} = $confdesc->{$opt};
1780     }
1781
1782     return $prop;
1783 }
1784
1785 sub check_type {
1786     my ($key, $value) = @_;
1787
1788     die "unknown setting '$key'\n" if !$confdesc->{$key};
1789
1790     my $type = $confdesc->{$key}->{type};
1791
1792     if (!defined($value)) {
1793         die "got undefined value\n";
1794     }
1795
1796     if ($value =~ m/[\n\r]/) {
1797         die "property contains a line feed\n";
1798     }
1799
1800     if ($type eq 'boolean') {
1801         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1802         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1803         die "type check ('boolean') failed - got '$value'\n";
1804     } elsif ($type eq 'integer') {
1805         return int($1) if $value =~ m/^(\d+)$/;
1806         die "type check ('integer') failed - got '$value'\n";
1807     } elsif ($type eq 'number') {
1808         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1809         die "type check ('number') failed - got '$value'\n";
1810     } elsif ($type eq 'string') {
1811         if (my $fmt = $confdesc->{$key}->{format}) {
1812             if ($fmt eq 'pve-qm-drive') {
1813                 # special case - we need to pass $key to parse_drive()
1814                 my $drive = parse_drive($key, $value);
1815                 return $value if $drive;
1816                 die "unable to parse drive options\n";
1817             }
1818             PVE::JSONSchema::check_format($fmt, $value);
1819             return $value;
1820         }
1821         $value =~ s/^\"(.*)\"$/$1/;
1822         return $value;
1823     } else {
1824         die "internal error"
1825     }
1826 }
1827
1828 sub lock_config_full {
1829     my ($vmid, $timeout, $code, @param) = @_;
1830
1831     my $filename = config_file_lock($vmid);
1832
1833     my $res = lock_file($filename, $timeout, $code, @param);
1834
1835     die $@ if $@;
1836
1837     return $res;
1838 }
1839
1840 sub lock_config_mode {
1841     my ($vmid, $timeout, $shared, $code, @param) = @_;
1842
1843     my $filename = config_file_lock($vmid);
1844
1845     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1846
1847     die $@ if $@;
1848
1849     return $res;
1850 }
1851
1852 sub lock_config {
1853     my ($vmid, $code, @param) = @_;
1854
1855     return lock_config_full($vmid, 10, $code, @param);
1856 }
1857
1858 sub cfs_config_path {
1859     my ($vmid, $node) = @_;
1860
1861     $node = $nodename if !$node;
1862     return "nodes/$node/qemu-server/$vmid.conf";
1863 }
1864
1865 sub check_iommu_support{
1866     #fixme : need to check IOMMU support
1867     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1868
1869     my $iommu=1;
1870     return $iommu;
1871
1872 }
1873
1874 sub config_file {
1875     my ($vmid, $node) = @_;
1876
1877     my $cfspath = cfs_config_path($vmid, $node);
1878     return "/etc/pve/$cfspath";
1879 }
1880
1881 sub config_file_lock {
1882     my ($vmid) = @_;
1883
1884     return "$lock_dir/lock-$vmid.conf";
1885 }
1886
1887 sub touch_config {
1888     my ($vmid) = @_;
1889
1890     my $conf = config_file($vmid);
1891     utime undef, undef, $conf;
1892 }
1893
1894 sub destroy_vm {
1895     my ($storecfg, $vmid, $keep_empty_config) = @_;
1896
1897     my $conffile = config_file($vmid);
1898
1899     my $conf = load_config($vmid);
1900
1901     check_lock($conf);
1902
1903     # only remove disks owned by this VM
1904     foreach_drive($conf, sub {
1905         my ($ds, $drive) = @_;
1906
1907         return if drive_is_cdrom($drive);
1908
1909         my $volid = $drive->{file};
1910
1911         return if !$volid || $volid =~ m|^/|;
1912
1913         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1914         return if !$path || !$owner || ($owner != $vmid);
1915
1916         PVE::Storage::vdisk_free($storecfg, $volid);
1917     });
1918
1919     if ($keep_empty_config) {
1920         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1921     } else {
1922         unlink $conffile;
1923     }
1924
1925     # also remove unused disk
1926     eval {
1927         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1928
1929         eval {
1930             PVE::Storage::foreach_volid($dl, sub {
1931                 my ($volid, $sid, $volname, $d) = @_;
1932                 PVE::Storage::vdisk_free($storecfg, $volid);
1933             });
1934         };
1935         warn $@ if $@;
1936
1937     };
1938     warn $@ if $@;
1939 }
1940
1941 sub load_config {
1942     my ($vmid, $node) = @_;
1943
1944     my $cfspath = cfs_config_path($vmid, $node);
1945
1946     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1947
1948     die "no such VM ('$vmid')\n" if !defined($conf);
1949
1950     return $conf;
1951 }
1952
1953 sub parse_vm_config {
1954     my ($filename, $raw) = @_;
1955
1956     return undef if !defined($raw);
1957
1958     my $res = {
1959         digest => Digest::SHA::sha1_hex($raw),
1960         snapshots => {},
1961         pending => {},
1962     };
1963
1964     $filename =~ m|/qemu-server/(\d+)\.conf$|
1965         || die "got strange filename '$filename'";
1966
1967     my $vmid = $1;
1968
1969     my $conf = $res;
1970     my $descr = '';
1971     my $section = '';
1972
1973     my @lines = split(/\n/, $raw);
1974     foreach my $line (@lines) {
1975         next if $line =~ m/^\s*$/;
1976
1977         if ($line =~ m/^\[PENDING\]\s*$/i) {
1978             $section = 'pending';
1979             $conf->{description} = $descr if $descr;
1980             $descr = '';
1981             $conf = $res->{$section} = {};
1982             next;
1983
1984         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1985             $section = $1;
1986             $conf->{description} = $descr if $descr;
1987             $descr = '';
1988             $conf = $res->{snapshots}->{$section} = {};
1989             next;
1990         }
1991
1992         if ($line =~ m/^\#(.*)\s*$/) {
1993             $descr .= PVE::Tools::decode_text($1) . "\n";
1994             next;
1995         }
1996
1997         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1998             $descr .= PVE::Tools::decode_text($2);
1999         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2000             $conf->{snapstate} = $1;
2001         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2002             my $key = $1;
2003             my $value = $2;
2004             $conf->{$key} = $value;
2005         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2006             my $value = $1;
2007             if ($section eq 'pending') {
2008                 $conf->{delete} = $value; # we parse this later
2009             } else {
2010                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2011             }
2012         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2013             my $key = $1;
2014             my $value = $2;
2015             eval { $value = check_type($key, $value); };
2016             if ($@) {
2017                 warn "vm $vmid - unable to parse value of '$key' - $@";
2018             } else {
2019                 my $fmt = $confdesc->{$key}->{format};
2020                 if ($fmt && $fmt eq 'pve-qm-drive') {
2021                     my $v = parse_drive($key, $value);
2022                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2023                         $v->{file} = $volid;
2024                         $value = print_drive($vmid, $v);
2025                     } else {
2026                         warn "vm $vmid - unable to parse value of '$key'\n";
2027                         next;
2028                     }
2029                 }
2030
2031                 if ($key eq 'cdrom') {
2032                     $conf->{ide2} = $value;
2033                 } else {
2034                     $conf->{$key} = $value;
2035                 }
2036             }
2037         }
2038     }
2039
2040     $conf->{description} = $descr if $descr;
2041
2042     delete $res->{snapstate}; # just to be sure
2043
2044     return $res;
2045 }
2046
2047 sub write_vm_config {
2048     my ($filename, $conf) = @_;
2049
2050     delete $conf->{snapstate}; # just to be sure
2051
2052     if ($conf->{cdrom}) {
2053         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2054         $conf->{ide2} = $conf->{cdrom};
2055         delete $conf->{cdrom};
2056     }
2057
2058     # we do not use 'smp' any longer
2059     if ($conf->{sockets}) {
2060         delete $conf->{smp};
2061     } elsif ($conf->{smp}) {
2062         $conf->{sockets} = $conf->{smp};
2063         delete $conf->{cores};
2064         delete $conf->{smp};
2065     }
2066
2067     my $used_volids = {};
2068
2069     my $cleanup_config = sub {
2070         my ($cref, $pending, $snapname) = @_;
2071
2072         foreach my $key (keys %$cref) {
2073             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2074                 $key eq 'snapstate' || $key eq 'pending';
2075             my $value = $cref->{$key};
2076             if ($key eq 'delete') {
2077                 die "propertry 'delete' is only allowed in [PENDING]\n"
2078                     if !$pending;
2079                 # fixme: check syntax?
2080                 next;
2081             }
2082             eval { $value = check_type($key, $value); };
2083             die "unable to parse value of '$key' - $@" if $@;
2084
2085             $cref->{$key} = $value;
2086
2087             if (!$snapname && valid_drivename($key)) {
2088                 my $drive = parse_drive($key, $value);
2089                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2090             }
2091         }
2092     };
2093
2094     &$cleanup_config($conf);
2095
2096     &$cleanup_config($conf->{pending}, 1);
2097
2098     foreach my $snapname (keys %{$conf->{snapshots}}) {
2099         die "internal error" if $snapname eq 'pending';
2100         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2101     }
2102
2103     # remove 'unusedX' settings if we re-add a volume
2104     foreach my $key (keys %$conf) {
2105         my $value = $conf->{$key};
2106         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2107             delete $conf->{$key};
2108         }
2109     }
2110
2111     my $generate_raw_config = sub {
2112         my ($conf) = @_;
2113
2114         my $raw = '';
2115
2116         # add description as comment to top of file
2117         my $descr = $conf->{description} || '';
2118         foreach my $cl (split(/\n/, $descr)) {
2119             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2120         }
2121
2122         foreach my $key (sort keys %$conf) {
2123             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2124             $raw .= "$key: $conf->{$key}\n";
2125         }
2126         return $raw;
2127     };
2128
2129     my $raw = &$generate_raw_config($conf);
2130
2131     if (scalar(keys %{$conf->{pending}})){
2132         $raw .= "\n[PENDING]\n";
2133         $raw .= &$generate_raw_config($conf->{pending});
2134     }
2135
2136     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2137         $raw .= "\n[$snapname]\n";
2138         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2139     }
2140
2141     return $raw;
2142 }
2143
2144 sub update_config_nolock {
2145     my ($vmid, $conf, $skiplock) = @_;
2146
2147     check_lock($conf) if !$skiplock;
2148
2149     my $cfspath = cfs_config_path($vmid);
2150
2151     PVE::Cluster::cfs_write_file($cfspath, $conf);
2152 }
2153
2154 sub update_config {
2155     my ($vmid, $conf, $skiplock) = @_;
2156
2157     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2158 }
2159
2160 sub load_defaults {
2161
2162     my $res = {};
2163
2164     # we use static defaults from our JSON schema configuration
2165     foreach my $key (keys %$confdesc) {
2166         if (defined(my $default = $confdesc->{$key}->{default})) {
2167             $res->{$key} = $default;
2168         }
2169     }
2170
2171     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2172     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2173
2174     return $res;
2175 }
2176
2177 sub config_list {
2178     my $vmlist = PVE::Cluster::get_vmlist();
2179     my $res = {};
2180     return $res if !$vmlist || !$vmlist->{ids};
2181     my $ids = $vmlist->{ids};
2182
2183     foreach my $vmid (keys %$ids) {
2184         my $d = $ids->{$vmid};
2185         next if !$d->{node} || $d->{node} ne $nodename;
2186         next if !$d->{type} || $d->{type} ne 'qemu';
2187         $res->{$vmid}->{exists} = 1;
2188     }
2189     return $res;
2190 }
2191
2192 # test if VM uses local resources (to prevent migration)
2193 sub check_local_resources {
2194     my ($conf, $noerr) = @_;
2195
2196     my $loc_res = 0;
2197
2198     $loc_res = 1 if $conf->{hostusb}; # old syntax
2199     $loc_res = 1 if $conf->{hostpci}; # old syntax
2200
2201     foreach my $k (keys %$conf) {
2202         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2203         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2204     }
2205
2206     die "VM uses local resources\n" if $loc_res && !$noerr;
2207
2208     return $loc_res;
2209 }
2210
2211 # check if used storages are available on all nodes (use by migrate)
2212 sub check_storage_availability {
2213     my ($storecfg, $conf, $node) = @_;
2214
2215     foreach_drive($conf, sub {
2216         my ($ds, $drive) = @_;
2217
2218         my $volid = $drive->{file};
2219         return if !$volid;
2220
2221         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2222         return if !$sid;
2223
2224         # check if storage is available on both nodes
2225         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2226         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2227    });
2228 }
2229
2230 # list nodes where all VM images are available (used by has_feature API)
2231 sub shared_nodes {
2232     my ($conf, $storecfg) = @_;
2233
2234     my $nodelist = PVE::Cluster::get_nodelist();
2235     my $nodehash = { map { $_ => 1 } @$nodelist };
2236     my $nodename = PVE::INotify::nodename();
2237
2238     foreach_drive($conf, sub {
2239         my ($ds, $drive) = @_;
2240
2241         my $volid = $drive->{file};
2242         return if !$volid;
2243
2244         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2245         if ($storeid) {
2246             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2247             if ($scfg->{disable}) {
2248                 $nodehash = {};
2249             } elsif (my $avail = $scfg->{nodes}) {
2250                 foreach my $node (keys %$nodehash) {
2251                     delete $nodehash->{$node} if !$avail->{$node};
2252                 }
2253             } elsif (!$scfg->{shared}) {
2254                 foreach my $node (keys %$nodehash) {
2255                     delete $nodehash->{$node} if $node ne $nodename
2256                 }
2257             }
2258         }
2259     });
2260
2261     return $nodehash
2262 }
2263
2264 sub check_lock {
2265     my ($conf) = @_;
2266
2267     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2268 }
2269
2270 sub check_cmdline {
2271     my ($pidfile, $pid) = @_;
2272
2273     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2274     if (defined($fh)) {
2275         my $line = <$fh>;
2276         $fh->close;
2277         return undef if !$line;
2278         my @param = split(/\0/, $line);
2279
2280         my $cmd = $param[0];
2281         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2282
2283         for (my $i = 0; $i < scalar (@param); $i++) {
2284             my $p = $param[$i];
2285             next if !$p;
2286             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2287                 my $p = $param[$i+1];
2288                 return 1 if $p && ($p eq $pidfile);
2289                 return undef;
2290             }
2291         }
2292     }
2293     return undef;
2294 }
2295
2296 sub check_running {
2297     my ($vmid, $nocheck, $node) = @_;
2298
2299     my $filename = config_file($vmid, $node);
2300
2301     die "unable to find configuration file for VM $vmid - no such machine\n"
2302         if !$nocheck && ! -f $filename;
2303
2304     my $pidfile = pidfile_name($vmid);
2305
2306     if (my $fd = IO::File->new("<$pidfile")) {
2307         my $st = stat($fd);
2308         my $line = <$fd>;
2309         close($fd);
2310
2311         my $mtime = $st->mtime;
2312         if ($mtime > time()) {
2313             warn "file '$filename' modified in future\n";
2314         }
2315
2316         if ($line =~ m/^(\d+)$/) {
2317             my $pid = $1;
2318             if (check_cmdline($pidfile, $pid)) {
2319                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2320                     return $pid;
2321                 }
2322             }
2323         }
2324     }
2325
2326     return undef;
2327 }
2328
2329 sub vzlist {
2330
2331     my $vzlist = config_list();
2332
2333     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2334
2335     while (defined(my $de = $fd->read)) {
2336         next if $de !~ m/^(\d+)\.pid$/;
2337         my $vmid = $1;
2338         next if !defined($vzlist->{$vmid});
2339         if (my $pid = check_running($vmid)) {
2340             $vzlist->{$vmid}->{pid} = $pid;
2341         }
2342     }
2343
2344     return $vzlist;
2345 }
2346
2347 sub disksize {
2348     my ($storecfg, $conf) = @_;
2349
2350     my $bootdisk = $conf->{bootdisk};
2351     return undef if !$bootdisk;
2352     return undef if !valid_drivename($bootdisk);
2353
2354     return undef if !$conf->{$bootdisk};
2355
2356     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2357     return undef if !defined($drive);
2358
2359     return undef if drive_is_cdrom($drive);
2360
2361     my $volid = $drive->{file};
2362     return undef if !$volid;
2363
2364     return $drive->{size};
2365 }
2366
2367 my $last_proc_pid_stat;
2368
2369 # get VM status information
2370 # This must be fast and should not block ($full == false)
2371 # We only query KVM using QMP if $full == true (this can be slow)
2372 sub vmstatus {
2373     my ($opt_vmid, $full) = @_;
2374
2375     my $res = {};
2376
2377     my $storecfg = PVE::Storage::config();
2378
2379     my $list = vzlist();
2380     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2381
2382     my $cpucount = $cpuinfo->{cpus} || 1;
2383
2384     foreach my $vmid (keys %$list) {
2385         next if $opt_vmid && ($vmid ne $opt_vmid);
2386
2387         my $cfspath = cfs_config_path($vmid);
2388         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2389
2390         my $d = {};
2391         $d->{pid} = $list->{$vmid}->{pid};
2392
2393         # fixme: better status?
2394         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2395
2396         my $size = disksize($storecfg, $conf);
2397         if (defined($size)) {
2398             $d->{disk} = 0; # no info available
2399             $d->{maxdisk} = $size;
2400         } else {
2401             $d->{disk} = 0;
2402             $d->{maxdisk} = 0;
2403         }
2404
2405         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2406         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2407         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2408
2409         $d->{name} = $conf->{name} || "VM $vmid";
2410         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2411
2412         if ($conf->{balloon}) {
2413             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2414             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2415         }
2416
2417         $d->{uptime} = 0;
2418         $d->{cpu} = 0;
2419         $d->{mem} = 0;
2420
2421         $d->{netout} = 0;
2422         $d->{netin} = 0;
2423
2424         $d->{diskread} = 0;
2425         $d->{diskwrite} = 0;
2426
2427         $d->{template} = is_template($conf);
2428
2429         $res->{$vmid} = $d;
2430     }
2431
2432     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2433     foreach my $dev (keys %$netdev) {
2434         next if $dev !~ m/^tap([1-9]\d*)i/;
2435         my $vmid = $1;
2436         my $d = $res->{$vmid};
2437         next if !$d;
2438
2439         $d->{netout} += $netdev->{$dev}->{receive};
2440         $d->{netin} += $netdev->{$dev}->{transmit};
2441     }
2442
2443     my $ctime = gettimeofday;
2444
2445     foreach my $vmid (keys %$list) {
2446
2447         my $d = $res->{$vmid};
2448         my $pid = $d->{pid};
2449         next if !$pid;
2450
2451         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2452         next if !$pstat; # not running
2453
2454         my $used = $pstat->{utime} + $pstat->{stime};
2455
2456         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2457
2458         if ($pstat->{vsize}) {
2459             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2460         }
2461
2462         my $old = $last_proc_pid_stat->{$pid};
2463         if (!$old) {
2464             $last_proc_pid_stat->{$pid} = {
2465                 time => $ctime,
2466                 used => $used,
2467                 cpu => 0,
2468             };
2469             next;
2470         }
2471
2472         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2473
2474         if ($dtime > 1000) {
2475             my $dutime = $used -  $old->{used};
2476
2477             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2478             $last_proc_pid_stat->{$pid} = {
2479                 time => $ctime,
2480                 used => $used,
2481                 cpu => $d->{cpu},
2482             };
2483         } else {
2484             $d->{cpu} = $old->{cpu};
2485         }
2486     }
2487
2488     return $res if !$full;
2489
2490     my $qmpclient = PVE::QMPClient->new();
2491
2492     my $ballooncb = sub {
2493         my ($vmid, $resp) = @_;
2494
2495         my $info = $resp->{'return'};
2496         return if !$info->{max_mem};
2497
2498         my $d = $res->{$vmid};
2499
2500         # use memory assigned to VM
2501         $d->{maxmem} = $info->{max_mem};
2502         $d->{balloon} = $info->{actual};
2503
2504         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2505             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2506             $d->{freemem} = $info->{free_mem};
2507         }
2508
2509     };
2510
2511     my $blockstatscb = sub {
2512         my ($vmid, $resp) = @_;
2513         my $data = $resp->{'return'} || [];
2514         my $totalrdbytes = 0;
2515         my $totalwrbytes = 0;
2516         for my $blockstat (@$data) {
2517             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2518             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2519         }
2520         $res->{$vmid}->{diskread} = $totalrdbytes;
2521         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2522     };
2523
2524     my $statuscb = sub {
2525         my ($vmid, $resp) = @_;
2526
2527         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2528         # this fails if ballon driver is not loaded, so this must be
2529         # the last commnand (following command are aborted if this fails).
2530         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2531
2532         my $status = 'unknown';
2533         if (!defined($status = $resp->{'return'}->{status})) {
2534             warn "unable to get VM status\n";
2535             return;
2536         }
2537
2538         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2539     };
2540
2541     foreach my $vmid (keys %$list) {
2542         next if $opt_vmid && ($vmid ne $opt_vmid);
2543         next if !$res->{$vmid}->{pid}; # not running
2544         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2545     }
2546
2547     $qmpclient->queue_execute(undef, 1);
2548
2549     foreach my $vmid (keys %$list) {
2550         next if $opt_vmid && ($vmid ne $opt_vmid);
2551         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2552     }
2553
2554     return $res;
2555 }
2556
2557 sub foreach_dimm {
2558     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2559
2560     my $dimm_id = 0;
2561     my $current_size = 1024;
2562     my $dimm_size = 512;
2563     return if $current_size == $memory;
2564
2565     for (my $j = 0; $j < 8; $j++) {
2566         for (my $i = 0; $i < 32; $i++) {
2567             my $name = "dimm${dimm_id}";
2568             $dimm_id++;
2569             my $numanode = $i % $sockets;
2570             $current_size += $dimm_size;
2571             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2572             return  $current_size if $current_size >= $memory;
2573         }
2574         $dimm_size *= 2;
2575     }
2576 }
2577
2578 sub foreach_drive {
2579     my ($conf, $func) = @_;
2580
2581     foreach my $ds (keys %$conf) {
2582         next if !valid_drivename($ds);
2583
2584         my $drive = parse_drive($ds, $conf->{$ds});
2585         next if !$drive;
2586
2587         &$func($ds, $drive);
2588     }
2589 }
2590
2591 sub foreach_volid {
2592     my ($conf, $func) = @_;
2593
2594     my $volhash = {};
2595
2596     my $test_volid = sub {
2597         my ($volid, $is_cdrom) = @_;
2598
2599         return if !$volid;
2600
2601         $volhash->{$volid} = $is_cdrom || 0;
2602     };
2603
2604     foreach_drive($conf, sub {
2605         my ($ds, $drive) = @_;
2606         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2607     });
2608
2609     foreach my $snapname (keys %{$conf->{snapshots}}) {
2610         my $snap = $conf->{snapshots}->{$snapname};
2611         &$test_volid($snap->{vmstate}, 0);
2612         foreach_drive($snap, sub {
2613             my ($ds, $drive) = @_;
2614             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2615         });
2616     }
2617
2618     foreach my $volid (keys %$volhash) {
2619         &$func($volid, $volhash->{$volid});
2620     }
2621 }
2622
2623 sub vga_conf_has_spice {
2624     my ($vga) = @_;
2625
2626     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2627
2628     return $1 || 1;
2629 }
2630
2631 sub config_to_command {
2632     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2633
2634     my $cmd = [];
2635     my $globalFlags = [];
2636     my $machineFlags = [];
2637     my $rtcFlags = [];
2638     my $cpuFlags = [];
2639     my $devices = [];
2640     my $pciaddr = '';
2641     my $bridges = {};
2642     my $kvmver = kvm_user_version();
2643     my $vernum = 0; # unknown
2644     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2645         $vernum = $1*1000000+$2*1000;
2646     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2647         $vernum = $1*1000000+$2*1000+$3;
2648     }
2649
2650     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2651
2652     my $have_ovz = -f '/proc/vz/vestat';
2653
2654     my $q35 = machine_type_is_q35($conf);
2655     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2656     my $machine_type = $forcemachine || $conf->{machine};
2657
2658     push @$cmd, '/usr/bin/kvm';
2659
2660     push @$cmd, '-id', $vmid;
2661
2662     my $use_virtio = 0;
2663
2664     my $qmpsocket = qmp_socket($vmid);
2665     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2666     push @$cmd, '-mon', "chardev=qmp,mode=control";
2667
2668     my $socket = vnc_socket($vmid);
2669     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2670
2671     push @$cmd, '-pidfile' , pidfile_name($vmid);
2672
2673     push @$cmd, '-daemonize';
2674
2675     if ($conf->{smbios1}) {
2676         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2677     }
2678
2679     if ($q35) {
2680         # the q35 chipset support native usb2, so we enable usb controller
2681         # by default for this machine type
2682         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2683     } else {
2684         $pciaddr = print_pci_addr("piix3", $bridges);
2685         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2686
2687         my $use_usb2 = 0;
2688         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2689             next if !$conf->{"usb$i"};
2690             $use_usb2 = 1;
2691         }
2692         # include usb device config
2693         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2694     }
2695
2696     my $vga = $conf->{vga};
2697
2698     my $qxlnum = vga_conf_has_spice($vga);
2699     $vga = 'qxl' if $qxlnum;
2700
2701     if (!$vga) {
2702         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2703                                 $conf->{ostype} eq 'win7' ||
2704                                 $conf->{ostype} eq 'w2k8')) {
2705             $vga = 'std';
2706         } else {
2707             $vga = 'cirrus';
2708         }
2709     }
2710
2711     # enable absolute mouse coordinates (needed by vnc)
2712     my $tablet;
2713     if (defined($conf->{tablet})) {
2714         $tablet = $conf->{tablet};
2715     } else {
2716         $tablet = $defaults->{tablet};
2717         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2718         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2719     }
2720
2721     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2722
2723     # host pci devices
2724     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2725         my $d = parse_hostpci($conf->{"hostpci$i"});
2726         next if !$d;
2727
2728         my $pcie = $d->{pcie};
2729         if($pcie){
2730             die "q35 machine model is not enabled" if !$q35;
2731             $pciaddr = print_pcie_addr("hostpci$i");
2732         }else{
2733             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2734         }
2735
2736         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2737         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2738         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2739         if ($xvga && $xvga ne '') {
2740             push @$cpuFlags, 'kvm=off';
2741             $vga = 'none';
2742         }
2743         $driver = "vfio-pci" if $xvga ne '';
2744         my $pcidevices = $d->{pciid};
2745         my $multifunction = 1 if @$pcidevices > 1;
2746
2747         my $j=0;
2748         foreach my $pcidevice (@$pcidevices) {
2749
2750             my $id = "hostpci$i";
2751             $id .= ".$j" if $multifunction;
2752             my $addr = $pciaddr;
2753             $addr .= ".$j" if $multifunction;
2754             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2755
2756             if($j == 0){
2757                 $devicestr .= "$rombar$xvga";
2758                 $devicestr .= ",multifunction=on" if $multifunction;
2759             }
2760
2761             push @$devices, '-device', $devicestr;
2762             $j++;
2763         }
2764     }
2765
2766     # usb devices
2767     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2768         my $d = parse_usb_device($conf->{"usb$i"});
2769         next if !$d;
2770         if ($d->{vendorid} && $d->{productid}) {
2771             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2772         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2773             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2774         } elsif ($d->{spice}) {
2775             # usb redir support for spice
2776             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2777             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2778         }
2779     }
2780
2781     # serial devices
2782     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2783         if (my $path = $conf->{"serial$i"}) {
2784             if ($path eq 'socket') {
2785                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2786                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2787                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2788             } else {
2789                 die "no such serial device\n" if ! -c $path;
2790                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2791                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2792             }
2793         }
2794     }
2795
2796     # parallel devices
2797     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2798         if (my $path = $conf->{"parallel$i"}) {
2799             die "no such parallel device\n" if ! -c $path;
2800             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2801             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2802             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2803         }
2804     }
2805
2806     my $vmname = $conf->{name} || "vm$vmid";
2807
2808     push @$cmd, '-name', $vmname;
2809
2810     my $sockets = 1;
2811     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2812     $sockets = $conf->{sockets} if  $conf->{sockets};
2813
2814     my $cores = $conf->{cores} || 1;
2815
2816     my $maxcpus = $sockets * $cores;
2817
2818     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2819
2820     my $allowed_vcpus = $cpuinfo->{cpus};
2821
2822     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2823         if ($allowed_vcpus < $maxcpus);
2824
2825     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2826
2827     push @$cmd, '-nodefaults';
2828
2829     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2830
2831     my $bootindex_hash = {};
2832     my $i = 1;
2833     foreach my $o (split(//, $bootorder)) {
2834         $bootindex_hash->{$o} = $i*100;
2835         $i++;
2836     }
2837
2838     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2839
2840     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2841
2842     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2843
2844     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2845
2846     # time drift fix
2847     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2848
2849     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2850     my $useLocaltime = $conf->{localtime};
2851
2852     if (my $ost = $conf->{ostype}) {
2853         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2854
2855         if ($ost =~ m/^w/) { # windows
2856             $useLocaltime = 1 if !defined($conf->{localtime});
2857
2858             # use time drift fix when acpi is enabled
2859             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2860                 $tdf = 1 if !defined($conf->{tdf});
2861             }
2862         }
2863
2864         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2865             $ost eq 'wvista') {
2866             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2867             push @$cmd, '-no-hpet';
2868             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2869                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2870                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2871                 push @$cpuFlags , 'hv_time' if !$nokvm;
2872
2873             } else {
2874                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2875             }
2876         }
2877
2878         if ($ost eq 'win7' || $ost eq 'win8') {
2879             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2880         }
2881     }
2882
2883     push @$rtcFlags, 'driftfix=slew' if $tdf;
2884
2885     if ($nokvm) {
2886         push @$machineFlags, 'accel=tcg';
2887     } else {
2888         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2889     }
2890
2891     if ($machine_type) {
2892         push @$machineFlags, "type=${machine_type}";
2893     }
2894
2895     if ($conf->{startdate}) {
2896         push @$rtcFlags, "base=$conf->{startdate}";
2897     } elsif ($useLocaltime) {
2898         push @$rtcFlags, 'base=localtime';
2899     }
2900
2901     my $cpu = $nokvm ? "qemu64" : "kvm64";
2902     $cpu = $conf->{cpu} if $conf->{cpu};
2903
2904     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2905
2906     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2907
2908     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2909
2910     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2911
2912     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2913
2914         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2915         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2916     }
2917
2918     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2919
2920     push @$cmd, '-cpu', "$cpu,enforce";
2921
2922     my $memory = $conf->{memory} || $defaults->{memory};
2923     my $static_memory = 0;
2924     my $dimm_memory = 0;
2925
2926     if ($hotplug_features->{memory}) {
2927         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2928         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2929         $static_memory = $STATICMEM;
2930         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2931         $dimm_memory = $memory - $static_memory;
2932         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2933
2934     } else {
2935
2936         $static_memory = $memory;
2937         push @$cmd, '-m', $static_memory;
2938     }
2939
2940     if ($conf->{numa}) {
2941
2942         my $numa_totalmemory = undef;
2943         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2944             next if !$conf->{"numa$i"};
2945             my $numa = parse_numa($conf->{"numa$i"});
2946             next if !$numa;
2947             # memory
2948             die "missing numa node$i memory value\n" if !$numa->{memory};
2949             my $numa_memory = $numa->{memory};
2950             $numa_totalmemory += $numa_memory;
2951             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2952
2953             # cpus
2954             my $cpus_start = $numa->{cpus}->{start};
2955             die "missing numa node$i cpus\n" if !defined($cpus_start);
2956             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2957             my $cpus = $cpus_start;
2958             if (defined($cpus_end)) {
2959                 $cpus .= "-$cpus_end";
2960                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2961             }
2962
2963             # hostnodes
2964             my $hostnodes_start = $numa->{hostnodes}->{start};
2965             if (defined($hostnodes_start)) {
2966                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2967                 my $hostnodes = $hostnodes_start;
2968                 if (defined($hostnodes_end)) {
2969                     $hostnodes .= "-$hostnodes_end";
2970                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2971                 }
2972
2973                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2974                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2975                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2976                 }
2977
2978                 # policy
2979                 my $policy = $numa->{policy};
2980                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2981                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2982             }
2983
2984             push @$cmd, '-object', $numa_object;
2985             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2986         }
2987
2988         die "total memory for NUMA nodes must be equal to vm static memory\n"
2989             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2990
2991         #if no custom tology, we split memory and cores across numa nodes
2992         if(!$numa_totalmemory) {
2993
2994             my $numa_memory = ($static_memory / $sockets) . "M";
2995
2996             for (my $i = 0; $i < $sockets; $i++)  {
2997
2998                 my $cpustart = ($cores * $i);
2999                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3000                 my $cpus = $cpustart;
3001                 $cpus .= "-$cpuend" if $cpuend;
3002
3003                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3004                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3005             }
3006         }
3007     }
3008
3009     if ($hotplug_features->{memory}) {
3010         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3011             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3012             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3013             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3014
3015             #if dimm_memory is not aligned to dimm map
3016             if($current_size > $memory) {
3017                  $conf->{memory} = $current_size;
3018                  update_config_nolock($vmid, $conf, 1);
3019             }
3020         });
3021     }
3022
3023     push @$cmd, '-S' if $conf->{freeze};
3024
3025     # set keyboard layout
3026     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3027     push @$cmd, '-k', $kb if $kb;
3028
3029     # enable sound
3030     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3031     #push @$cmd, '-soundhw', 'es1370';
3032     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3033
3034     if($conf->{agent}) {
3035         my $qgasocket = qmp_socket($vmid, 1);
3036         my $pciaddr = print_pci_addr("qga0", $bridges);
3037         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3038         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3039         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3040     }
3041
3042     my $spice_port;
3043
3044     if ($qxlnum) {
3045         if ($qxlnum > 1) {
3046             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3047                 for(my $i = 1; $i < $qxlnum; $i++){
3048                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3049                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3050                 }
3051             } else {
3052                 # assume other OS works like Linux
3053                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3054                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3055             }
3056         }
3057
3058         my $pciaddr = print_pci_addr("spice", $bridges);
3059
3060         $spice_port = PVE::Tools::next_spice_port();
3061
3062         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3063
3064         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3065         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3066         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3067     }
3068
3069     # enable balloon by default, unless explicitly disabled
3070     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3071         $pciaddr = print_pci_addr("balloon0", $bridges);
3072         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3073     }
3074
3075     if ($conf->{watchdog}) {
3076         my $wdopts = parse_watchdog($conf->{watchdog});
3077         $pciaddr = print_pci_addr("watchdog", $bridges);
3078         my $watchdog = $wdopts->{model} || 'i6300esb';
3079         push @$devices, '-device', "$watchdog$pciaddr";
3080         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3081     }
3082
3083     my $vollist = [];
3084     my $scsicontroller = {};
3085     my $ahcicontroller = {};
3086     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3087
3088     # Add iscsi initiator name if available
3089     if (my $initiator = get_initiator_name()) {
3090         push @$devices, '-iscsi', "initiator-name=$initiator";
3091     }
3092
3093     foreach_drive($conf, sub {
3094         my ($ds, $drive) = @_;
3095
3096         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3097             push @$vollist, $drive->{file};
3098         }
3099
3100         $use_virtio = 1 if $ds =~ m/^virtio/;
3101
3102         if (drive_is_cdrom ($drive)) {
3103             if ($bootindex_hash->{d}) {
3104                 $drive->{bootindex} = $bootindex_hash->{d};
3105                 $bootindex_hash->{d} += 1;
3106             }
3107         } else {
3108             if ($bootindex_hash->{c}) {
3109                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3110                 $bootindex_hash->{c} += 1;
3111             }
3112         }
3113
3114         if($drive->{interface} eq 'virtio'){
3115            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3116         }
3117
3118         if ($drive->{interface} eq 'scsi') {
3119
3120             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3121
3122             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3123             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3124
3125             my $iothread = '';
3126             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3127                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3128                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3129             }
3130
3131             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread" if !$scsicontroller->{$controller};
3132             $scsicontroller->{$controller}=1;
3133         }
3134
3135         if ($drive->{interface} eq 'sata') {
3136            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3137            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3138            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3139            $ahcicontroller->{$controller}=1;
3140         }
3141
3142         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3143         push @$devices, '-drive',$drive_cmd;
3144         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3145     });
3146
3147     for (my $i = 0; $i < $MAX_NETS; $i++) {
3148          next if !$conf->{"net$i"};
3149          my $d = parse_net($conf->{"net$i"});
3150          next if !$d;
3151
3152          $use_virtio = 1 if $d->{model} eq 'virtio';
3153
3154          if ($bootindex_hash->{n}) {
3155             $d->{bootindex} = $bootindex_hash->{n};
3156             $bootindex_hash->{n} += 1;
3157          }
3158
3159          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3160          push @$devices, '-netdev', $netdevfull;
3161
3162          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3163          push @$devices, '-device', $netdevicefull;
3164     }
3165
3166     if (!$q35) {
3167         # add pci bridges
3168         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3169            $bridges->{1} = 1;
3170            $bridges->{2} = 1;
3171         }
3172
3173         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3174
3175         while (my ($k, $v) = each %$bridges) {
3176             $pciaddr = print_pci_addr("pci.$k");
3177             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3178         }
3179     }
3180
3181     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3182     # when the VM uses virtio devices.
3183     if (!$use_virtio && $have_ovz) {
3184
3185         my $cpuunits = defined($conf->{cpuunits}) ?
3186             $conf->{cpuunits} : $defaults->{cpuunits};
3187
3188         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3189
3190         # fixme: cpulimit is currently ignored
3191         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3192     }
3193
3194     # add custom args
3195     if ($conf->{args}) {
3196         my $aa = PVE::Tools::split_args($conf->{args});
3197         push @$cmd, @$aa;
3198     }
3199
3200     push @$cmd, @$devices;
3201     push @$cmd, '-rtc', join(',', @$rtcFlags)
3202         if scalar(@$rtcFlags);
3203     push @$cmd, '-machine', join(',', @$machineFlags)
3204         if scalar(@$machineFlags);
3205     push @$cmd, '-global', join(',', @$globalFlags)
3206         if scalar(@$globalFlags);
3207
3208     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3209 }
3210
3211 sub vnc_socket {
3212     my ($vmid) = @_;
3213     return "${var_run_tmpdir}/$vmid.vnc";
3214 }
3215
3216 sub spice_port {
3217     my ($vmid) = @_;
3218
3219     my $res = vm_mon_cmd($vmid, 'query-spice');
3220
3221     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3222 }
3223
3224 sub qmp_socket {
3225     my ($vmid, $qga) = @_;
3226     my $sockettype = $qga ? 'qga' : 'qmp';
3227     return "${var_run_tmpdir}/$vmid.$sockettype";
3228 }
3229
3230 sub pidfile_name {
3231     my ($vmid) = @_;
3232     return "${var_run_tmpdir}/$vmid.pid";
3233 }
3234
3235 sub vm_devices_list {
3236     my ($vmid) = @_;
3237
3238     my $res = vm_mon_cmd($vmid, 'query-pci');
3239     my $devices = {};
3240     foreach my $pcibus (@$res) {
3241         foreach my $device (@{$pcibus->{devices}}) {
3242             next if !$device->{'qdev_id'};
3243             if ($device->{'pci_bridge'}) {
3244                 $devices->{$device->{'qdev_id'}} = 1;
3245                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3246                     next if !$bridge_device->{'qdev_id'};
3247                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3248                     $devices->{$device->{'qdev_id'}}++;
3249                 }
3250             } else {
3251                 $devices->{$device->{'qdev_id'}} = 1;
3252             }
3253         }
3254     }
3255
3256     my $resblock = vm_mon_cmd($vmid, 'query-block');
3257     foreach my $block (@$resblock) {
3258         if($block->{device} =~ m/^drive-(\S+)/){
3259                 $devices->{$1} = 1;
3260         }
3261     }
3262
3263     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3264     foreach my $mice (@$resmice) {
3265         if ($mice->{name} eq 'QEMU HID Tablet') {
3266             $devices->{tablet} = 1;
3267             last;
3268         }
3269     }
3270
3271     return $devices;
3272 }
3273
3274 sub vm_deviceplug {
3275     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3276
3277     my $q35 = machine_type_is_q35($conf);
3278
3279     my $devices_list = vm_devices_list($vmid);
3280     return 1 if defined($devices_list->{$deviceid});
3281
3282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3283
3284     if ($deviceid eq 'tablet') {
3285
3286         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3287
3288     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3289
3290         qemu_iothread_add($vmid, $deviceid, $device);
3291
3292         qemu_driveadd($storecfg, $vmid, $device);
3293         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3294
3295         qemu_deviceadd($vmid, $devicefull);
3296         eval { qemu_deviceaddverify($vmid, $deviceid); };
3297         if (my $err = $@) {
3298             eval { qemu_drivedel($vmid, $deviceid); };
3299             warn $@ if $@;
3300             die $err;
3301         }
3302
3303     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3304
3305
3306         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3307         my $pciaddr = print_pci_addr($deviceid);
3308         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3309
3310         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3311
3312         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3313             qemu_iothread_add($vmid, $deviceid, $device);
3314             $devicefull .= ",iothread=iothread-$deviceid";
3315         }
3316
3317         qemu_deviceadd($vmid, $devicefull);
3318         qemu_deviceaddverify($vmid, $deviceid);
3319
3320     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3321
3322         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3323         qemu_driveadd($storecfg, $vmid, $device);
3324
3325         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3326         eval { qemu_deviceadd($vmid, $devicefull); };
3327         if (my $err = $@) {
3328             eval { qemu_drivedel($vmid, $deviceid); };
3329             warn $@ if $@;
3330             die $err;
3331         }
3332
3333     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3334
3335         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3336         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3337         qemu_deviceadd($vmid, $netdevicefull);
3338         eval { qemu_deviceaddverify($vmid, $deviceid); };
3339         if (my $err = $@) {
3340             eval { qemu_netdevdel($vmid, $deviceid); };
3341             warn $@ if $@;
3342             die $err;
3343         }
3344
3345     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3346
3347         my $bridgeid = $2;
3348         my $pciaddr = print_pci_addr($deviceid);
3349         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3350
3351         qemu_deviceadd($vmid, $devicefull);
3352         qemu_deviceaddverify($vmid, $deviceid);
3353
3354     } else {
3355         die "can't hotplug device '$deviceid'\n";
3356     }
3357
3358     return 1;
3359 }
3360
3361 # fixme: this should raise exceptions on error!
3362 sub vm_deviceunplug {
3363     my ($vmid, $conf, $deviceid) = @_;
3364
3365     my $devices_list = vm_devices_list($vmid);
3366     return 1 if !defined($devices_list->{$deviceid});
3367
3368     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3369
3370     if ($deviceid eq 'tablet') {
3371
3372         qemu_devicedel($vmid, $deviceid);
3373
3374     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3375
3376         qemu_devicedel($vmid, $deviceid);
3377         qemu_devicedelverify($vmid, $deviceid);
3378         qemu_drivedel($vmid, $deviceid);
3379         qemu_iothread_del($conf, $vmid, $deviceid);
3380
3381     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3382
3383         qemu_devicedel($vmid, $deviceid);
3384         qemu_devicedelverify($vmid, $deviceid);
3385         qemu_iothread_del($conf, $vmid, $deviceid);
3386
3387     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3388
3389         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3390         my $device = parse_drive($deviceid, $conf->{$deviceid});
3391         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3392
3393         qemu_devicedel($vmid, $deviceid);
3394         qemu_drivedel($vmid, $deviceid);
3395         qemu_deletescsihw($conf, $vmid, $deviceid);
3396
3397     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3398
3399         qemu_devicedel($vmid, $deviceid);
3400         qemu_devicedelverify($vmid, $deviceid);
3401         qemu_netdevdel($vmid, $deviceid);
3402
3403     } else {
3404         die "can't unplug device '$deviceid'\n";
3405     }
3406
3407     return 1;
3408 }
3409
3410 sub qemu_deviceadd {
3411     my ($vmid, $devicefull) = @_;
3412
3413     $devicefull = "driver=".$devicefull;
3414     my %options =  split(/[=,]/, $devicefull);
3415
3416     vm_mon_cmd($vmid, "device_add" , %options);
3417 }
3418
3419 sub qemu_devicedel {
3420     my ($vmid, $deviceid) = @_;
3421
3422     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3423 }
3424
3425 sub qemu_iothread_add {
3426     my($vmid, $deviceid, $device) = @_;
3427
3428     if ($device->{iothread}) {
3429         my $iothreads = vm_iothreads_list($vmid);
3430         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3431     }
3432 }
3433
3434 sub qemu_iothread_del {
3435     my($conf, $vmid, $deviceid) = @_;
3436
3437     my $device = parse_drive($deviceid, $conf->{$deviceid});
3438     if ($device->{iothread}) {
3439         my $iothreads = vm_iothreads_list($vmid);
3440         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3441     }
3442 }
3443
3444 sub qemu_objectadd {
3445     my($vmid, $objectid, $qomtype) = @_;
3446
3447     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3448
3449     return 1;
3450 }
3451
3452 sub qemu_objectdel {
3453     my($vmid, $objectid) = @_;
3454
3455     vm_mon_cmd($vmid, "object-del", id => $objectid);
3456
3457     return 1;
3458 }
3459
3460 sub qemu_driveadd {
3461     my ($storecfg, $vmid, $device) = @_;
3462
3463     my $drive = print_drive_full($storecfg, $vmid, $device);
3464     $drive =~ s/\\/\\\\/g;
3465     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3466
3467     # If the command succeeds qemu prints: "OK"
3468     return 1 if $ret =~ m/OK/s;
3469
3470     die "adding drive failed: $ret\n";
3471 }
3472
3473 sub qemu_drivedel {
3474     my($vmid, $deviceid) = @_;
3475
3476     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3477     $ret =~ s/^\s+//;
3478
3479     return 1 if $ret eq "";
3480
3481     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3482     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3483
3484     die "deleting drive $deviceid failed : $ret\n";
3485 }
3486
3487 sub qemu_deviceaddverify {
3488     my ($vmid, $deviceid) = @_;
3489
3490     for (my $i = 0; $i <= 5; $i++) {
3491          my $devices_list = vm_devices_list($vmid);
3492          return 1 if defined($devices_list->{$deviceid});
3493          sleep 1;
3494     }
3495
3496     die "error on hotplug device '$deviceid'\n";
3497 }
3498
3499
3500 sub qemu_devicedelverify {
3501     my ($vmid, $deviceid) = @_;
3502
3503     # need to verify that the device is correctly removed as device_del
3504     # is async and empty return is not reliable
3505
3506     for (my $i = 0; $i <= 5; $i++) {
3507          my $devices_list = vm_devices_list($vmid);
3508          return 1 if !defined($devices_list->{$deviceid});
3509          sleep 1;
3510     }
3511
3512     die "error on hot-unplugging device '$deviceid'\n";
3513 }
3514
3515 sub qemu_findorcreatescsihw {
3516     my ($storecfg, $conf, $vmid, $device) = @_;
3517
3518     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3519
3520     my $scsihwid="$controller_prefix$controller";
3521     my $devices_list = vm_devices_list($vmid);
3522
3523     if(!defined($devices_list->{$scsihwid})) {
3524         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3525     }
3526
3527     return 1;
3528 }
3529
3530 sub qemu_deletescsihw {
3531     my ($conf, $vmid, $opt) = @_;
3532
3533     my $device = parse_drive($opt, $conf->{$opt});
3534
3535     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3536         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3537         return 1;
3538     }
3539
3540     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3541
3542     my $devices_list = vm_devices_list($vmid);
3543     foreach my $opt (keys %{$devices_list}) {
3544         if (PVE::QemuServer::valid_drivename($opt)) {
3545             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3546             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3547                 return 1;
3548             }
3549         }
3550     }
3551
3552     my $scsihwid="scsihw$controller";
3553
3554     vm_deviceunplug($vmid, $conf, $scsihwid);
3555
3556     return 1;
3557 }
3558
3559 sub qemu_add_pci_bridge {
3560     my ($storecfg, $conf, $vmid, $device) = @_;
3561
3562     my $bridges = {};
3563
3564     my $bridgeid;
3565
3566     print_pci_addr($device, $bridges);
3567
3568     while (my ($k, $v) = each %$bridges) {
3569         $bridgeid = $k;
3570     }
3571     return 1 if !defined($bridgeid) || $bridgeid < 1;
3572
3573     my $bridge = "pci.$bridgeid";
3574     my $devices_list = vm_devices_list($vmid);
3575
3576     if (!defined($devices_list->{$bridge})) {
3577         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3578     }
3579
3580     return 1;
3581 }
3582
3583 sub qemu_set_link_status {
3584     my ($vmid, $device, $up) = @_;
3585
3586     vm_mon_cmd($vmid, "set_link", name => $device,
3587                up => $up ? JSON::true : JSON::false);
3588 }
3589
3590 sub qemu_netdevadd {
3591     my ($vmid, $conf, $device, $deviceid) = @_;
3592
3593     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3594     my %options =  split(/[=,]/, $netdev);
3595
3596     vm_mon_cmd($vmid, "netdev_add",  %options);
3597     return 1;
3598 }
3599
3600 sub qemu_netdevdel {
3601     my ($vmid, $deviceid) = @_;
3602
3603     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3604 }
3605
3606 sub qemu_cpu_hotplug {
3607     my ($vmid, $conf, $vcpus) = @_;
3608
3609     my $sockets = 1;
3610     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3611     $sockets = $conf->{sockets} if  $conf->{sockets};
3612     my $cores = $conf->{cores} || 1;
3613     my $maxcpus = $sockets * $cores;
3614
3615     $vcpus = $maxcpus if !$vcpus;
3616
3617     die "you can't add more vcpus than maxcpus\n"
3618         if $vcpus > $maxcpus;
3619
3620     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3621     die "online cpu unplug is not yet possible\n"
3622         if $vcpus < $currentvcpus;
3623
3624     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3625     die "vcpus in running vm is different than configuration\n"
3626         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3627
3628     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3629         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3630     }
3631 }
3632
3633 sub qemu_memory_hotplug {
3634     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3635
3636     return $value if !check_running($vmid);
3637
3638     my $memory = $conf->{memory} || $defaults->{memory};
3639     $value = $defaults->{memory} if !$value;
3640     return $value if $value == $memory;
3641
3642     my $static_memory = $STATICMEM;
3643     my $dimm_memory = $memory - $static_memory;
3644
3645     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3646     die "memory unplug is not yet available" if $value < $memory;
3647     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3648
3649
3650     my $sockets = 1;
3651     $sockets = $conf->{sockets} if $conf->{sockets};
3652
3653     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3654         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3655
3656             return if $current_size <= $conf->{memory};
3657
3658             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3659             if (my $err = $@) {
3660                 eval { qemu_objectdel($vmid, "mem-$name"); };
3661                 die $err;
3662             }
3663
3664             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3665             if (my $err = $@) {
3666                 eval { qemu_objectdel($vmid, "mem-$name"); };
3667                 die $err;
3668             }
3669             #update conf after each succesful module hotplug
3670             $conf->{memory} = $current_size;
3671             update_config_nolock($vmid, $conf, 1);
3672     });
3673 }
3674
3675 sub qemu_block_set_io_throttle {
3676     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3677
3678     return if !check_running($vmid) ;
3679
3680     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3681
3682 }
3683
3684 # old code, only used to shutdown old VM after update
3685 sub __read_avail {
3686     my ($fh, $timeout) = @_;
3687
3688     my $sel = new IO::Select;
3689     $sel->add($fh);
3690
3691     my $res = '';
3692     my $buf;
3693
3694     my @ready;
3695     while (scalar (@ready = $sel->can_read($timeout))) {
3696         my $count;
3697         if ($count = $fh->sysread($buf, 8192)) {
3698             if ($buf =~ /^(.*)\(qemu\) $/s) {
3699                 $res .= $1;
3700                 last;
3701             } else {
3702                 $res .= $buf;
3703             }
3704         } else {
3705             if (!defined($count)) {
3706                 die "$!\n";
3707             }
3708             last;
3709         }
3710     }
3711
3712     die "monitor read timeout\n" if !scalar(@ready);
3713
3714     return $res;
3715 }
3716
3717 # old code, only used to shutdown old VM after update
3718 sub vm_monitor_command {
3719     my ($vmid, $cmdstr, $nocheck) = @_;
3720
3721     my $res;
3722
3723     eval {
3724         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3725
3726         my $sname = "${var_run_tmpdir}/$vmid.mon";
3727
3728         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3729             die "unable to connect to VM $vmid socket - $!\n";
3730
3731         my $timeout = 3;
3732
3733         # hack: migrate sometime blocks the monitor (when migrate_downtime
3734         # is set)
3735         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3736             $timeout = 60*60; # 1 hour
3737         }
3738
3739         # read banner;
3740         my $data = __read_avail($sock, $timeout);
3741
3742         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3743             die "got unexpected qemu monitor banner\n";
3744         }
3745
3746         my $sel = new IO::Select;
3747         $sel->add($sock);
3748
3749         if (!scalar(my @ready = $sel->can_write($timeout))) {
3750             die "monitor write error - timeout";
3751         }
3752
3753         my $fullcmd = "$cmdstr\r";
3754
3755         # syslog('info', "VM $vmid monitor command: $cmdstr");
3756
3757         my $b;
3758         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3759             die "monitor write error - $!";
3760         }
3761
3762         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3763
3764         $timeout = 20;
3765
3766         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3767             $timeout = 60*60; # 1 hour
3768         } elsif ($cmdstr =~ m/^(eject|change)/) {
3769             $timeout = 60; # note: cdrom mount command is slow
3770         }
3771         if ($res = __read_avail($sock, $timeout)) {
3772
3773             my @lines = split("\r?\n", $res);
3774
3775             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3776
3777             $res = join("\n", @lines);
3778             $res .= "\n";
3779         }
3780     };
3781
3782     my $err = $@;
3783
3784     if ($err) {
3785         syslog("err", "VM $vmid monitor command failed - $err");
3786         die $err;
3787     }
3788
3789     return $res;
3790 }
3791
3792 sub qemu_block_resize {
3793     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3794
3795     my $running = check_running($vmid);
3796
3797     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3798
3799     return if !$running;
3800
3801     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3802
3803 }
3804
3805 sub qemu_volume_snapshot {
3806     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3807
3808     my $running = check_running($vmid);
3809
3810     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3811
3812     return if !$running;
3813
3814     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3815
3816 }
3817
3818 sub qemu_volume_snapshot_delete {
3819     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3820
3821     my $running = check_running($vmid);
3822
3823     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3824
3825     return if !$running;
3826
3827     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3828 }
3829
3830 sub set_migration_caps {
3831     my ($vmid) = @_;
3832
3833     my $cap_ref = [];
3834
3835     my $enabled_cap = {
3836         "auto-converge" => 1,
3837         "xbzrle" => 0,
3838         "x-rdma-pin-all" => 0,
3839         "zero-blocks" => 0,
3840     };
3841
3842     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3843
3844     for my $supported_capability (@$supported_capabilities) {
3845         push @$cap_ref, {
3846             capability => $supported_capability->{capability},
3847             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3848         };
3849     }
3850
3851     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3852 }
3853
3854 my $fast_plug_option = {
3855     'lock' => 1,
3856     'name' => 1,
3857     'onboot' => 1,
3858     'shares' => 1,
3859     'startup' => 1,
3860 };
3861
3862 # hotplug changes in [PENDING]
3863 # $selection hash can be used to only apply specified options, for
3864 # example: { cores => 1 } (only apply changed 'cores')
3865 # $errors ref is used to return error messages
3866 sub vmconfig_hotplug_pending {
3867     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3868
3869     my $defaults = load_defaults();
3870
3871     # commit values which do not have any impact on running VM first
3872     # Note: those option cannot raise errors, we we do not care about
3873     # $selection and always apply them.
3874
3875     my $add_error = sub {
3876         my ($opt, $msg) = @_;
3877         $errors->{$opt} = "hotplug problem - $msg";
3878     };
3879
3880     my $changes = 0;
3881     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3882         if ($fast_plug_option->{$opt}) {
3883             $conf->{$opt} = $conf->{pending}->{$opt};
3884             delete $conf->{pending}->{$opt};
3885             $changes = 1;
3886         }
3887     }
3888
3889     if ($changes) {
3890         update_config_nolock($vmid, $conf, 1);
3891         $conf = load_config($vmid); # update/reload
3892     }
3893
3894     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3895
3896     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3897     foreach my $opt (@delete) {
3898         next if $selection && !$selection->{$opt};
3899         eval {
3900             if ($opt eq 'hotplug') {
3901                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3902             } elsif ($opt eq 'tablet') {
3903                 die "skip\n" if !$hotplug_features->{usb};
3904                 if ($defaults->{tablet}) {
3905                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3906                 } else {
3907                     vm_deviceunplug($vmid, $conf, $opt);
3908                 }
3909             } elsif ($opt eq 'vcpus') {
3910                 die "skip\n" if !$hotplug_features->{cpu};
3911                 qemu_cpu_hotplug($vmid, $conf, undef);
3912             } elsif ($opt eq 'balloon') {
3913                 # enable balloon device is not hotpluggable
3914                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3915             } elsif ($fast_plug_option->{$opt}) {
3916                 # do nothing
3917             } elsif ($opt =~ m/^net(\d+)$/) {
3918                 die "skip\n" if !$hotplug_features->{network};
3919                 vm_deviceunplug($vmid, $conf, $opt);
3920             } elsif (valid_drivename($opt)) {
3921                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3922                 vm_deviceunplug($vmid, $conf, $opt);
3923                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3924             } elsif ($opt =~ m/^memory$/) {
3925                 die "skip\n" if !$hotplug_features->{memory};
3926                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3927             } else {
3928                 die "skip\n";
3929             }
3930         };
3931         if (my $err = $@) {
3932             &$add_error($opt, $err) if $err ne "skip\n";
3933         } else {
3934             # save new config if hotplug was successful
3935             delete $conf->{$opt};
3936             vmconfig_undelete_pending_option($conf, $opt);
3937             update_config_nolock($vmid, $conf, 1);
3938             $conf = load_config($vmid); # update/reload
3939         }
3940     }
3941
3942     foreach my $opt (keys %{$conf->{pending}}) {
3943         next if $selection && !$selection->{$opt};
3944         my $value = $conf->{pending}->{$opt};
3945         eval {
3946             if ($opt eq 'hotplug') {
3947                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3948             } elsif ($opt eq 'tablet') {
3949                 die "skip\n" if !$hotplug_features->{usb};
3950                 if ($value == 1) {
3951                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3952                 } elsif ($value == 0) {
3953                     vm_deviceunplug($vmid, $conf, $opt);
3954                 }
3955             } elsif ($opt eq 'vcpus') {
3956                 die "skip\n" if !$hotplug_features->{cpu};
3957                 qemu_cpu_hotplug($vmid, $conf, $value);
3958             } elsif ($opt eq 'balloon') {
3959                 # enable/disable balloning device is not hotpluggable
3960                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3961                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3962                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3963
3964                 # allow manual ballooning if shares is set to zero
3965                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3966                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3967                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3968                 }
3969             } elsif ($opt =~ m/^net(\d+)$/) {
3970                 # some changes can be done without hotplug
3971                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3972                                     $vmid, $opt, $value);
3973             } elsif (valid_drivename($opt)) {
3974                 # some changes can be done without hotplug
3975                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3976                                      $vmid, $opt, $value, 1);
3977             } elsif ($opt =~ m/^memory$/) { #dimms
3978                 die "skip\n" if !$hotplug_features->{memory};
3979                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3980             } else {
3981                 die "skip\n";  # skip non-hot-pluggable options
3982             }
3983         };
3984         if (my $err = $@) {
3985             &$add_error($opt, $err) if $err ne "skip\n";
3986         } else {
3987             # save new config if hotplug was successful
3988             $conf->{$opt} = $value;
3989             delete $conf->{pending}->{$opt};
3990             update_config_nolock($vmid, $conf, 1);
3991             $conf = load_config($vmid); # update/reload
3992         }
3993     }
3994 }
3995
3996 sub vmconfig_apply_pending {
3997     my ($vmid, $conf, $storecfg) = @_;
3998
3999     # cold plug
4000
4001     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
4002     foreach my $opt (@delete) { # delete
4003         die "internal error" if $opt =~ m/^unused/;
4004         $conf = load_config($vmid); # update/reload
4005         if (!defined($conf->{$opt})) {
4006             vmconfig_undelete_pending_option($conf, $opt);
4007             update_config_nolock($vmid, $conf, 1);
4008         } elsif (valid_drivename($opt)) {
4009             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
4010             vmconfig_undelete_pending_option($conf, $opt);
4011             delete $conf->{$opt};
4012             update_config_nolock($vmid, $conf, 1);
4013         } else {
4014             vmconfig_undelete_pending_option($conf, $opt);
4015             delete $conf->{$opt};
4016             update_config_nolock($vmid, $conf, 1);
4017         }
4018     }
4019
4020     $conf = load_config($vmid); # update/reload
4021
4022     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4023         $conf = load_config($vmid); # update/reload
4024
4025         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4026             # skip if nothing changed
4027         } elsif (valid_drivename($opt)) {
4028             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4029                 if defined($conf->{$opt});
4030             $conf->{$opt} = $conf->{pending}->{$opt};
4031         } else {
4032             $conf->{$opt} = $conf->{pending}->{$opt};
4033         }
4034
4035         delete $conf->{pending}->{$opt};
4036         update_config_nolock($vmid, $conf, 1);
4037     }
4038 }
4039
4040 my $safe_num_ne = sub {
4041     my ($a, $b) = @_;
4042
4043     return 0 if !defined($a) && !defined($b);
4044     return 1 if !defined($a);
4045     return 1 if !defined($b);
4046
4047     return $a != $b;
4048 };
4049
4050 my $safe_string_ne = sub {
4051     my ($a, $b) = @_;
4052
4053     return 0 if !defined($a) && !defined($b);
4054     return 1 if !defined($a);
4055     return 1 if !defined($b);
4056
4057     return $a ne $b;
4058 };
4059
4060 sub vmconfig_update_net {
4061     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4062
4063     my $newnet = parse_net($value);
4064
4065     if ($conf->{$opt}) {
4066         my $oldnet = parse_net($conf->{$opt});
4067
4068         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4069             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4070             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4071             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4072
4073             # for non online change, we try to hot-unplug
4074             die "skip\n" if !$hotplug;
4075             vm_deviceunplug($vmid, $conf, $opt);
4076         } else {
4077
4078             die "internal error" if $opt !~ m/net(\d+)/;
4079             my $iface = "tap${vmid}i$1";
4080
4081             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4082                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4083             }
4084
4085             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4086                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4087                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4088                 PVE::Network::tap_unplug($iface);
4089                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4090             }
4091
4092             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4093                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4094             }
4095
4096             return 1;
4097         }
4098     }
4099
4100     if ($hotplug) {
4101         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4102     } else {
4103         die "skip\n";
4104     }
4105 }
4106
4107 sub vmconfig_update_disk {
4108     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4109
4110     # fixme: do we need force?
4111
4112     my $drive = parse_drive($opt, $value);
4113
4114     if ($conf->{$opt}) {
4115
4116         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4117
4118             my $media = $drive->{media} || 'disk';
4119             my $oldmedia = $old_drive->{media} || 'disk';
4120             die "unable to change media type\n" if $media ne $oldmedia;
4121
4122             if (!drive_is_cdrom($old_drive)) {
4123
4124                 if ($drive->{file} ne $old_drive->{file}) {
4125
4126                     die "skip\n" if !$hotplug;
4127
4128                     # unplug and register as unused
4129                     vm_deviceunplug($vmid, $conf, $opt);
4130                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4131
4132                 } else {
4133                     # update existing disk
4134
4135                     # skip non hotpluggable value
4136                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4137                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4138                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4139                         die "skip\n";
4140                     }
4141
4142                     # apply throttle
4143                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4144                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4145                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4146                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4147                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4148                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4149                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4150                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4151                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4152                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4153                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4154                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4155
4156                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4157                                                    ($drive->{mbps} || 0)*1024*1024,
4158                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4159                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4160                                                    $drive->{iops} || 0,
4161                                                    $drive->{iops_rd} || 0,
4162                                                    $drive->{iops_wr} || 0,
4163                                                    ($drive->{mbps_max} || 0)*1024*1024,
4164                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4165                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4166                                                    $drive->{iops_max} || 0,
4167                                                    $drive->{iops_rd_max} || 0,
4168                                                    $drive->{iops_wr_max} || 0);
4169
4170                     }
4171
4172                     return 1;
4173                 }
4174
4175             } else { # cdrom
4176
4177                 if ($drive->{file} eq 'none') {
4178                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4179                 } else {
4180                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4181                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4182                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4183                 }
4184
4185                 return 1;
4186             }
4187         }
4188     }
4189
4190     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4191     # hotplug new disks
4192     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4193 }
4194
4195 sub vm_start {
4196     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4197
4198     lock_config($vmid, sub {
4199         my $conf = load_config($vmid, $migratedfrom);
4200
4201         die "you can't start a vm if it's a template\n" if is_template($conf);
4202
4203         check_lock($conf) if !$skiplock;
4204
4205         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4206
4207         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4208             vmconfig_apply_pending($vmid, $conf, $storecfg);
4209             $conf = load_config($vmid); # update/reload
4210         }
4211
4212         my $defaults = load_defaults();
4213
4214         # set environment variable useful inside network script
4215         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4216
4217         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4218
4219         my $migrate_port = 0;
4220         my $migrate_uri;
4221         if ($statefile) {
4222             if ($statefile eq 'tcp') {
4223                 my $localip = "localhost";
4224                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4225                 if ($datacenterconf->{migration_unsecure}) {
4226                         my $nodename = PVE::INotify::nodename();
4227                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4228                 }
4229                 $migrate_port = PVE::Tools::next_migrate_port();
4230                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4231                 push @$cmd, '-incoming', $migrate_uri;
4232                 push @$cmd, '-S';
4233             } else {
4234                 push @$cmd, '-loadstate', $statefile;
4235             }
4236         } elsif ($paused) {
4237             push @$cmd, '-S';
4238         }
4239
4240         # host pci devices
4241         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4242           my $d = parse_hostpci($conf->{"hostpci$i"});
4243           next if !$d;
4244           my $pcidevices = $d->{pciid};
4245           foreach my $pcidevice (@$pcidevices) {
4246                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4247
4248                 my $info = pci_device_info("0000:$pciid");
4249                 die "IOMMU not present\n" if !check_iommu_support();
4250                 die "no pci device info for device '$pciid'\n" if !$info;
4251
4252                 if ($d->{driver} && $d->{driver} eq "vfio") {
4253                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4254                 } else {
4255                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4256                 }
4257
4258                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4259           }
4260         }
4261
4262         PVE::Storage::activate_volumes($storecfg, $vollist);
4263
4264         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4265                     umask => 0077); };
4266         my $err = $@;
4267         die "start failed: $err" if $err;
4268
4269         print "migration listens on $migrate_uri\n" if $migrate_uri;
4270
4271         if ($statefile && $statefile ne 'tcp')  {
4272             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4273             warn $@ if $@;
4274         }
4275
4276         if ($migratedfrom) {
4277
4278             eval {
4279                 set_migration_caps($vmid);
4280             };
4281             warn $@ if $@;
4282
4283             if ($spice_port) {
4284                 print "spice listens on port $spice_port\n";
4285                 if ($spice_ticket) {
4286                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4287                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4288                 }
4289             }
4290
4291         } else {
4292
4293             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4294                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4295                     if $conf->{balloon};
4296             }
4297
4298             foreach my $opt (keys %$conf) {
4299                 next if $opt !~  m/^net\d+$/;
4300                 my $nicconf = parse_net($conf->{$opt});
4301                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4302             }
4303         }
4304
4305         vm_mon_cmd_nocheck($vmid, 'qom-set',
4306                     path => "machine/peripheral/balloon0",
4307                     property => "guest-stats-polling-interval",
4308                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4309
4310     });
4311 }
4312
4313 sub vm_mon_cmd {
4314     my ($vmid, $execute, %params) = @_;
4315
4316     my $cmd = { execute => $execute, arguments => \%params };
4317     vm_qmp_command($vmid, $cmd);
4318 }
4319
4320 sub vm_mon_cmd_nocheck {
4321     my ($vmid, $execute, %params) = @_;
4322
4323     my $cmd = { execute => $execute, arguments => \%params };
4324     vm_qmp_command($vmid, $cmd, 1);
4325 }
4326
4327 sub vm_qmp_command {
4328     my ($vmid, $cmd, $nocheck) = @_;
4329
4330     my $res;
4331
4332     my $timeout;
4333     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4334         $timeout = $cmd->{arguments}->{timeout};
4335         delete $cmd->{arguments}->{timeout};
4336     }
4337
4338     eval {
4339         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4340         my $sname = qmp_socket($vmid);
4341         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4342             my $qmpclient = PVE::QMPClient->new();
4343
4344             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4345         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4346             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4347                 if scalar(%{$cmd->{arguments}});
4348             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4349         } else {
4350             die "unable to open monitor socket\n";
4351         }
4352     };
4353     if (my $err = $@) {
4354         syslog("err", "VM $vmid qmp command failed - $err");
4355         die $err;
4356     }
4357
4358     return $res;
4359 }
4360
4361 sub vm_human_monitor_command {
4362     my ($vmid, $cmdline) = @_;
4363
4364     my $res;
4365
4366     my $cmd = {
4367         execute => 'human-monitor-command',
4368         arguments => { 'command-line' => $cmdline},
4369     };
4370
4371     return vm_qmp_command($vmid, $cmd);
4372 }
4373
4374 sub vm_commandline {
4375     my ($storecfg, $vmid) = @_;
4376
4377     my $conf = load_config($vmid);
4378
4379     my $defaults = load_defaults();
4380
4381     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4382
4383     return join(' ', @$cmd);
4384 }
4385
4386 sub vm_reset {
4387     my ($vmid, $skiplock) = @_;
4388
4389     lock_config($vmid, sub {
4390
4391         my $conf = load_config($vmid);
4392
4393         check_lock($conf) if !$skiplock;
4394
4395         vm_mon_cmd($vmid, "system_reset");
4396     });
4397 }
4398
4399 sub get_vm_volumes {
4400     my ($conf) = @_;
4401
4402     my $vollist = [];
4403     foreach_volid($conf, sub {
4404         my ($volid, $is_cdrom) = @_;
4405
4406         return if $volid =~ m|^/|;
4407
4408         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4409         return if !$sid;
4410
4411         push @$vollist, $volid;
4412     });
4413
4414     return $vollist;
4415 }
4416
4417 sub vm_stop_cleanup {
4418     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4419
4420     eval {
4421         fairsched_rmnod($vmid); # try to destroy group
4422
4423         if (!$keepActive) {
4424             my $vollist = get_vm_volumes($conf);
4425             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4426         }
4427
4428         foreach my $ext (qw(mon qmp pid vnc qga)) {
4429             unlink "/var/run/qemu-server/${vmid}.$ext";
4430         }
4431
4432         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4433     };
4434     warn $@ if $@; # avoid errors - just warn
4435 }
4436
4437 # Note: use $nockeck to skip tests if VM configuration file exists.
4438 # We need that when migration VMs to other nodes (files already moved)
4439 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4440 sub vm_stop {
4441     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4442
4443     $force = 1 if !defined($force) && !$shutdown;
4444
4445     if ($migratedfrom){
4446         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4447         kill 15, $pid if $pid;
4448         my $conf = load_config($vmid, $migratedfrom);
4449         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4450         return;
4451     }
4452
4453     lock_config($vmid, sub {
4454
4455         my $pid = check_running($vmid, $nocheck);
4456         return if !$pid;
4457
4458         my $conf;
4459         if (!$nocheck) {
4460             $conf = load_config($vmid);
4461             check_lock($conf) if !$skiplock;
4462             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4463                 my $opts = parse_startup($conf->{startup});
4464                 $timeout = $opts->{down} if $opts->{down};
4465             }
4466         }
4467
4468         $timeout = 60 if !defined($timeout);
4469
4470         eval {
4471             if ($shutdown) {
4472                 if (defined($conf) && $conf->{agent}) {
4473                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4474                 } else {
4475                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4476                 }
4477             } else {
4478                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4479             }
4480         };
4481         my $err = $@;
4482
4483         if (!$err) {
4484             my $count = 0;
4485             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4486                 $count++;
4487                 sleep 1;
4488             }
4489
4490             if ($count >= $timeout) {
4491                 if ($force) {
4492                     warn "VM still running - terminating now with SIGTERM\n";
4493                     kill 15, $pid;
4494                 } else {
4495                     die "VM quit/powerdown failed - got timeout\n";
4496                 }
4497             } else {
4498                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4499                 return;
4500             }
4501         } else {
4502             if ($force) {
4503                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4504                 kill 15, $pid;
4505             } else {
4506                 die "VM quit/powerdown failed\n";
4507             }
4508         }
4509
4510         # wait again
4511         $timeout = 10;
4512
4513         my $count = 0;
4514         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4515             $count++;
4516             sleep 1;
4517         }
4518
4519         if ($count >= $timeout) {
4520             warn "VM still running - terminating now with SIGKILL\n";
4521             kill 9, $pid;
4522             sleep 1;
4523         }
4524
4525         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4526    });
4527 }
4528
4529 sub vm_suspend {
4530     my ($vmid, $skiplock) = @_;
4531
4532     lock_config($vmid, sub {
4533
4534         my $conf = load_config($vmid);
4535
4536         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4537
4538         vm_mon_cmd($vmid, "stop");
4539     });
4540 }
4541
4542 sub vm_resume {
4543     my ($vmid, $skiplock) = @_;
4544
4545     lock_config($vmid, sub {
4546
4547         my $conf = load_config($vmid);
4548
4549         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4550
4551         vm_mon_cmd($vmid, "cont");
4552     });
4553 }
4554
4555 sub vm_sendkey {
4556     my ($vmid, $skiplock, $key) = @_;
4557
4558     lock_config($vmid, sub {
4559
4560         my $conf = load_config($vmid);
4561
4562         # there is no qmp command, so we use the human monitor command
4563         vm_human_monitor_command($vmid, "sendkey $key");
4564     });
4565 }
4566
4567 sub vm_destroy {
4568     my ($storecfg, $vmid, $skiplock) = @_;
4569
4570     lock_config($vmid, sub {
4571
4572         my $conf = load_config($vmid);
4573
4574         check_lock($conf) if !$skiplock;
4575
4576         if (!check_running($vmid)) {
4577             fairsched_rmnod($vmid); # try to destroy group
4578             destroy_vm($storecfg, $vmid);
4579         } else {
4580             die "VM $vmid is running - destroy failed\n";
4581         }
4582     });
4583 }
4584
4585 # pci helpers
4586
4587 sub file_write {
4588     my ($filename, $buf) = @_;
4589
4590     my $fh = IO::File->new($filename, "w");
4591     return undef if !$fh;
4592
4593     my $res = print $fh $buf;
4594
4595     $fh->close();
4596
4597     return $res;
4598 }
4599
4600 sub pci_device_info {
4601     my ($name) = @_;
4602
4603     my $res;
4604
4605     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4606     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4607
4608     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4609     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4610
4611     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4612     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4613
4614     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4615     return undef if !defined($product) || $product !~ s/^0x//;
4616
4617     $res = {
4618         name => $name,
4619         vendor => $vendor,
4620         product => $product,
4621         domain => $domain,
4622         bus => $bus,
4623         slot => $slot,
4624         func => $func,
4625         irq => $irq,
4626         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4627     };
4628
4629     return $res;
4630 }
4631
4632 sub pci_dev_reset {
4633     my ($dev) = @_;
4634
4635     my $name = $dev->{name};
4636
4637     my $fn = "$pcisysfs/devices/$name/reset";
4638
4639     return file_write($fn, "1");
4640 }
4641
4642 sub pci_dev_bind_to_stub {
4643     my ($dev) = @_;
4644
4645     my $name = $dev->{name};
4646
4647     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4648     return 1 if -d $testdir;
4649
4650     my $data = "$dev->{vendor} $dev->{product}";
4651     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4652
4653     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4654     if (!file_write($fn, $name)) {
4655         return undef if -f $fn;
4656     }
4657
4658     $fn = "$pcisysfs/drivers/pci-stub/bind";
4659     if (! -d $testdir) {
4660         return undef if !file_write($fn, $name);
4661     }
4662
4663     return -d $testdir;
4664 }
4665
4666 sub pci_dev_bind_to_vfio {
4667     my ($dev) = @_;
4668
4669     my $name = $dev->{name};
4670
4671     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4672
4673     if (!-d $vfio_basedir) {
4674         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4675     }
4676     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4677
4678     my $testdir = "$vfio_basedir/$name";
4679     return 1 if -d $testdir;
4680
4681     my $data = "$dev->{vendor} $dev->{product}";
4682     return undef if !file_write("$vfio_basedir/new_id", $data);
4683
4684     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4685     if (!file_write($fn, $name)) {
4686         return undef if -f $fn;
4687     }
4688
4689     $fn = "$vfio_basedir/bind";
4690     if (! -d $testdir) {
4691         return undef if !file_write($fn, $name);
4692     }
4693
4694     return -d $testdir;
4695 }
4696
4697 sub pci_dev_group_bind_to_vfio {
4698     my ($pciid) = @_;
4699
4700     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4701
4702     if (!-d $vfio_basedir) {
4703         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4704     }
4705     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4706
4707     # get IOMMU group devices
4708     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4709       my @devs = grep /^0000:/, readdir($D);
4710     closedir($D);
4711
4712     foreach my $pciid (@devs) {
4713         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4714
4715         # pci bridges, switches or root ports are not supported
4716         # they have a pci_bus subdirectory so skip them
4717         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4718
4719         my $info = pci_device_info($1);
4720         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4721     }
4722
4723     return 1;
4724 }
4725
4726 sub print_pci_addr {
4727     my ($id, $bridges) = @_;
4728
4729     my $res = '';
4730     my $devices = {
4731         piix3 => { bus => 0, addr => 1 },
4732         #addr2 : first videocard
4733         balloon0 => { bus => 0, addr => 3 },
4734         watchdog => { bus => 0, addr => 4 },
4735         scsihw0 => { bus => 0, addr => 5 },
4736         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4737         scsihw1 => { bus => 0, addr => 6 },
4738         ahci0 => { bus => 0, addr => 7 },
4739         qga0 => { bus => 0, addr => 8 },
4740         spice => { bus => 0, addr => 9 },
4741         virtio0 => { bus => 0, addr => 10 },
4742         virtio1 => { bus => 0, addr => 11 },
4743         virtio2 => { bus => 0, addr => 12 },
4744         virtio3 => { bus => 0, addr => 13 },
4745         virtio4 => { bus => 0, addr => 14 },
4746         virtio5 => { bus => 0, addr => 15 },
4747         hostpci0 => { bus => 0, addr => 16 },
4748         hostpci1 => { bus => 0, addr => 17 },
4749         net0 => { bus => 0, addr => 18 },
4750         net1 => { bus => 0, addr => 19 },
4751         net2 => { bus => 0, addr => 20 },
4752         net3 => { bus => 0, addr => 21 },
4753         net4 => { bus => 0, addr => 22 },
4754         net5 => { bus => 0, addr => 23 },
4755         vga1 => { bus => 0, addr => 24 },
4756         vga2 => { bus => 0, addr => 25 },
4757         vga3 => { bus => 0, addr => 26 },
4758         hostpci2 => { bus => 0, addr => 27 },
4759         hostpci3 => { bus => 0, addr => 28 },
4760         #addr29 : usb-host (pve-usb.cfg)
4761         'pci.1' => { bus => 0, addr => 30 },
4762         'pci.2' => { bus => 0, addr => 31 },
4763         'net6' => { bus => 1, addr => 1 },
4764         'net7' => { bus => 1, addr => 2 },
4765         'net8' => { bus => 1, addr => 3 },
4766         'net9' => { bus => 1, addr => 4 },
4767         'net10' => { bus => 1, addr => 5 },
4768         'net11' => { bus => 1, addr => 6 },
4769         'net12' => { bus => 1, addr => 7 },
4770         'net13' => { bus => 1, addr => 8 },
4771         'net14' => { bus => 1, addr => 9 },
4772         'net15' => { bus => 1, addr => 10 },
4773         'net16' => { bus => 1, addr => 11 },
4774         'net17' => { bus => 1, addr => 12 },
4775         'net18' => { bus => 1, addr => 13 },
4776         'net19' => { bus => 1, addr => 14 },
4777         'net20' => { bus => 1, addr => 15 },
4778         'net21' => { bus => 1, addr => 16 },
4779         'net22' => { bus => 1, addr => 17 },
4780         'net23' => { bus => 1, addr => 18 },
4781         'net24' => { bus => 1, addr => 19 },
4782         'net25' => { bus => 1, addr => 20 },
4783         'net26' => { bus => 1, addr => 21 },
4784         'net27' => { bus => 1, addr => 22 },
4785         'net28' => { bus => 1, addr => 23 },
4786         'net29' => { bus => 1, addr => 24 },
4787         'net30' => { bus => 1, addr => 25 },
4788         'net31' => { bus => 1, addr => 26 },
4789         'virtio6' => { bus => 2, addr => 1 },
4790         'virtio7' => { bus => 2, addr => 2 },
4791         'virtio8' => { bus => 2, addr => 3 },
4792         'virtio9' => { bus => 2, addr => 4 },
4793         'virtio10' => { bus => 2, addr => 5 },
4794         'virtio11' => { bus => 2, addr => 6 },
4795         'virtio12' => { bus => 2, addr => 7 },
4796         'virtio13' => { bus => 2, addr => 8 },
4797         'virtio14' => { bus => 2, addr => 9 },
4798         'virtio15' => { bus => 2, addr => 10 },
4799         'virtioscsi0' => { bus => 3, addr => 1 },
4800         'virtioscsi1' => { bus => 3, addr => 2 },
4801         'virtioscsi2' => { bus => 3, addr => 3 },
4802         'virtioscsi3' => { bus => 3, addr => 4 },
4803         'virtioscsi4' => { bus => 3, addr => 5 },
4804         'virtioscsi5' => { bus => 3, addr => 6 },
4805         'virtioscsi6' => { bus => 3, addr => 7 },
4806         'virtioscsi7' => { bus => 3, addr => 8 },
4807         'virtioscsi8' => { bus => 3, addr => 9 },
4808         'virtioscsi9' => { bus => 3, addr => 10 },
4809         'virtioscsi10' => { bus => 3, addr => 11 },
4810         'virtioscsi11' => { bus => 3, addr => 12 },
4811         'virtioscsi12' => { bus => 3, addr => 13 },
4812         'virtioscsi13' => { bus => 3, addr => 14 },
4813         'virtioscsi14' => { bus => 3, addr => 15 },
4814         'virtioscsi15' => { bus => 3, addr => 16 },
4815         'virtioscsi16' => { bus => 3, addr => 17 },
4816         'virtioscsi17' => { bus => 3, addr => 18 },
4817         'virtioscsi18' => { bus => 3, addr => 19 },
4818         'virtioscsi19' => { bus => 3, addr => 20 },
4819         'virtioscsi20' => { bus => 3, addr => 21 },
4820         'virtioscsi21' => { bus => 3, addr => 22 },
4821         'virtioscsi22' => { bus => 3, addr => 23 },
4822         'virtioscsi23' => { bus => 3, addr => 24 },
4823         'virtioscsi24' => { bus => 3, addr => 25 },
4824         'virtioscsi25' => { bus => 3, addr => 26 },
4825         'virtioscsi26' => { bus => 3, addr => 27 },
4826         'virtioscsi27' => { bus => 3, addr => 28 },
4827         'virtioscsi28' => { bus => 3, addr => 29 },
4828         'virtioscsi29' => { bus => 3, addr => 30 },
4829         'virtioscsi30' => { bus => 3, addr => 31 },
4830
4831     };
4832
4833     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4834            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4835            my $bus = $devices->{$id}->{bus};
4836            $res = ",bus=pci.$bus,addr=$addr";
4837            $bridges->{$bus} = 1 if $bridges;
4838     }
4839     return $res;
4840
4841 }
4842
4843 sub print_pcie_addr {
4844     my ($id) = @_;
4845
4846     my $res = '';
4847     my $devices = {
4848         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4849         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4850         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4851         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4852     };
4853
4854     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4855            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4856            my $bus = $devices->{$id}->{bus};
4857            $res = ",bus=$bus,addr=$addr";
4858     }
4859     return $res;
4860
4861 }
4862
4863 # vzdump restore implementaion
4864
4865 sub tar_archive_read_firstfile {
4866     my $archive = shift;
4867
4868     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4869
4870     # try to detect archive type first
4871     my $pid = open (TMP, "tar tf '$archive'|") ||
4872         die "unable to open file '$archive'\n";
4873     my $firstfile = <TMP>;
4874     kill 15, $pid;
4875     close TMP;
4876
4877     die "ERROR: archive contaions no data\n" if !$firstfile;
4878     chomp $firstfile;
4879
4880     return $firstfile;
4881 }
4882
4883 sub tar_restore_cleanup {
4884     my ($storecfg, $statfile) = @_;
4885
4886     print STDERR "starting cleanup\n";
4887
4888     if (my $fd = IO::File->new($statfile, "r")) {
4889         while (defined(my $line = <$fd>)) {
4890             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4891                 my $volid = $2;
4892                 eval {
4893                     if ($volid =~ m|^/|) {
4894                         unlink $volid || die 'unlink failed\n';
4895                     } else {
4896                         PVE::Storage::vdisk_free($storecfg, $volid);
4897                     }
4898                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4899                 };
4900                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4901             } else {
4902                 print STDERR "unable to parse line in statfile - $line";
4903             }
4904         }
4905         $fd->close();
4906     }
4907 }
4908
4909 sub restore_archive {
4910     my ($archive, $vmid, $user, $opts) = @_;
4911
4912     my $format = $opts->{format};
4913     my $comp;
4914
4915     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4916         $format = 'tar' if !$format;
4917         $comp = 'gzip';
4918     } elsif ($archive =~ m/\.tar$/) {
4919         $format = 'tar' if !$format;
4920     } elsif ($archive =~ m/.tar.lzo$/) {
4921         $format = 'tar' if !$format;
4922         $comp = 'lzop';
4923     } elsif ($archive =~ m/\.vma$/) {
4924         $format = 'vma' if !$format;
4925     } elsif ($archive =~ m/\.vma\.gz$/) {
4926         $format = 'vma' if !$format;
4927         $comp = 'gzip';
4928     } elsif ($archive =~ m/\.vma\.lzo$/) {
4929         $format = 'vma' if !$format;
4930         $comp = 'lzop';
4931     } else {
4932         $format = 'vma' if !$format; # default
4933     }
4934
4935     # try to detect archive format
4936     if ($format eq 'tar') {
4937         return restore_tar_archive($archive, $vmid, $user, $opts);
4938     } else {
4939         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4940     }
4941 }
4942
4943 sub restore_update_config_line {
4944     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4945
4946     return if $line =~ m/^\#qmdump\#/;
4947     return if $line =~ m/^\#vzdump\#/;
4948     return if $line =~ m/^lock:/;
4949     return if $line =~ m/^unused\d+:/;
4950     return if $line =~ m/^parent:/;
4951     return if $line =~ m/^template:/; # restored VM is never a template
4952
4953     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4954         # try to convert old 1.X settings
4955         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4956         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4957             my ($model, $macaddr) = split(/\=/, $devconfig);
4958             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4959             my $net = {
4960                 model => $model,
4961                 bridge => "vmbr$ind",
4962                 macaddr => $macaddr,
4963             };
4964             my $netstr = print_net($net);
4965
4966             print $outfd "net$cookie->{netcount}: $netstr\n";
4967             $cookie->{netcount}++;
4968         }
4969     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4970         my ($id, $netstr) = ($1, $2);
4971         my $net = parse_net($netstr);
4972         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4973         $netstr = print_net($net);
4974         print $outfd "$id: $netstr\n";
4975     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4976         my $virtdev = $1;
4977         my $value = $3;
4978         if ($line =~ m/backup=no/) {
4979             print $outfd "#$line";
4980         } elsif ($virtdev && $map->{$virtdev}) {
4981             my $di = parse_drive($virtdev, $value);
4982             delete $di->{format}; # format can change on restore
4983             $di->{file} = $map->{$virtdev};
4984             $value = print_drive($vmid, $di);
4985             print $outfd "$virtdev: $value\n";
4986         } else {
4987             print $outfd $line;
4988         }
4989     } else {
4990         print $outfd $line;
4991     }
4992 }
4993
4994 sub scan_volids {
4995     my ($cfg, $vmid) = @_;
4996
4997     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4998
4999     my $volid_hash = {};
5000     foreach my $storeid (keys %$info) {
5001         foreach my $item (@{$info->{$storeid}}) {
5002             next if !($item->{volid} && $item->{size});
5003             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5004             $volid_hash->{$item->{volid}} = $item;
5005         }
5006     }
5007
5008     return $volid_hash;
5009 }
5010
5011 sub get_used_paths {
5012     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5013
5014     my $used_path = {};
5015
5016     my $scan_config = sub {
5017         my ($cref, $snapname) = @_;
5018
5019         foreach my $key (keys %$cref) {
5020             my $value = $cref->{$key};
5021             if (valid_drivename($key)) {
5022                 next if $skip_drive && $key eq $skip_drive;
5023                 my $drive = parse_drive($key, $value);
5024                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5025                 if ($drive->{file} =~ m!^/!) {
5026                     $used_path->{$drive->{file}}++; # = 1;
5027                 } else {
5028                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5029                     next if !$storeid;
5030                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5031                     next if !$scfg;
5032                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5033                     $used_path->{$path}++; # = 1;
5034                 }
5035             }
5036         }
5037     };
5038
5039     &$scan_config($conf);
5040
5041     undef $skip_drive;
5042
5043     if ($scan_snapshots) {
5044         foreach my $snapname (keys %{$conf->{snapshots}}) {
5045             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5046         }
5047     }
5048
5049     return $used_path;
5050 }
5051
5052 sub update_disksize {
5053     my ($vmid, $conf, $volid_hash) = @_;
5054
5055     my $changes;
5056
5057     my $used = {};
5058
5059     # Note: it is allowed to define multiple storages with same path (alias), so
5060     # we need to check both 'volid' and real 'path' (two different volid can point
5061     # to the same path).
5062
5063     my $usedpath = {};
5064
5065     # update size info
5066     foreach my $opt (keys %$conf) {
5067         if (valid_drivename($opt)) {
5068             my $drive = parse_drive($opt, $conf->{$opt});
5069             my $volid = $drive->{file};
5070             next if !$volid;
5071
5072             $used->{$volid} = 1;
5073             if ($volid_hash->{$volid} &&
5074                 (my $path = $volid_hash->{$volid}->{path})) {
5075                 $usedpath->{$path} = 1;
5076             }
5077
5078             next if drive_is_cdrom($drive);
5079             next if !$volid_hash->{$volid};
5080
5081             $drive->{size} = $volid_hash->{$volid}->{size};
5082             my $new = print_drive($vmid, $drive);
5083             if ($new ne $conf->{$opt}) {
5084                 $changes = 1;
5085                 $conf->{$opt} = $new;
5086             }
5087         }
5088     }
5089
5090     # remove 'unusedX' entry if volume is used
5091     foreach my $opt (keys %$conf) {
5092         next if $opt !~ m/^unused\d+$/;
5093         my $volid = $conf->{$opt};
5094         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5095         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5096             $changes = 1;
5097             delete $conf->{$opt};
5098         }
5099     }
5100
5101     foreach my $volid (sort keys %$volid_hash) {
5102         next if $volid =~ m/vm-$vmid-state-/;
5103         next if $used->{$volid};
5104         my $path = $volid_hash->{$volid}->{path};
5105         next if !$path; # just to be sure
5106         next if $usedpath->{$path};
5107         $changes = 1;
5108         add_unused_volume($conf, $volid);
5109         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5110     }
5111
5112     return $changes;
5113 }
5114
5115 sub rescan {
5116     my ($vmid, $nolock) = @_;
5117
5118     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5119
5120     my $volid_hash = scan_volids($cfg, $vmid);
5121
5122     my $updatefn =  sub {
5123         my ($vmid) = @_;
5124
5125         my $conf = load_config($vmid);
5126
5127         check_lock($conf);
5128
5129         my $vm_volids = {};
5130         foreach my $volid (keys %$volid_hash) {
5131             my $info = $volid_hash->{$volid};
5132             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5133         }
5134
5135         my $changes = update_disksize($vmid, $conf, $vm_volids);
5136
5137         update_config_nolock($vmid, $conf, 1) if $changes;
5138     };
5139
5140     if (defined($vmid)) {
5141         if ($nolock) {
5142             &$updatefn($vmid);
5143         } else {
5144             lock_config($vmid, $updatefn, $vmid);
5145         }
5146     } else {
5147         my $vmlist = config_list();
5148         foreach my $vmid (keys %$vmlist) {
5149             if ($nolock) {
5150                 &$updatefn($vmid);
5151             } else {
5152                 lock_config($vmid, $updatefn, $vmid);
5153             }
5154         }
5155     }
5156 }
5157
5158 sub restore_vma_archive {
5159     my ($archive, $vmid, $user, $opts, $comp) = @_;
5160
5161     my $input = $archive eq '-' ? "<&STDIN" : undef;
5162     my $readfrom = $archive;
5163
5164     my $uncomp = '';
5165     if ($comp) {
5166         $readfrom = '-';
5167         my $qarchive = PVE::Tools::shellquote($archive);
5168         if ($comp eq 'gzip') {
5169             $uncomp = "zcat $qarchive|";
5170         } elsif ($comp eq 'lzop') {
5171             $uncomp = "lzop -d -c $qarchive|";
5172         } else {
5173             die "unknown compression method '$comp'\n";
5174         }
5175
5176     }
5177
5178     my $tmpdir = "/var/tmp/vzdumptmp$$";
5179     rmtree $tmpdir;
5180
5181     # disable interrupts (always do cleanups)
5182     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5183         warn "got interrupt - ignored\n";
5184     };
5185
5186     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5187     POSIX::mkfifo($mapfifo, 0600);
5188     my $fifofh;
5189
5190     my $openfifo = sub {
5191         open($fifofh, '>', $mapfifo) || die $!;
5192     };
5193
5194     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5195
5196     my $oldtimeout;
5197     my $timeout = 5;
5198
5199     my $devinfo = {};
5200
5201     my $rpcenv = PVE::RPCEnvironment::get();
5202
5203     my $conffile = config_file($vmid);
5204     my $tmpfn = "$conffile.$$.tmp";
5205
5206     # Note: $oldconf is undef if VM does not exists
5207     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5208
5209     my $print_devmap = sub {
5210         my $virtdev_hash = {};
5211
5212         my $cfgfn = "$tmpdir/qemu-server.conf";
5213
5214         # we can read the config - that is already extracted
5215         my $fh = IO::File->new($cfgfn, "r") ||
5216             "unable to read qemu-server.conf - $!\n";
5217
5218         while (defined(my $line = <$fh>)) {
5219             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5220                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5221                 die "archive does not contain data for drive '$virtdev'\n"
5222                     if !$devinfo->{$devname};
5223                 if (defined($opts->{storage})) {
5224                     $storeid = $opts->{storage} || 'local';
5225                 } elsif (!$storeid) {
5226                     $storeid = 'local';
5227                 }
5228                 $format = 'raw' if !$format;
5229                 $devinfo->{$devname}->{devname} = $devname;
5230                 $devinfo->{$devname}->{virtdev} = $virtdev;
5231                 $devinfo->{$devname}->{format} = $format;
5232                 $devinfo->{$devname}->{storeid} = $storeid;
5233
5234                 # check permission on storage
5235                 my $pool = $opts->{pool}; # todo: do we need that?
5236                 if ($user ne 'root@pam') {
5237                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5238                 }
5239
5240                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5241             }
5242         }
5243
5244         foreach my $devname (keys %$devinfo) {
5245             die "found no device mapping information for device '$devname'\n"
5246                 if !$devinfo->{$devname}->{virtdev};
5247         }
5248
5249         my $cfg = cfs_read_file('storage.cfg');
5250
5251         # create empty/temp config
5252         if ($oldconf) {
5253             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5254             foreach_drive($oldconf, sub {
5255                 my ($ds, $drive) = @_;
5256
5257                 return if drive_is_cdrom($drive);
5258
5259                 my $volid = $drive->{file};
5260
5261                 return if !$volid || $volid =~ m|^/|;
5262
5263                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5264                 return if !$path || !$owner || ($owner != $vmid);
5265
5266                 # Note: only delete disk we want to restore
5267                 # other volumes will become unused
5268                 if ($virtdev_hash->{$ds}) {
5269                     PVE::Storage::vdisk_free($cfg, $volid);
5270                 }
5271             });
5272         }
5273
5274         my $map = {};
5275         foreach my $virtdev (sort keys %$virtdev_hash) {
5276             my $d = $virtdev_hash->{$virtdev};
5277             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5278             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5279
5280             # test if requested format is supported
5281             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5282             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5283             $d->{format} = $defFormat if !$supported;
5284
5285             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5286                                                   $d->{format}, undef, $alloc_size);
5287             print STDERR "new volume ID is '$volid'\n";
5288             $d->{volid} = $volid;
5289             my $path = PVE::Storage::path($cfg, $volid);
5290
5291             my $write_zeros = 1;
5292             # fixme: what other storages types initialize volumes with zero?
5293             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5294                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5295                 $write_zeros = 0;
5296             }
5297
5298             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5299
5300             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5301             $map->{$virtdev} = $volid;
5302         }
5303
5304         $fh->seek(0, 0) || die "seek failed - $!\n";
5305
5306         my $outfd = new IO::File ($tmpfn, "w") ||
5307             die "unable to write config for VM $vmid\n";
5308
5309         my $cookie = { netcount => 0 };
5310         while (defined(my $line = <$fh>)) {
5311             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5312         }
5313
5314         $fh->close();
5315         $outfd->close();
5316     };
5317
5318     eval {
5319         # enable interrupts
5320         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5321             die "interrupted by signal\n";
5322         };
5323         local $SIG{ALRM} = sub { die "got timeout\n"; };
5324
5325         $oldtimeout = alarm($timeout);
5326
5327         my $parser = sub {
5328             my $line = shift;
5329
5330             print "$line\n";
5331
5332             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5333                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5334                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5335             } elsif ($line =~ m/^CTIME: /) {
5336                 # we correctly received the vma config, so we can disable
5337                 # the timeout now for disk allocation (set to 10 minutes, so
5338                 # that we always timeout if something goes wrong)
5339                 alarm(600);
5340                 &$print_devmap();
5341                 print $fifofh "done\n";
5342                 my $tmp = $oldtimeout || 0;
5343                 $oldtimeout = undef;
5344                 alarm($tmp);
5345                 close($fifofh);
5346             }
5347         };
5348
5349         print "restore vma archive: $cmd\n";
5350         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5351     };
5352     my $err = $@;
5353
5354     alarm($oldtimeout) if $oldtimeout;
5355
5356     unlink $mapfifo;
5357
5358     if ($err) {
5359         rmtree $tmpdir;
5360         unlink $tmpfn;
5361
5362         my $cfg = cfs_read_file('storage.cfg');
5363         foreach my $devname (keys %$devinfo) {
5364             my $volid = $devinfo->{$devname}->{volid};
5365             next if !$volid;
5366             eval {
5367                 if ($volid =~ m|^/|) {
5368                     unlink $volid || die 'unlink failed\n';
5369                 } else {
5370                     PVE::Storage::vdisk_free($cfg, $volid);
5371                 }
5372                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5373             };
5374             print STDERR "unable to cleanup '$volid' - $@" if $@;
5375         }
5376         die $err;
5377     }
5378
5379     rmtree $tmpdir;
5380
5381     rename($tmpfn, $conffile) ||
5382         die "unable to commit configuration file '$conffile'\n";
5383
5384     PVE::Cluster::cfs_update(); # make sure we read new file
5385
5386     eval { rescan($vmid, 1); };
5387     warn $@ if $@;
5388 }
5389
5390 sub restore_tar_archive {
5391     my ($archive, $vmid, $user, $opts) = @_;
5392
5393     if ($archive ne '-') {
5394         my $firstfile = tar_archive_read_firstfile($archive);
5395         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5396             if $firstfile ne 'qemu-server.conf';
5397     }
5398
5399     my $storecfg = cfs_read_file('storage.cfg');
5400
5401     # destroy existing data - keep empty config
5402     my $vmcfgfn = config_file($vmid);
5403     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5404
5405     my $tocmd = "/usr/lib/qemu-server/qmextract";
5406
5407     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5408     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5409     $tocmd .= ' --prealloc' if $opts->{prealloc};
5410     $tocmd .= ' --info' if $opts->{info};
5411
5412     # tar option "xf" does not autodetect compression when read from STDIN,
5413     # so we pipe to zcat
5414     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5415         PVE::Tools::shellquote("--to-command=$tocmd");
5416
5417     my $tmpdir = "/var/tmp/vzdumptmp$$";
5418     mkpath $tmpdir;
5419
5420     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5421     local $ENV{VZDUMP_VMID} = $vmid;
5422     local $ENV{VZDUMP_USER} = $user;
5423
5424     my $conffile = config_file($vmid);
5425     my $tmpfn = "$conffile.$$.tmp";
5426
5427     # disable interrupts (always do cleanups)
5428     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5429         print STDERR "got interrupt - ignored\n";
5430     };
5431
5432     eval {
5433         # enable interrupts
5434         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5435             die "interrupted by signal\n";
5436         };
5437
5438         if ($archive eq '-') {
5439             print "extracting archive from STDIN\n";
5440             run_command($cmd, input => "<&STDIN");
5441         } else {
5442             print "extracting archive '$archive'\n";
5443             run_command($cmd);
5444         }
5445
5446         return if $opts->{info};
5447
5448         # read new mapping
5449         my $map = {};
5450         my $statfile = "$tmpdir/qmrestore.stat";
5451         if (my $fd = IO::File->new($statfile, "r")) {
5452             while (defined (my $line = <$fd>)) {
5453                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5454                     $map->{$1} = $2 if $1;
5455                 } else {
5456                     print STDERR "unable to parse line in statfile - $line\n";
5457                 }
5458             }
5459             $fd->close();
5460         }
5461
5462         my $confsrc = "$tmpdir/qemu-server.conf";
5463
5464         my $srcfd = new IO::File($confsrc, "r") ||
5465             die "unable to open file '$confsrc'\n";
5466
5467         my $outfd = new IO::File ($tmpfn, "w") ||
5468             die "unable to write config for VM $vmid\n";
5469
5470         my $cookie = { netcount => 0 };
5471         while (defined (my $line = <$srcfd>)) {
5472             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5473         }
5474
5475         $srcfd->close();
5476         $outfd->close();
5477     };
5478     my $err = $@;
5479
5480     if ($err) {
5481
5482         unlink $tmpfn;
5483
5484         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5485
5486         die $err;
5487     }
5488
5489     rmtree $tmpdir;
5490
5491     rename $tmpfn, $conffile ||
5492         die "unable to commit configuration file '$conffile'\n";
5493
5494     PVE::Cluster::cfs_update(); # make sure we read new file
5495
5496     eval { rescan($vmid, 1); };
5497     warn $@ if $@;
5498 };
5499
5500
5501 # Internal snapshots
5502
5503 # NOTE: Snapshot create/delete involves several non-atomic
5504 # action, and can take a long time.
5505 # So we try to avoid locking the file and use 'lock' variable
5506 # inside the config file instead.
5507
5508 my $snapshot_copy_config = sub {
5509     my ($source, $dest) = @_;
5510
5511     foreach my $k (keys %$source) {
5512         next if $k eq 'snapshots';
5513         next if $k eq 'snapstate';
5514         next if $k eq 'snaptime';
5515         next if $k eq 'vmstate';
5516         next if $k eq 'lock';
5517         next if $k eq 'digest';
5518         next if $k eq 'description';
5519         next if $k =~ m/^unused\d+$/;
5520
5521         $dest->{$k} = $source->{$k};
5522     }
5523 };
5524
5525 my $snapshot_apply_config = sub {
5526     my ($conf, $snap) = @_;
5527
5528     # copy snapshot list
5529     my $newconf = {
5530         snapshots => $conf->{snapshots},
5531     };
5532
5533     # keep description and list of unused disks
5534     foreach my $k (keys %$conf) {
5535         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5536         $newconf->{$k} = $conf->{$k};
5537     }
5538
5539     &$snapshot_copy_config($snap, $newconf);
5540
5541     return $newconf;
5542 };
5543
5544 sub foreach_writable_storage {
5545     my ($conf, $func) = @_;
5546
5547     my $sidhash = {};
5548
5549     foreach my $ds (keys %$conf) {
5550         next if !valid_drivename($ds);
5551
5552         my $drive = parse_drive($ds, $conf->{$ds});
5553         next if !$drive;
5554         next if drive_is_cdrom($drive);
5555
5556         my $volid = $drive->{file};
5557
5558         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5559         $sidhash->{$sid} = $sid if $sid;
5560     }
5561
5562     foreach my $sid (sort keys %$sidhash) {
5563         &$func($sid);
5564     }
5565 }
5566
5567 my $alloc_vmstate_volid = sub {
5568     my ($storecfg, $vmid, $conf, $snapname) = @_;
5569
5570     # Note: we try to be smart when selecting a $target storage
5571
5572     my $target;
5573
5574     # search shared storage first
5575     foreach_writable_storage($conf, sub {
5576         my ($sid) = @_;
5577         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5578         return if !$scfg->{shared};
5579
5580         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5581     });
5582
5583     if (!$target) {
5584         # now search local storage
5585         foreach_writable_storage($conf, sub {
5586             my ($sid) = @_;
5587             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5588             return if $scfg->{shared};
5589
5590             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5591         });
5592     }
5593
5594     $target = 'local' if !$target;
5595
5596     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5597     # we abort live save after $conf->{memory}, so we need at max twice that space
5598     my $size = $conf->{memory}*2 + $driver_state_size;
5599
5600     my $name = "vm-$vmid-state-$snapname";
5601     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5602     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5603     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5604
5605     return $volid;
5606 };
5607
5608 my $snapshot_prepare = sub {
5609     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5610
5611     my $snap;
5612
5613     my $updatefn =  sub {
5614
5615         my $conf = load_config($vmid);
5616
5617         die "you can't take a snapshot if it's a template\n"
5618             if is_template($conf);
5619
5620         check_lock($conf);
5621
5622         $conf->{lock} = 'snapshot';
5623
5624         die "snapshot name '$snapname' already used\n"
5625             if defined($conf->{snapshots}->{$snapname});
5626
5627         my $storecfg = PVE::Storage::config();
5628         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5629
5630         $snap = $conf->{snapshots}->{$snapname} = {};
5631
5632         if ($save_vmstate && check_running($vmid)) {
5633             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5634         }
5635
5636         &$snapshot_copy_config($conf, $snap);
5637
5638         $snap->{snapstate} = "prepare";
5639         $snap->{snaptime} = time();
5640         $snap->{description} = $comment if $comment;
5641
5642         # always overwrite machine if we save vmstate. This makes sure we
5643         # can restore it later using correct machine type
5644         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5645
5646         update_config_nolock($vmid, $conf, 1);
5647     };
5648
5649     lock_config($vmid, $updatefn);
5650
5651     return $snap;
5652 };
5653
5654 my $snapshot_commit = sub {
5655     my ($vmid, $snapname) = @_;
5656
5657     my $updatefn = sub {
5658
5659         my $conf = load_config($vmid);
5660
5661         die "missing snapshot lock\n"
5662             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5663
5664         my $has_machine_config = defined($conf->{machine});
5665
5666         my $snap = $conf->{snapshots}->{$snapname};
5667
5668         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5669
5670         die "wrong snapshot state\n"
5671             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5672
5673         delete $snap->{snapstate};
5674         delete $conf->{lock};
5675
5676         my $newconf = &$snapshot_apply_config($conf, $snap);
5677
5678         delete $newconf->{machine} if !$has_machine_config;
5679
5680         $newconf->{parent} = $snapname;
5681
5682         update_config_nolock($vmid, $newconf, 1);
5683     };
5684
5685     lock_config($vmid, $updatefn);
5686 };
5687
5688 sub snapshot_rollback {
5689     my ($vmid, $snapname) = @_;
5690
5691     my $prepare = 1;
5692
5693     my $storecfg = PVE::Storage::config();
5694
5695     my $conf = load_config($vmid);
5696
5697     my $get_snapshot_config = sub {
5698
5699         die "you can't rollback if vm is a template\n" if is_template($conf);
5700
5701         my $res = $conf->{snapshots}->{$snapname};
5702
5703         die "snapshot '$snapname' does not exist\n" if !defined($res);
5704
5705         return $res;
5706     };
5707
5708     my $snap = &$get_snapshot_config();
5709
5710     foreach_drive($snap, sub {
5711         my ($ds, $drive) = @_;
5712
5713         return if drive_is_cdrom($drive);
5714
5715         my $volid = $drive->{file};
5716
5717         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5718     });
5719
5720     my $updatefn = sub {
5721
5722         $conf = load_config($vmid);
5723
5724         $snap = &$get_snapshot_config();
5725
5726         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5727             if $snap->{snapstate};
5728
5729         if ($prepare) {
5730             check_lock($conf);
5731             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5732         }
5733
5734         die "unable to rollback vm $vmid: vm is running\n"
5735             if check_running($vmid);
5736
5737         if ($prepare) {
5738             $conf->{lock} = 'rollback';
5739         } else {
5740             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5741             delete $conf->{lock};
5742         }
5743
5744         my $forcemachine;
5745
5746         if (!$prepare) {
5747             my $has_machine_config = defined($conf->{machine});
5748
5749             # copy snapshot config to current config
5750             $conf = &$snapshot_apply_config($conf, $snap);
5751             $conf->{parent} = $snapname;
5752
5753             # Note: old code did not store 'machine', so we try to be smart
5754             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5755             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5756             # we remove the 'machine' configuration if not explicitly specified
5757             # in the original config.
5758             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5759         }
5760
5761         update_config_nolock($vmid, $conf, 1);
5762
5763         if (!$prepare && $snap->{vmstate}) {
5764             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5765             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5766         }
5767     };
5768
5769     lock_config($vmid, $updatefn);
5770
5771     foreach_drive($snap, sub {
5772         my ($ds, $drive) = @_;
5773
5774         return if drive_is_cdrom($drive);
5775
5776         my $volid = $drive->{file};
5777         my $device = "drive-$ds";
5778
5779         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5780     });
5781
5782     $prepare = 0;
5783     lock_config($vmid, $updatefn);
5784 }
5785
5786 my $savevm_wait = sub {
5787     my ($vmid) = @_;
5788
5789     for(;;) {
5790         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5791         if (!$stat->{status}) {
5792             die "savevm not active\n";
5793         } elsif ($stat->{status} eq 'active') {
5794             sleep(1);
5795             next;
5796         } elsif ($stat->{status} eq 'completed') {
5797             last;
5798         } else {
5799             die "query-savevm returned status '$stat->{status}'\n";
5800         }
5801     }
5802 };
5803
5804 sub snapshot_create {
5805     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5806
5807     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5808
5809     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5810
5811     my $config = load_config($vmid);
5812
5813     my $running = check_running($vmid);
5814
5815     my $freezefs = $running && $config->{agent};
5816     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5817
5818     my $drivehash = {};
5819
5820     if ($freezefs) {
5821         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5822         warn "guest-fsfreeze-freeze problems - $@" if $@;
5823     }
5824
5825     eval {
5826         # create internal snapshots of all drives
5827
5828         my $storecfg = PVE::Storage::config();
5829
5830         if ($running) {
5831             if ($snap->{vmstate}) {
5832                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5833                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5834                 &$savevm_wait($vmid);
5835             } else {
5836                 vm_mon_cmd($vmid, "savevm-start");
5837             }
5838         };
5839
5840         foreach_drive($snap, sub {
5841             my ($ds, $drive) = @_;
5842
5843             return if drive_is_cdrom($drive);
5844
5845             my $volid = $drive->{file};
5846             my $device = "drive-$ds";
5847
5848             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5849             $drivehash->{$ds} = 1;
5850        });
5851     };
5852     my $err = $@;
5853
5854     if ($running) {
5855         eval { vm_mon_cmd($vmid, "savevm-end")  };
5856         warn $@ if $@;
5857
5858         if ($freezefs) {
5859             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5860             warn "guest-fsfreeze-thaw problems - $@" if $@;
5861         }
5862
5863         # savevm-end is async, we need to wait
5864         for (;;) {
5865             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5866             if (!$stat->{bytes}) {
5867                 last;
5868             } else {
5869                 print "savevm not yet finished\n";
5870                 sleep(1);
5871                 next;
5872             }
5873         }
5874     }
5875
5876     if ($err) {
5877         warn "snapshot create failed: starting cleanup\n";
5878         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5879         warn $@ if $@;
5880         die $err;
5881     }
5882
5883     &$snapshot_commit($vmid, $snapname);
5884 }
5885
5886 # Note: $drivehash is only set when called from snapshot_create.
5887 sub snapshot_delete {
5888     my ($vmid, $snapname, $force, $drivehash) = @_;
5889
5890     my $prepare = 1;
5891
5892     my $snap;
5893     my $unused = [];
5894
5895     my $unlink_parent = sub {
5896         my ($confref, $new_parent) = @_;
5897
5898         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5899             if ($new_parent) {
5900                 $confref->{parent} = $new_parent;
5901             } else {
5902                 delete $confref->{parent};
5903             }
5904         }
5905     };
5906
5907     my $updatefn =  sub {
5908         my ($remove_drive) = @_;
5909
5910         my $conf = load_config($vmid);
5911
5912         if (!$drivehash) {
5913             check_lock($conf);
5914             die "you can't delete a snapshot if vm is a template\n"
5915                 if is_template($conf);
5916         }
5917
5918         $snap = $conf->{snapshots}->{$snapname};
5919
5920         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5921
5922         # remove parent refs
5923         if (!$prepare) {
5924             &$unlink_parent($conf, $snap->{parent});
5925             foreach my $sn (keys %{$conf->{snapshots}}) {
5926                 next if $sn eq $snapname;
5927                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5928             }
5929         }
5930
5931         if ($remove_drive) {
5932             if ($remove_drive eq 'vmstate') {
5933                 delete $snap->{$remove_drive};
5934             } else {
5935                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5936                 my $volid = $drive->{file};
5937                 delete $snap->{$remove_drive};
5938                 add_unused_volume($conf, $volid);
5939             }
5940         }
5941
5942         if ($prepare) {
5943             $snap->{snapstate} = 'delete';
5944         } else {
5945             delete $conf->{snapshots}->{$snapname};
5946             delete $conf->{lock} if $drivehash;
5947             foreach my $volid (@$unused) {
5948                 add_unused_volume($conf, $volid);
5949             }
5950         }
5951
5952         update_config_nolock($vmid, $conf, 1);
5953     };
5954
5955     lock_config($vmid, $updatefn);
5956
5957     # now remove vmstate file
5958
5959     my $storecfg = PVE::Storage::config();
5960
5961     if ($snap->{vmstate}) {
5962         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5963         if (my $err = $@) {
5964             die $err if !$force;
5965             warn $err;
5966         }
5967         # save changes (remove vmstate from snapshot)
5968         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5969     };
5970
5971     # now remove all internal snapshots
5972     foreach_drive($snap, sub {
5973         my ($ds, $drive) = @_;
5974
5975         return if drive_is_cdrom($drive);
5976
5977         my $volid = $drive->{file};
5978         my $device = "drive-$ds";
5979
5980         if (!$drivehash || $drivehash->{$ds}) {
5981             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5982             if (my $err = $@) {
5983                 die $err if !$force;
5984                 warn $err;
5985             }
5986         }
5987
5988         # save changes (remove drive fron snapshot)
5989         lock_config($vmid, $updatefn, $ds) if !$force;
5990         push @$unused, $volid;
5991     });
5992
5993     # now cleanup config
5994     $prepare = 0;
5995     lock_config($vmid, $updatefn);
5996 }
5997
5998 sub has_feature {
5999     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6000
6001     my $err;
6002     foreach_drive($conf, sub {
6003         my ($ds, $drive) = @_;
6004
6005         return if drive_is_cdrom($drive);
6006         my $volid = $drive->{file};
6007         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6008     });
6009
6010     return $err ? 0 : 1;
6011 }
6012
6013 sub template_create {
6014     my ($vmid, $conf, $disk) = @_;
6015
6016     my $storecfg = PVE::Storage::config();
6017
6018     foreach_drive($conf, sub {
6019         my ($ds, $drive) = @_;
6020
6021         return if drive_is_cdrom($drive);
6022         return if $disk && $ds ne $disk;
6023
6024         my $volid = $drive->{file};
6025         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6026
6027         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6028         $drive->{file} = $voliddst;
6029         $conf->{$ds} = print_drive($vmid, $drive);
6030         update_config_nolock($vmid, $conf, 1);
6031     });
6032 }
6033
6034 sub is_template {
6035     my ($conf) = @_;
6036
6037     return 1 if defined $conf->{template} && $conf->{template} == 1;
6038 }
6039
6040 sub qemu_img_convert {
6041     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6042
6043     my $storecfg = PVE::Storage::config();
6044     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6045     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6046
6047     if ($src_storeid && $dst_storeid) {
6048         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6049         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6050
6051         my $src_format = qemu_img_format($src_scfg, $src_volname);
6052         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6053
6054         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6055         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6056
6057         my $cmd = [];
6058         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6059         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6060         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6061
6062         my $parser = sub {
6063             my $line = shift;
6064             if($line =~ m/\((\S+)\/100\%\)/){
6065                 my $percent = $1;
6066                 my $transferred = int($size * $percent / 100);
6067                 my $remaining = $size - $transferred;
6068
6069                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6070             }
6071
6072         };
6073
6074         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6075         my $err = $@;
6076         die "copy failed: $err" if $err;
6077     }
6078 }
6079
6080 sub qemu_img_format {
6081     my ($scfg, $volname) = @_;
6082
6083     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6084         return $1;
6085     } elsif ($scfg->{type} eq 'iscsi') {
6086         return "host_device";
6087     } else {
6088         return "raw";
6089     }
6090 }
6091
6092 sub qemu_drive_mirror {
6093     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6094
6095     my $count = 0;
6096     my $old_len = 0;
6097     my $frozen = undef;
6098     my $maxwait = 120;
6099
6100     my $storecfg = PVE::Storage::config();
6101     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6102
6103     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6104
6105     my $format;
6106     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6107         $format = $1;
6108     }
6109
6110     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6111
6112     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6113     $opts->{format} = $format if $format;
6114
6115     #fixme : sometime drive-mirror timeout, but works fine after.
6116     # (I have see the problem with big volume > 200GB), so we need to eval
6117     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6118     # ignore errors here
6119
6120     eval {
6121         while (1) {
6122             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6123             my $stat = @$stats[0];
6124             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6125             die "error job is not mirroring" if $stat->{type} ne "mirror";
6126
6127             my $busy = $stat->{busy};
6128
6129             if (my $total = $stat->{len}) {
6130                 my $transferred = $stat->{offset} || 0;
6131                 my $remaining = $total - $transferred;
6132                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6133
6134                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy\n";
6135             }
6136
6137             if ($stat->{len} == $stat->{offset}) {
6138                 if ($busy eq 'false') {
6139
6140                     last if $vmiddst != $vmid;
6141
6142                     # try to switch the disk if source and destination are on the same guest
6143                     eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6144                     last if !$@;
6145                     die $@ if $@ !~ m/cannot be completed/;
6146                 }
6147
6148                 if ($count > $maxwait) {
6149                     # if too much writes to disk occurs at the end of migration
6150                     #the disk needs to be freezed to be able to complete the migration
6151                     vm_suspend($vmid,1);
6152                     $frozen = 1;
6153                 }
6154                 $count ++
6155             }
6156             $old_len = $stat->{offset};
6157             sleep 1;
6158         }
6159
6160         vm_resume($vmid, 1) if $frozen;
6161
6162     };
6163     my $err = $@;
6164
6165     my $cancel_job = sub {
6166         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6167         while (1) {
6168             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6169             my $stat = @$stats[0];
6170             last if !$stat;
6171             sleep 1;
6172         }
6173     };
6174
6175     if ($err) {
6176         eval { &$cancel_job(); };
6177         die "mirroring error: $err";
6178     }
6179
6180     if ($vmiddst != $vmid) {
6181         # if we clone a disk for a new target vm, we don't switch the disk
6182         &$cancel_job(); # so we call block-job-cancel
6183     }
6184 }
6185
6186 sub clone_disk {
6187     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6188         $newvmid, $storage, $format, $full, $newvollist) = @_;
6189
6190     my $newvolid;
6191
6192     if (!$full) {
6193         print "create linked clone of drive $drivename ($drive->{file})\n";
6194         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6195         push @$newvollist, $newvolid;
6196     } else {
6197         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6198         $storeid = $storage if $storage;
6199
6200         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6201         if (!$format) {
6202             $format = $drive->{format} || $defFormat;
6203         }
6204
6205         # test if requested format is supported - else use default
6206         my $supported = grep { $_ eq $format } @$validFormats;
6207         $format = $defFormat if !$supported;
6208
6209         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6210
6211         print "create full clone of drive $drivename ($drive->{file})\n";
6212         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6213         push @$newvollist, $newvolid;
6214
6215         if (!$running || $snapname) {
6216             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6217         } else {
6218             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6219         }
6220     }
6221
6222     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6223
6224     my $disk = $drive;
6225     $disk->{format} = undef;
6226     $disk->{file} = $newvolid;
6227     $disk->{size} = $size;
6228
6229     return $disk;
6230 }
6231
6232 # this only works if VM is running
6233 sub get_current_qemu_machine {
6234     my ($vmid) = @_;
6235
6236     my $cmd = { execute => 'query-machines', arguments => {} };
6237     my $res = vm_qmp_command($vmid, $cmd);
6238
6239     my ($current, $default);
6240     foreach my $e (@$res) {
6241         $default = $e->{name} if $e->{'is-default'};
6242         $current = $e->{name} if $e->{'is-current'};
6243     }
6244
6245     # fallback to the default machine if current is not supported by qemu
6246     return $current || $default || 'pc';
6247 }
6248
6249 sub qemu_machine_feature_enabled {
6250     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6251
6252     my $current_major;
6253     my $current_minor;
6254
6255     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6256
6257         $current_major = $3;
6258         $current_minor = $4;
6259
6260     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6261
6262         $current_major = $1;
6263         $current_minor = $2;
6264     }
6265
6266     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6267
6268
6269 }
6270
6271 sub lspci {
6272
6273     my $devices = {};
6274
6275     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6276             my (undef, $id, $function) = @_;
6277             my $res = { id => $id, function => $function};
6278             push @{$devices->{$id}}, $res;
6279     });
6280
6281     return $devices;
6282 }
6283
6284 sub vm_iothreads_list {
6285     my ($vmid) = @_;
6286
6287     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6288
6289     my $iothreads = {};
6290     foreach my $iothread (@$res) {
6291         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6292     }
6293
6294     return $iothreads;
6295 }
6296
6297 sub scsihw_infos {
6298     my ($conf, $drive) = @_;
6299
6300     my $maxdev = 0;
6301
6302     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6303         $maxdev = 7;
6304     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6305         $maxdev = 1;
6306     } else {
6307         $maxdev = 256;
6308     }
6309
6310     my $controller = int($drive->{index} / $maxdev);
6311     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6312
6313     return ($maxdev, $controller, $controller_prefix);
6314 }
6315
6316 1;