PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     onboot => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Specifies whether a VM will be started during system bootup.",
 159         default => 0,
 160     },
 161     autostart => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Automatic restart after crash (currently ignored).",
 165         default => 0,
 166     },
 167     hotplug => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Allow hotplug for disk and network device",
 171         default => 0,
 172     },
 173     reboot => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 177         default => 1,
 178     },
 179     lock => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Lock/unlock the VM.",
 183         enum => [qw(migrate backup snapshot rollback)],
 184     },
 185     cpulimit => {
 186         optional => 1,
 187         type => 'integer',
 188         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 189         minimum => 0,
 190         default => 0,
 191     },
 192     cpuunits => {
 193         optional => 1,
 194         type => 'integer',
 195         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 196         minimum => 0,
 197         maximum => 500000,
 198         default => 1000,
 199     },
 200     memory => {
 201         optional => 1,
 202         type => 'integer',
 203         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 204         minimum => 16,
 205         default => 512,
 206     },
 207     balloon => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 211         minimum => 0,
 212     },
 213     shares => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 217         minimum => 0,
 218         maximum => 50000,
 219         default => 1000,
 220     },
 221     keyboard => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 225         enum => PVE::Tools::kvmkeymaplist(),
 226         default => 'en-us',
 227     },
 228     name => {
 229         optional => 1,
 230         type => 'string', format => 'dns-name',
 231         description => "Set a name for the VM. Only used on the configuration web interface.",
 232     },
 233     scsihw => {
 234         optional => 1,
 235         type => 'string',
 236         description => "scsi controller model",
 237         enum => [qw(lsi lsi53c810 virtio-scsi-pci megasas pvscsi)],
 238         default => 'lsi',
 239     },
 240     description => {
 241         optional => 1,
 242         type => 'string',
 243         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 244     },
 245     ostype => {
 246         optional => 1,
 247         type => 'string',
 248         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 249         description => <<EODESC,
 250 Used to enable special optimization/features for specific
 251 operating systems:
 252
 253 other  => unspecified OS
 254 wxp    => Microsoft Windows XP
 255 w2k    => Microsoft Windows 2000
 256 w2k3   => Microsoft Windows 2003
 257 w2k8   => Microsoft Windows 2008
 258 wvista => Microsoft Windows Vista
 259 win7   => Microsoft Windows 7
 260 win8   => Microsoft Windows 8/2012
 261 l24    => Linux 2.4 Kernel
 262 l26    => Linux 2.6/3.X Kernel
 263 solaris => solaris/opensolaris/openindiania kernel
 264
 265 other|l24|l26|solaris                       ... no special behaviour
 266 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 267 EODESC
 268     },
 269     boot => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 273         pattern => '[acdn]{1,4}',
 274         default => 'cdn',
 275     },
 276     bootdisk => {
 277         optional => 1,
 278         type => 'string', format => 'pve-qm-bootdisk',
 279         description => "Enable booting from specified disk.",
 280         pattern => '(ide|sata|scsi|virtio)\d+',
 281     },
 282     smp => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "The number of CPUs. Please use option -sockets instead.",
 286         minimum => 1,
 287         default => 1,
 288     },
 289     sockets => {
 290         optional => 1,
 291         type => 'integer',
 292         description => "The number of CPU sockets.",
 293         minimum => 1,
 294         default => 1,
 295     },
 296     cores => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of cores per socket.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     maxcpus => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "Maximum cpus for hotplug.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     acpi => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable ACPI.",
 314         default => 1,
 315     },
 316     agent => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable Qemu GuestAgent.",
 320         default => 0,
 321     },
 322     kvm => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable KVM hardware virtualization.",
 326         default => 1,
 327     },
 328     tdf => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable time drift fix.",
 332         default => 0,
 333     },
 334     localtime => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 338     },
 339     freeze => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 343     },
 344     vga => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 348         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 349     },
 350     watchdog => {
 351         optional => 1,
 352         type => 'string', format => 'pve-qm-watchdog',
 353         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 354         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 355     },
 356     startdate => {
 357         optional => 1,
 358         type => 'string',
 359         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 360         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 361         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 362         default => 'now',
 363     },
 364     startup => {
 365         optional => 1,
 366         type => 'string', format => 'pve-qm-startup',
 367         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 368         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 369     },
 370     template => {
 371         optional => 1,
 372         type => 'boolean',
 373         description => "Enable/disable Template.",
 374         default => 0,
 375     },
 376     args => {
 377         optional => 1,
 378         type => 'string',
 379         description => <<EODESCR,
 380 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 381
 382 args: -no-reboot -no-hpet
 383 EODESCR
 384     },
 385     tablet => {
 386         optional => 1,
 387         type => 'boolean',
 388         default => 1,
 389         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 390     },
 391     migrate_speed => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 395         minimum => 0,
 396         default => 0,
 397     },
 398     migrate_downtime => {
 399         optional => 1,
 400         type => 'number',
 401         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 402         minimum => 0,
 403         default => 0.1,
 404     },
 405     cdrom => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-drive',
 408         typetext => 'volume',
 409         description => "This is an alias for option -ide2",
 410     },
 411     cpu => {
 412         optional => 1,
 413         description => "Emulated CPU type.",
 414         type => 'string',
 415         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge Haswell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 416         default => 'kvm64',
 417     },
 418     parent => get_standard_option('pve-snapshot-name', {
 419         optional => 1,
 420         description => "Parent snapshot name. This is used internally, and should not be modified.",
 421     }),
 422     snaptime => {
 423         optional => 1,
 424         description => "Timestamp for snapshots.",
 425         type => 'integer',
 426         minimum => 0,
 427     },
 428     vmstate => {
 429         optional => 1,
 430         type => 'string', format => 'pve-volume-id',
 431         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 432     },
 433     machine => {
 434         description => "Specific the Qemu machine type.",
 435         type => 'string',
 436         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 437         maxLength => 40,
 438         optional => 1,
 439     },
 440     smbios1 => {
 441         description => "Specify SMBIOS type 1 fields.",
 442         type => 'string', format => 'pve-qm-smbios1',
 443         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 444         maxLength => 256,
 445         optional => 1,
 446     },
 447 };
 448
 449 # what about other qemu settings ?
 450 #cpu => 'string',
 451 #machine => 'string',
 452 #fda => 'file',
 453 #fdb => 'file',
 454 #mtdblock => 'file',
 455 #sd => 'file',
 456 #pflash => 'file',
 457 #snapshot => 'bool',
 458 #bootp => 'file',
 459 ##tftp => 'dir',
 460 ##smb => 'dir',
 461 #kernel => 'file',
 462 #append => 'string',
 463 #initrd => 'file',
 464 ##soundhw => 'string',
 465
 466 while (my ($k, $v) = each %$confdesc) {
 467     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 468 }
 469
 470 my $MAX_IDE_DISKS = 4;
 471 my $MAX_SCSI_DISKS = 14;
 472 my $MAX_VIRTIO_DISKS = 16;
 473 my $MAX_SATA_DISKS = 6;
 474 my $MAX_USB_DEVICES = 5;
 475 my $MAX_NETS = 32;
 476 my $MAX_UNUSED_DISKS = 8;
 477 my $MAX_HOSTPCI_DEVICES = 2;
 478 my $MAX_SERIAL_PORTS = 4;
 479 my $MAX_PARALLEL_PORTS = 3;
 480
 481 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 482                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3'];
 483 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 484
 485 my $netdesc = {
 486     optional => 1,
 487     type => 'string', format => 'pve-qm-net',
 488     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>][,tag=<vlanid>][,firewall=0|1]",
 489     description => <<EODESCR,
 490 Specify network devices.
 491
 492 MODEL is one of: $nic_model_list_txt
 493
 494 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 495 automatically generated if not specified.
 496
 497 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 498
 499 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 500
 501 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 502
 503 10.0.2.2   Gateway
 504 10.0.2.3   DNS Server
 505 10.0.2.4   SMB Server
 506
 507 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 508
 509 EODESCR
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 512
 513 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 514     $confdesc->{"net$i"} = $netdesc;
 515 }
 516
 517 my $drivename_hash;
 518
 519 my $idedesc = {
 520     optional => 1,
 521     type => 'string', format => 'pve-qm-drive',
 522     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 523     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 524 };
 525 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 526
 527 my $scsidesc = {
 528     optional => 1,
 529     type => 'string', format => 'pve-qm-drive',
 530     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 531     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 532 };
 533 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 534
 535 my $satadesc = {
 536     optional => 1,
 537     type => 'string', format => 'pve-qm-drive',
 538     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 539     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 540 };
 541 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 542
 543 my $virtiodesc = {
 544     optional => 1,
 545     type => 'string', format => 'pve-qm-drive',
 546     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 547     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 548 };
 549 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 550
 551 my $usbdesc = {
 552     optional => 1,
 553     type => 'string', format => 'pve-qm-usb-device',
 554     typetext => 'host=HOSTUSBDEVICE|spice',
 555     description => <<EODESCR,
 556 Configure an USB device (n is 0 to 4). This can be used to
 557 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 558
 559 'bus-port(.port)*' (decimal numbers) or
 560 'vendor_id:product_id' (hexadeciaml numbers)
 561
 562 You can use the 'lsusb -t' command to list existing usb devices.
 563
 564 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 565
 566 The value 'spice' can be used to add a usb redirection devices for spice.
 567
 568 EODESCR
 569 };
 570 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 571
 572 my $hostpcidesc = {
 573         optional => 1,
 574         type => 'string', format => 'pve-qm-hostpci',
 575         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 576         description => <<EODESCR,
 577 Map host pci devices. HOSTPCIDEVICE syntax is:
 578
 579 'bus:dev.func' (hexadecimal numbers)
 580
 581 You can us the 'lspci' command to list existing pci devices.
 582
 583 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 584
 585 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 586
 587 Experimental: user reported problems with this option.
 588 EODESCR
 589 };
 590 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 591
 592 my $serialdesc = {
 593         optional => 1,
 594         type => 'string',
 595         pattern => '(/dev/ttyS\d+|socket)',
 596         description =>  <<EODESCR,
 597 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device, or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 598
 599 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 600
 601 Experimental: user reported problems with this option.
 602 EODESCR
 603 };
 604
 605 my $paralleldesc= {
 606         optional => 1,
 607         type => 'string',
 608         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 609         description =>  <<EODESCR,
 610 Map host parallel devices (n is 0 to 2).
 611
 612 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 613
 614 Experimental: user reported problems with this option.
 615 EODESCR
 616 };
 617
 618 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 619     $confdesc->{"parallel$i"} = $paralleldesc;
 620 }
 621
 622 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 623     $confdesc->{"serial$i"} = $serialdesc;
 624 }
 625
 626 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 627     $confdesc->{"hostpci$i"} = $hostpcidesc;
 628 }
 629
 630 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 631     $drivename_hash->{"ide$i"} = 1;
 632     $confdesc->{"ide$i"} = $idedesc;
 633 }
 634
 635 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 636     $drivename_hash->{"sata$i"} = 1;
 637     $confdesc->{"sata$i"} = $satadesc;
 638 }
 639
 640 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 641     $drivename_hash->{"scsi$i"} = 1;
 642     $confdesc->{"scsi$i"} = $scsidesc ;
 643 }
 644
 645 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 646     $drivename_hash->{"virtio$i"} = 1;
 647     $confdesc->{"virtio$i"} = $virtiodesc;
 648 }
 649
 650 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 651     $confdesc->{"usb$i"} = $usbdesc;
 652 }
 653
 654 my $unuseddesc = {
 655     optional => 1,
 656     type => 'string', format => 'pve-volume-id',
 657     description => "Reference to unused volumes.",
 658 };
 659
 660 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 661     $confdesc->{"unused$i"} = $unuseddesc;
 662 }
 663
 664 my $kvm_api_version = 0;
 665
 666 sub kvm_version {
 667
 668     return $kvm_api_version if $kvm_api_version;
 669
 670     my $fh = IO::File->new("</dev/kvm") ||
 671         return 0;
 672
 673     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 674         $kvm_api_version = $v;
 675     }
 676
 677     $fh->close();
 678
 679     return  $kvm_api_version;
 680 }
 681
 682 my $kvm_user_version;
 683
 684 sub kvm_user_version {
 685
 686     return $kvm_user_version if $kvm_user_version;
 687
 688     $kvm_user_version = 'unknown';
 689
 690     my $tmp = `kvm -help 2>/dev/null`;
 691
 692     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 693         $kvm_user_version = $2;
 694     }
 695
 696     return $kvm_user_version;
 697
 698 }
 699
 700 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 701
 702 sub disknames {
 703     # order is important - used to autoselect boot disk
 704     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 705             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 706             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 707             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 708 }
 709
 710 sub valid_drivename {
 711     my $dev = shift;
 712
 713     return defined($drivename_hash->{$dev});
 714 }
 715
 716 sub option_exists {
 717     my $key = shift;
 718     return defined($confdesc->{$key});
 719 }
 720
 721 sub nic_models {
 722     return $nic_model_list;
 723 }
 724
 725 sub os_list_description {
 726
 727     return {
 728         other => 'Other',
 729         wxp => 'Windows XP',
 730         w2k => 'Windows 2000',
 731         w2k3 =>, 'Windows 2003',
 732         w2k8 => 'Windows 2008',
 733         wvista => 'Windows Vista',
 734         win7 => 'Windows 7',
 735         win8 => 'Windows 8/2012',
 736         l24 => 'Linux 2.4',
 737         l26 => 'Linux 2.6',
 738     };
 739 }
 740
 741 my $cdrom_path;
 742
 743 sub get_cdrom_path {
 744
 745     return  $cdrom_path if $cdrom_path;
 746
 747     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 748     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 749     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 750 }
 751
 752 sub get_iso_path {
 753     my ($storecfg, $vmid, $cdrom) = @_;
 754
 755     if ($cdrom eq 'cdrom') {
 756         return get_cdrom_path();
 757     } elsif ($cdrom eq 'none') {
 758         return '';
 759     } elsif ($cdrom =~ m|^/|) {
 760         return $cdrom;
 761     } else {
 762         return PVE::Storage::path($storecfg, $cdrom);
 763     }
 764 }
 765
 766 # try to convert old style file names to volume IDs
 767 sub filename_to_volume_id {
 768     my ($vmid, $file, $media) = @_;
 769
 770     if (!($file eq 'none' || $file eq 'cdrom' ||
 771           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 772
 773         return undef if $file =~ m|/|;
 774
 775         if ($media && $media eq 'cdrom') {
 776             $file = "local:iso/$file";
 777         } else {
 778             $file = "local:$vmid/$file";
 779         }
 780     }
 781
 782     return $file;
 783 }
 784
 785 sub verify_media_type {
 786     my ($opt, $vtype, $media) = @_;
 787
 788     return if !$media;
 789
 790     my $etype;
 791     if ($media eq 'disk') {
 792         $etype = 'images';
 793     } elsif ($media eq 'cdrom') {
 794         $etype = 'iso';
 795     } else {
 796         die "internal error";
 797     }
 798
 799     return if ($vtype eq $etype);
 800
 801     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 802 }
 803
 804 sub cleanup_drive_path {
 805     my ($opt, $storecfg, $drive) = @_;
 806
 807     # try to convert filesystem paths to volume IDs
 808
 809     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 810         ($drive->{file} !~ m|^/dev/.+|) &&
 811         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 812         ($drive->{file} !~ m/^\d+$/)) {
 813         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 814         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 815         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 816         verify_media_type($opt, $vtype, $drive->{media});
 817         $drive->{file} = $volid;
 818     }
 819
 820     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 821 }
 822
 823 sub create_conf_nolock {
 824     my ($vmid, $settings) = @_;
 825
 826     my $filename = config_file($vmid);
 827
 828     die "configuration file '$filename' already exists\n" if -f $filename;
 829
 830     my $defaults = load_defaults();
 831
 832     $settings->{name} = "vm$vmid" if !$settings->{name};
 833     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 834
 835     my $data = '';
 836     foreach my $opt (keys %$settings) {
 837         next if !$confdesc->{$opt};
 838
 839         my $value = $settings->{$opt};
 840         next if !$value;
 841
 842         $data .= "$opt: $value\n";
 843     }
 844
 845     PVE::Tools::file_set_contents($filename, $data);
 846 }
 847
 848 my $parse_size = sub {
 849     my ($value) = @_;
 850
 851     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 852     my ($size, $unit) = ($1, $3);
 853     if ($unit) {
 854         if ($unit eq 'K') {
 855             $size = $size * 1024;
 856         } elsif ($unit eq 'M') {
 857             $size = $size * 1024 * 1024;
 858         } elsif ($unit eq 'G') {
 859             $size = $size * 1024 * 1024 * 1024;
 860         }
 861     }
 862     return int($size);
 863 };
 864
 865 my $format_size = sub {
 866     my ($size) = @_;
 867
 868     $size = int($size);
 869
 870     my $kb = int($size/1024);
 871     return $size if $kb*1024 != $size;
 872
 873     my $mb = int($kb/1024);
 874     return "${kb}K" if $mb*1024 != $kb;
 875
 876     my $gb = int($mb/1024);
 877     return "${mb}M" if $gb*1024 != $mb;
 878
 879     return "${gb}G";
 880 };
 881
 882 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 883 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 884 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 885 #        [,aio=native|threads][,discard=ignore|on]
 886
 887 sub parse_drive {
 888     my ($key, $data) = @_;
 889
 890     my $res = {};
 891
 892     # $key may be undefined - used to verify JSON parameters
 893     if (!defined($key)) {
 894         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 895         $res->{index} = 0;
 896     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 897         $res->{interface} = $1;
 898         $res->{index} = $2;
 899     } else {
 900         return undef;
 901     }
 902
 903     foreach my $p (split (/,/, $data)) {
 904         next if $p =~ m/^\s*$/;
 905
 906         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard)=(.+)$/) {
 907             my ($k, $v) = ($1, $2);
 908
 909             $k = 'file' if $k eq 'volume';
 910
 911             return undef if defined $res->{$k};
 912
 913             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 914                 return undef if !$v || $v !~ m/^\d+/;
 915                 $k = "m$k";
 916                 $v = sprintf("%.3f", $v / (1024*1024));
 917             }
 918             $res->{$k} = $v;
 919         } else {
 920             if (!$res->{file} && $p !~ m/=/) {
 921                 $res->{file} = $p;
 922             } else {
 923                 return undef;
 924             }
 925         }
 926     }
 927
 928     return undef if !$res->{file};
 929
 930     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 931         $res->{format} = $1;
 932     }
 933
 934     return undef if $res->{cache} &&
 935         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 936     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 937     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 938     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 939     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 940     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 941     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 942     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 943     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 944     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 945     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 946     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 947     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 948
 949     return undef if $res->{mbps_rd} && $res->{mbps};
 950     return undef if $res->{mbps_wr} && $res->{mbps};
 951
 952     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 957     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 958
 959     return undef if $res->{iops_rd} && $res->{iops};
 960     return undef if $res->{iops_wr} && $res->{iops};
 961
 962
 963     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 964     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 966     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 968     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 969
 970
 971     if ($res->{size}) {
 972         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 973     }
 974
 975     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 976         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 977         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 978         return undef if $res->{interface} eq 'virtio';
 979     }
 980
 981     # rerror does not work with scsi drives
 982     if ($res->{rerror}) {
 983         return undef if $res->{interface} eq 'scsi';
 984     }
 985
 986     return $res;
 987 }
 988
 989 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 990
 991 sub print_drive {
 992     my ($vmid, $drive) = @_;
 993
 994     my $opts = '';
 995     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup') {
 996         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 997     }
 998
 999     if ($drive->{size}) {
1000         $opts .= ",size=" . &$format_size($drive->{size});
1001     }
1002
1003     return "$drive->{file}$opts";
1004 }
1005
1006 sub scsi_inquiry {
1007     my($fh, $noerr) = @_;
1008
1009     my $SG_IO = 0x2285;
1010     my $SG_GET_VERSION_NUM = 0x2282;
1011
1012     my $versionbuf = "\x00" x 8;
1013     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1014     if (!$ret) {
1015         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1016         return undef;
1017     }
1018     my $version = unpack("I", $versionbuf);
1019     if ($version < 30000) {
1020         die "scsi generic interface too old\n"  if !$noerr;
1021         return undef;
1022     }
1023
1024     my $buf = "\x00" x 36;
1025     my $sensebuf = "\x00" x 8;
1026     my $cmd = pack("C x3 C x1", 0x12, 36);
1027
1028     # see /usr/include/scsi/sg.h
1029     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1030
1031     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1032                       length($sensebuf), 0, length($buf), $buf,
1033                       $cmd, $sensebuf, 6000);
1034
1035     $ret = ioctl($fh, $SG_IO, $packet);
1036     if (!$ret) {
1037         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my @res = unpack($sg_io_hdr_t, $packet);
1042     if ($res[17] || $res[18]) {
1043         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1044         return undef;
1045     }
1046
1047     my $res = {};
1048     (my $byte0, my $byte1, $res->{vendor},
1049      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1050
1051     $res->{removable} = $byte1 & 128 ? 1 : 0;
1052     $res->{type} = $byte0 & 31;
1053
1054     return $res;
1055 }
1056
1057 sub path_is_scsi {
1058     my ($path) = @_;
1059
1060     my $fh = IO::File->new("+<$path") || return undef;
1061     my $res = scsi_inquiry($fh, 1);
1062     close($fh);
1063
1064     return $res;
1065 }
1066
1067 sub machine_type_is_q35 {
1068     my ($conf) = @_;
1069
1070     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1071 }
1072
1073 sub print_tabletdevice_full {
1074     my ($conf) = @_;
1075
1076     my $q35 = machine_type_is_q35($conf);
1077
1078     # we use uhci for old VMs because tablet driver was buggy in older qemu
1079     my $usbbus = $q35 ? "ehci" : "uhci";
1080
1081     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1082 }
1083
1084 sub print_drivedevice_full {
1085     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1086
1087     my $device = '';
1088     my $maxdev = 0;
1089
1090     if ($drive->{interface} eq 'virtio') {
1091         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1092         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1093     } elsif ($drive->{interface} eq 'scsi') {
1094         $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
1095         my $controller = int($drive->{index} / $maxdev);
1096         my $unit = $drive->{index} % $maxdev;
1097         my $devicetype = 'hd';
1098         my $path = '';
1099         if (drive_is_cdrom($drive)) {
1100               $devicetype = 'cd';
1101         } else {
1102               if ($drive->{file} =~ m|^/|) {
1103                   $path = $drive->{file};
1104               } else {
1105                   $path = PVE::Storage::path($storecfg, $drive->{file});
1106               }
1107
1108               if($path =~ m/^iscsi\:\/\//){
1109                   $devicetype = 'generic';
1110               } else {
1111                   if (my $info = path_is_scsi($path)) {
1112                       if ($info->{type} == 0) {
1113                           $devicetype = 'block';
1114                       } elsif ($info->{type} == 1) { # tape
1115                           $devicetype = 'generic';
1116                       }
1117                   }
1118               }
1119          }
1120
1121         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1122             $device = "scsi-$devicetype,bus=scsihw$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123         } else {
1124             $device = "scsi-$devicetype,bus=scsihw$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1125         }
1126
1127     } elsif ($drive->{interface} eq 'ide'){
1128         $maxdev = 2;
1129         my $controller = int($drive->{index} / $maxdev);
1130         my $unit = $drive->{index} % $maxdev;
1131         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1132
1133         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1134     } elsif ($drive->{interface} eq 'sata'){
1135         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1136         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1137         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1138     } elsif ($drive->{interface} eq 'usb') {
1139         die "implement me";
1140         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1141     } else {
1142         die "unsupported interface type";
1143     }
1144
1145     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1146
1147     return $device;
1148 }
1149
1150 sub get_initiator_name {
1151     my $initiator;
1152
1153     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1154     while (defined(my $line = <$fh>)) {
1155         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1156         $initiator = $1;
1157         last;
1158     }
1159     $fh->close();
1160
1161     return $initiator;
1162 }
1163
1164 sub print_drive_full {
1165     my ($storecfg, $vmid, $drive) = @_;
1166
1167     my $opts = '';
1168     foreach my $o (@qemu_drive_options) {
1169         next if $o eq 'bootindex';
1170         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1171     }
1172
1173     foreach my $o (qw(bps bps_rd bps_wr)) {
1174         my $v = $drive->{"m$o"};
1175         $opts .= ",$o=" . int($v*1024*1024) if $v;
1176     }
1177
1178     # use linux-aio by default (qemu default is threads)
1179     $opts .= ",aio=native" if !$drive->{aio};
1180
1181     my $path;
1182     my $volid = $drive->{file};
1183     if (drive_is_cdrom($drive)) {
1184         $path = get_iso_path($storecfg, $vmid, $volid);
1185     } else {
1186         if ($volid =~ m|^/|) {
1187             $path = $volid;
1188         } else {
1189             $path = PVE::Storage::path($storecfg, $volid);
1190         }
1191     }
1192
1193     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1194
1195     my $pathinfo = $path ? "file=$path," : '';
1196
1197     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1198 }
1199
1200 sub print_netdevice_full {
1201     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1202
1203     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1204
1205     my $device = $net->{model};
1206     if ($net->{model} eq 'virtio') {
1207          $device = 'virtio-net-pci';
1208      };
1209
1210     # qemu > 0.15 always try to boot from network - we disable that by
1211     # not loading the pxe rom file
1212     my $extra = ($bootorder !~ m/n/) ? "romfile=," : '';
1213     my $pciaddr = print_pci_addr("$netid", $bridges);
1214     my $tmpstr = "$device,${extra}mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1215     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1216         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1217         my $vectors = $net->{queues} * 2 + 2;
1218         $tmpstr .= ",vectors=$vectors,mq=on";
1219     }
1220     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1221     return $tmpstr;
1222 }
1223
1224 sub print_netdev_full {
1225     my ($vmid, $conf, $net, $netid) = @_;
1226
1227     my $i = '';
1228     if ($netid =~ m/^net(\d+)$/) {
1229         $i = int($1);
1230     }
1231
1232     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1233
1234     my $ifname = "tap${vmid}i$i";
1235
1236     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1237     die "interface name '$ifname' is too long (max 15 character)\n"
1238         if length($ifname) >= 16;
1239
1240     my $vhostparam = '';
1241     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1242
1243     my $vmname = $conf->{name} || "vm$vmid";
1244
1245     my $netdev = "";
1246
1247     if ($net->{bridge}) {
1248         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1249     } else {
1250         $netdev = "type=user,id=$netid,hostname=$vmname";
1251     }
1252
1253     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1254
1255     return $netdev;
1256 }
1257
1258 sub drive_is_cdrom {
1259     my ($drive) = @_;
1260
1261     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1262
1263 }
1264
1265 sub parse_hostpci {
1266     my ($value) = @_;
1267
1268     return undef if !$value;
1269
1270
1271     my @list = split(/,/, $value);
1272     my $found;
1273
1274     my $res = {};
1275     foreach my $kv (@list) {
1276
1277         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1278             $found = 1;
1279             if(defined($4)){
1280                 push @{$res->{pciid}}, { id => $2 , function => $4};
1281
1282             }else{
1283                 my $pcidevices = lspci($2);
1284                 $res->{pciid} = $pcidevices->{$2};
1285             }
1286         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1287             $res->{driver} = $1;
1288         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1289             $res->{rombar} = $1;
1290         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1291             $res->{'x-vga'} = $1;
1292         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1293             $res->{pcie} = 1 if $1 == 1;
1294         } else {
1295             warn "unknown hostpci setting '$kv'\n";
1296         }
1297     }
1298
1299     return undef if !$found;
1300
1301     return $res;
1302 }
1303
1304 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1305 sub parse_net {
1306     my ($data) = @_;
1307
1308     my $res = {};
1309
1310     foreach my $kvp (split(/,/, $data)) {
1311
1312         if ($kvp =~ m/^(ne2k_pci|e1000|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1313             my $model = lc($1);
1314             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1315             $res->{model} = $model;
1316             $res->{macaddr} = $mac;
1317         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1318             $res->{bridge} = $1;
1319         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1320             $res->{queues} = $1;
1321         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1322             $res->{rate} = $1;
1323         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1324             $res->{tag} = $1;
1325         } elsif ($kvp =~ m/^firewall=(\d+)$/) {
1326             $res->{firewall} = $1;
1327         } else {
1328             return undef;
1329         }
1330
1331     }
1332
1333     return undef if !$res->{model};
1334
1335     return $res;
1336 }
1337
1338 sub print_net {
1339     my $net = shift;
1340
1341     my $res = "$net->{model}";
1342     $res .= "=$net->{macaddr}" if $net->{macaddr};
1343     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1344     $res .= ",rate=$net->{rate}" if $net->{rate};
1345     $res .= ",tag=$net->{tag}" if $net->{tag};
1346     $res .= ",firewall=$net->{firewall}" if $net->{firewall};
1347
1348     return $res;
1349 }
1350
1351 sub add_random_macs {
1352     my ($settings) = @_;
1353
1354     foreach my $opt (keys %$settings) {
1355         next if $opt !~ m/^net(\d+)$/;
1356         my $net = parse_net($settings->{$opt});
1357         next if !$net;
1358         $settings->{$opt} = print_net($net);
1359     }
1360 }
1361
1362 sub add_unused_volume {
1363     my ($config, $volid) = @_;
1364
1365     my $key;
1366     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1367         my $test = "unused$ind";
1368         if (my $vid = $config->{$test}) {
1369             return if $vid eq $volid; # do not add duplicates
1370         } else {
1371             $key = $test;
1372         }
1373     }
1374
1375     die "To many unused volume - please delete them first.\n" if !$key;
1376
1377     $config->{$key} = $volid;
1378
1379     return $key;
1380 }
1381
1382 my $valid_smbios1_options = {
1383     manufacturer => '\S+',
1384     product => '\S+',
1385     version => '\S+',
1386     serial => '\S+',
1387     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1388     sku => '\S+',
1389     family => '\S+',
1390 };
1391
1392 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1393 sub parse_smbios1 {
1394     my ($data) = @_;
1395
1396     my $res = {};
1397
1398     foreach my $kvp (split(/,/, $data)) {
1399         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1400         my ($k, $v) = split(/=/, $kvp);
1401         return undef if !defined($k) || !defined($v);
1402         return undef if !$valid_smbios1_options->{$k};
1403         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1404         $res->{$k} = $v;
1405     }
1406
1407     return $res;
1408 }
1409
1410 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1411 sub verify_smbios1 {
1412     my ($value, $noerr) = @_;
1413
1414     return $value if parse_smbios1($value);
1415
1416     return undef if $noerr;
1417
1418     die "unable to parse smbios (type 1) options\n";
1419 }
1420
1421 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1422 sub verify_bootdisk {
1423     my ($value, $noerr) = @_;
1424
1425     return $value if valid_drivename($value);
1426
1427     return undef if $noerr;
1428
1429     die "invalid boot disk '$value'\n";
1430 }
1431
1432 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1433 sub verify_net {
1434     my ($value, $noerr) = @_;
1435
1436     return $value if parse_net($value);
1437
1438     return undef if $noerr;
1439
1440     die "unable to parse network options\n";
1441 }
1442
1443 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1444 sub verify_drive {
1445     my ($value, $noerr) = @_;
1446
1447     return $value if parse_drive(undef, $value);
1448
1449     return undef if $noerr;
1450
1451     die "unable to parse drive options\n";
1452 }
1453
1454 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1455 sub verify_hostpci {
1456     my ($value, $noerr) = @_;
1457
1458     return $value if parse_hostpci($value);
1459
1460     return undef if $noerr;
1461
1462     die "unable to parse pci id\n";
1463 }
1464
1465 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1466 sub verify_watchdog {
1467     my ($value, $noerr) = @_;
1468
1469     return $value if parse_watchdog($value);
1470
1471     return undef if $noerr;
1472
1473     die "unable to parse watchdog options\n";
1474 }
1475
1476 sub parse_watchdog {
1477     my ($value) = @_;
1478
1479     return undef if !$value;
1480
1481     my $res = {};
1482
1483     foreach my $p (split(/,/, $value)) {
1484         next if $p =~ m/^\s*$/;
1485
1486         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1487             $res->{model} = $2;
1488         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1489             $res->{action} = $2;
1490         } else {
1491             return undef;
1492         }
1493     }
1494
1495     return $res;
1496 }
1497
1498 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1499 sub verify_startup {
1500     my ($value, $noerr) = @_;
1501
1502     return $value if parse_startup($value);
1503
1504     return undef if $noerr;
1505
1506     die "unable to parse startup options\n";
1507 }
1508
1509 sub parse_startup {
1510     my ($value) = @_;
1511
1512     return undef if !$value;
1513
1514     my $res = {};
1515
1516     foreach my $p (split(/,/, $value)) {
1517         next if $p =~ m/^\s*$/;
1518
1519         if ($p =~ m/^(order=)?(\d+)$/) {
1520             $res->{order} = $2;
1521         } elsif ($p =~ m/^up=(\d+)$/) {
1522             $res->{up} = $1;
1523         } elsif ($p =~ m/^down=(\d+)$/) {
1524             $res->{down} = $1;
1525         } else {
1526             return undef;
1527         }
1528     }
1529
1530     return $res;
1531 }
1532
1533 sub parse_usb_device {
1534     my ($value) = @_;
1535
1536     return undef if !$value;
1537
1538     my @dl = split(/,/, $value);
1539     my $found;
1540
1541     my $res = {};
1542     foreach my $v (@dl) {
1543         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1544             $found = 1;
1545             $res->{vendorid} = $2;
1546             $res->{productid} = $4;
1547         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1548             $found = 1;
1549             $res->{hostbus} = $1;
1550             $res->{hostport} = $2;
1551         } elsif ($v =~ m/^spice$/) {
1552             $found = 1;
1553             $res->{spice} = 1;
1554         } else {
1555             return undef;
1556         }
1557     }
1558     return undef if !$found;
1559
1560     return $res;
1561 }
1562
1563 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1564 sub verify_usb_device {
1565     my ($value, $noerr) = @_;
1566
1567     return $value if parse_usb_device($value);
1568
1569     return undef if $noerr;
1570
1571     die "unable to parse usb device\n";
1572 }
1573
1574 # add JSON properties for create and set function
1575 sub json_config_properties {
1576     my $prop = shift;
1577
1578     foreach my $opt (keys %$confdesc) {
1579         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1580         $prop->{$opt} = $confdesc->{$opt};
1581     }
1582
1583     return $prop;
1584 }
1585
1586 sub check_type {
1587     my ($key, $value) = @_;
1588
1589     die "unknown setting '$key'\n" if !$confdesc->{$key};
1590
1591     my $type = $confdesc->{$key}->{type};
1592
1593     if (!defined($value)) {
1594         die "got undefined value\n";
1595     }
1596
1597     if ($value =~ m/[\n\r]/) {
1598         die "property contains a line feed\n";
1599     }
1600
1601     if ($type eq 'boolean') {
1602         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1603         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1604         die "type check ('boolean') failed - got '$value'\n";
1605     } elsif ($type eq 'integer') {
1606         return int($1) if $value =~ m/^(\d+)$/;
1607         die "type check ('integer') failed - got '$value'\n";
1608     } elsif ($type eq 'number') {
1609         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1610         die "type check ('number') failed - got '$value'\n";
1611     } elsif ($type eq 'string') {
1612         if (my $fmt = $confdesc->{$key}->{format}) {
1613             if ($fmt eq 'pve-qm-drive') {
1614                 # special case - we need to pass $key to parse_drive()
1615                 my $drive = parse_drive($key, $value);
1616                 return $value if $drive;
1617                 die "unable to parse drive options\n";
1618             }
1619             PVE::JSONSchema::check_format($fmt, $value);
1620             return $value;
1621         }
1622         $value =~ s/^\"(.*)\"$/$1/;
1623         return $value;
1624     } else {
1625         die "internal error"
1626     }
1627 }
1628
1629 sub lock_config_full {
1630     my ($vmid, $timeout, $code, @param) = @_;
1631
1632     my $filename = config_file_lock($vmid);
1633
1634     my $res = lock_file($filename, $timeout, $code, @param);
1635
1636     die $@ if $@;
1637
1638     return $res;
1639 }
1640
1641 sub lock_config_mode {
1642     my ($vmid, $timeout, $shared, $code, @param) = @_;
1643
1644     my $filename = config_file_lock($vmid);
1645
1646     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1647
1648     die $@ if $@;
1649
1650     return $res;
1651 }
1652
1653 sub lock_config {
1654     my ($vmid, $code, @param) = @_;
1655
1656     return lock_config_full($vmid, 10, $code, @param);
1657 }
1658
1659 sub cfs_config_path {
1660     my ($vmid, $node) = @_;
1661
1662     $node = $nodename if !$node;
1663     return "nodes/$node/qemu-server/$vmid.conf";
1664 }
1665
1666 sub check_iommu_support{
1667     #fixme : need to check IOMMU support
1668     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1669
1670     my $iommu=1;
1671     return $iommu;
1672
1673 }
1674
1675 sub config_file {
1676     my ($vmid, $node) = @_;
1677
1678     my $cfspath = cfs_config_path($vmid, $node);
1679     return "/etc/pve/$cfspath";
1680 }
1681
1682 sub config_file_lock {
1683     my ($vmid) = @_;
1684
1685     return "$lock_dir/lock-$vmid.conf";
1686 }
1687
1688 sub touch_config {
1689     my ($vmid) = @_;
1690
1691     my $conf = config_file($vmid);
1692     utime undef, undef, $conf;
1693 }
1694
1695 sub destroy_vm {
1696     my ($storecfg, $vmid, $keep_empty_config) = @_;
1697
1698     my $conffile = config_file($vmid);
1699
1700     my $conf = load_config($vmid);
1701
1702     check_lock($conf);
1703
1704     # only remove disks owned by this VM
1705     foreach_drive($conf, sub {
1706         my ($ds, $drive) = @_;
1707
1708         return if drive_is_cdrom($drive);
1709
1710         my $volid = $drive->{file};
1711
1712         return if !$volid || $volid =~ m|^/|;
1713
1714         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1715         return if !$path || !$owner || ($owner != $vmid);
1716
1717         PVE::Storage::vdisk_free($storecfg, $volid);
1718     });
1719
1720     if ($keep_empty_config) {
1721         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1722     } else {
1723         unlink $conffile;
1724     }
1725
1726     # also remove unused disk
1727     eval {
1728         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1729
1730         eval {
1731             PVE::Storage::foreach_volid($dl, sub {
1732                 my ($volid, $sid, $volname, $d) = @_;
1733                 PVE::Storage::vdisk_free($storecfg, $volid);
1734             });
1735         };
1736         warn $@ if $@;
1737
1738     };
1739     warn $@ if $@;
1740 }
1741
1742 sub load_config {
1743     my ($vmid, $node) = @_;
1744
1745     my $cfspath = cfs_config_path($vmid, $node);
1746
1747     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1748
1749     die "no such VM ('$vmid')\n" if !defined($conf);
1750
1751     return $conf;
1752 }
1753
1754 sub parse_vm_config {
1755     my ($filename, $raw) = @_;
1756
1757     return undef if !defined($raw);
1758
1759     my $res = {
1760         digest => Digest::SHA::sha1_hex($raw),
1761         snapshots => {},
1762     };
1763
1764     $filename =~ m|/qemu-server/(\d+)\.conf$|
1765         || die "got strange filename '$filename'";
1766
1767     my $vmid = $1;
1768
1769     my $conf = $res;
1770     my $descr = '';
1771
1772     my @lines = split(/\n/, $raw);
1773     foreach my $line (@lines) {
1774         next if $line =~ m/^\s*$/;
1775
1776         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1777             my $snapname = $1;
1778             $conf->{description} = $descr if $descr;
1779             $descr = '';
1780             $conf = $res->{snapshots}->{$snapname} = {};
1781             next;
1782         }
1783
1784         if ($line =~ m/^\#(.*)\s*$/) {
1785             $descr .= PVE::Tools::decode_text($1) . "\n";
1786             next;
1787         }
1788
1789         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1790             $descr .= PVE::Tools::decode_text($2);
1791         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1792             $conf->{snapstate} = $1;
1793         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1794             my $key = $1;
1795             my $value = $2;
1796             $conf->{$key} = $value;
1797         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1798             my $key = $1;
1799             my $value = $2;
1800             eval { $value = check_type($key, $value); };
1801             if ($@) {
1802                 warn "vm $vmid - unable to parse value of '$key' - $@";
1803             } else {
1804                 my $fmt = $confdesc->{$key}->{format};
1805                 if ($fmt && $fmt eq 'pve-qm-drive') {
1806                     my $v = parse_drive($key, $value);
1807                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1808                         $v->{file} = $volid;
1809                         $value = print_drive($vmid, $v);
1810                     } else {
1811                         warn "vm $vmid - unable to parse value of '$key'\n";
1812                         next;
1813                     }
1814                 }
1815
1816                 if ($key eq 'cdrom') {
1817                     $conf->{ide2} = $value;
1818                 } else {
1819                     $conf->{$key} = $value;
1820                 }
1821             }
1822         }
1823     }
1824
1825     $conf->{description} = $descr if $descr;
1826
1827     delete $res->{snapstate}; # just to be sure
1828
1829     return $res;
1830 }
1831
1832 sub write_vm_config {
1833     my ($filename, $conf) = @_;
1834
1835     delete $conf->{snapstate}; # just to be sure
1836
1837     if ($conf->{cdrom}) {
1838         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1839         $conf->{ide2} = $conf->{cdrom};
1840         delete $conf->{cdrom};
1841     }
1842
1843     # we do not use 'smp' any longer
1844     if ($conf->{sockets}) {
1845         delete $conf->{smp};
1846     } elsif ($conf->{smp}) {
1847         $conf->{sockets} = $conf->{smp};
1848         delete $conf->{cores};
1849         delete $conf->{smp};
1850     }
1851
1852     if ($conf->{maxcpus} && $conf->{sockets}) {
1853         delete $conf->{sockets};
1854     }
1855
1856     my $used_volids = {};
1857
1858     my $cleanup_config = sub {
1859         my ($cref, $snapname) = @_;
1860
1861         foreach my $key (keys %$cref) {
1862             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1863                 $key eq 'snapstate';
1864             my $value = $cref->{$key};
1865             eval { $value = check_type($key, $value); };
1866             die "unable to parse value of '$key' - $@" if $@;
1867
1868             $cref->{$key} = $value;
1869
1870             if (!$snapname && valid_drivename($key)) {
1871                 my $drive = parse_drive($key, $value);
1872                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1873             }
1874         }
1875     };
1876
1877     &$cleanup_config($conf);
1878     foreach my $snapname (keys %{$conf->{snapshots}}) {
1879         &$cleanup_config($conf->{snapshots}->{$snapname}, $snapname);
1880     }
1881
1882     # remove 'unusedX' settings if we re-add a volume
1883     foreach my $key (keys %$conf) {
1884         my $value = $conf->{$key};
1885         if ($key =~ m/^unused/ && $used_volids->{$value}) {
1886             delete $conf->{$key};
1887         }
1888     }
1889
1890     my $generate_raw_config = sub {
1891         my ($conf) = @_;
1892
1893         my $raw = '';
1894
1895         # add description as comment to top of file
1896         my $descr = $conf->{description} || '';
1897         foreach my $cl (split(/\n/, $descr)) {
1898             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
1899         }
1900
1901         foreach my $key (sort keys %$conf) {
1902             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots';
1903             $raw .= "$key: $conf->{$key}\n";
1904         }
1905         return $raw;
1906     };
1907
1908     my $raw = &$generate_raw_config($conf);
1909     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1910         $raw .= "\n[$snapname]\n";
1911         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1912     }
1913
1914     return $raw;
1915 }
1916
1917 sub update_config_nolock {
1918     my ($vmid, $conf, $skiplock) = @_;
1919
1920     check_lock($conf) if !$skiplock;
1921
1922     my $cfspath = cfs_config_path($vmid);
1923
1924     PVE::Cluster::cfs_write_file($cfspath, $conf);
1925 }
1926
1927 sub update_config {
1928     my ($vmid, $conf, $skiplock) = @_;
1929
1930     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
1931 }
1932
1933 sub load_defaults {
1934
1935     my $res = {};
1936
1937     # we use static defaults from our JSON schema configuration
1938     foreach my $key (keys %$confdesc) {
1939         if (defined(my $default = $confdesc->{$key}->{default})) {
1940             $res->{$key} = $default;
1941         }
1942     }
1943
1944     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
1945     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
1946
1947     return $res;
1948 }
1949
1950 sub config_list {
1951     my $vmlist = PVE::Cluster::get_vmlist();
1952     my $res = {};
1953     return $res if !$vmlist || !$vmlist->{ids};
1954     my $ids = $vmlist->{ids};
1955
1956     foreach my $vmid (keys %$ids) {
1957         my $d = $ids->{$vmid};
1958         next if !$d->{node} || $d->{node} ne $nodename;
1959         next if !$d->{type} || $d->{type} ne 'qemu';
1960         $res->{$vmid}->{exists} = 1;
1961     }
1962     return $res;
1963 }
1964
1965 # test if VM uses local resources (to prevent migration)
1966 sub check_local_resources {
1967     my ($conf, $noerr) = @_;
1968
1969     my $loc_res = 0;
1970
1971     $loc_res = 1 if $conf->{hostusb}; # old syntax
1972     $loc_res = 1 if $conf->{hostpci}; # old syntax
1973
1974     foreach my $k (keys %$conf) {
1975         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
1976         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
1977     }
1978
1979     die "VM uses local resources\n" if $loc_res && !$noerr;
1980
1981     return $loc_res;
1982 }
1983
1984 # check if used storages are available on all nodes (use by migrate)
1985 sub check_storage_availability {
1986     my ($storecfg, $conf, $node) = @_;
1987
1988     foreach_drive($conf, sub {
1989         my ($ds, $drive) = @_;
1990
1991         my $volid = $drive->{file};
1992         return if !$volid;
1993
1994         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1995         return if !$sid;
1996
1997         # check if storage is available on both nodes
1998         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
1999         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2000    });
2001 }
2002
2003 # list nodes where all VM images are available (used by has_feature API)
2004 sub shared_nodes {
2005     my ($conf, $storecfg) = @_;
2006
2007     my $nodelist = PVE::Cluster::get_nodelist();
2008     my $nodehash = { map { $_ => 1 } @$nodelist };
2009     my $nodename = PVE::INotify::nodename();
2010
2011     foreach_drive($conf, sub {
2012         my ($ds, $drive) = @_;
2013
2014         my $volid = $drive->{file};
2015         return if !$volid;
2016
2017         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2018         if ($storeid) {
2019             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2020             if ($scfg->{disable}) {
2021                 $nodehash = {};
2022             } elsif (my $avail = $scfg->{nodes}) {
2023                 foreach my $node (keys %$nodehash) {
2024                     delete $nodehash->{$node} if !$avail->{$node};
2025                 }
2026             } elsif (!$scfg->{shared}) {
2027                 foreach my $node (keys %$nodehash) {
2028                     delete $nodehash->{$node} if $node ne $nodename
2029                 }
2030             }
2031         }
2032     });
2033
2034     return $nodehash
2035 }
2036
2037 sub check_lock {
2038     my ($conf) = @_;
2039
2040     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2041 }
2042
2043 sub check_cmdline {
2044     my ($pidfile, $pid) = @_;
2045
2046     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2047     if (defined($fh)) {
2048         my $line = <$fh>;
2049         $fh->close;
2050         return undef if !$line;
2051         my @param = split(/\0/, $line);
2052
2053         my $cmd = $param[0];
2054         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2055
2056         for (my $i = 0; $i < scalar (@param); $i++) {
2057             my $p = $param[$i];
2058             next if !$p;
2059             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2060                 my $p = $param[$i+1];
2061                 return 1 if $p && ($p eq $pidfile);
2062                 return undef;
2063             }
2064         }
2065     }
2066     return undef;
2067 }
2068
2069 sub check_running {
2070     my ($vmid, $nocheck, $node) = @_;
2071
2072     my $filename = config_file($vmid, $node);
2073
2074     die "unable to find configuration file for VM $vmid - no such machine\n"
2075         if !$nocheck && ! -f $filename;
2076
2077     my $pidfile = pidfile_name($vmid);
2078
2079     if (my $fd = IO::File->new("<$pidfile")) {
2080         my $st = stat($fd);
2081         my $line = <$fd>;
2082         close($fd);
2083
2084         my $mtime = $st->mtime;
2085         if ($mtime > time()) {
2086             warn "file '$filename' modified in future\n";
2087         }
2088
2089         if ($line =~ m/^(\d+)$/) {
2090             my $pid = $1;
2091             if (check_cmdline($pidfile, $pid)) {
2092                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2093                     return $pid;
2094                 }
2095             }
2096         }
2097     }
2098
2099     return undef;
2100 }
2101
2102 sub vzlist {
2103
2104     my $vzlist = config_list();
2105
2106     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2107
2108     while (defined(my $de = $fd->read)) {
2109         next if $de !~ m/^(\d+)\.pid$/;
2110         my $vmid = $1;
2111         next if !defined($vzlist->{$vmid});
2112         if (my $pid = check_running($vmid)) {
2113             $vzlist->{$vmid}->{pid} = $pid;
2114         }
2115     }
2116
2117     return $vzlist;
2118 }
2119
2120 sub disksize {
2121     my ($storecfg, $conf) = @_;
2122
2123     my $bootdisk = $conf->{bootdisk};
2124     return undef if !$bootdisk;
2125     return undef if !valid_drivename($bootdisk);
2126
2127     return undef if !$conf->{$bootdisk};
2128
2129     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2130     return undef if !defined($drive);
2131
2132     return undef if drive_is_cdrom($drive);
2133
2134     my $volid = $drive->{file};
2135     return undef if !$volid;
2136
2137     return $drive->{size};
2138 }
2139
2140 my $last_proc_pid_stat;
2141
2142 # get VM status information
2143 # This must be fast and should not block ($full == false)
2144 # We only query KVM using QMP if $full == true (this can be slow)
2145 sub vmstatus {
2146     my ($opt_vmid, $full) = @_;
2147
2148     my $res = {};
2149
2150     my $storecfg = PVE::Storage::config();
2151
2152     my $list = vzlist();
2153     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2154
2155     my $cpucount = $cpuinfo->{cpus} || 1;
2156
2157     foreach my $vmid (keys %$list) {
2158         next if $opt_vmid && ($vmid ne $opt_vmid);
2159
2160         my $cfspath = cfs_config_path($vmid);
2161         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2162
2163         my $d = {};
2164         $d->{pid} = $list->{$vmid}->{pid};
2165
2166         # fixme: better status?
2167         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2168
2169         my $size = disksize($storecfg, $conf);
2170         if (defined($size)) {
2171             $d->{disk} = 0; # no info available
2172             $d->{maxdisk} = $size;
2173         } else {
2174             $d->{disk} = 0;
2175             $d->{maxdisk} = 0;
2176         }
2177
2178         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2179         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2180
2181         $d->{name} = $conf->{name} || "VM $vmid";
2182         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2183
2184         if ($conf->{balloon}) {
2185             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2186             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2187         }
2188
2189         $d->{uptime} = 0;
2190         $d->{cpu} = 0;
2191         $d->{mem} = 0;
2192
2193         $d->{netout} = 0;
2194         $d->{netin} = 0;
2195
2196         $d->{diskread} = 0;
2197         $d->{diskwrite} = 0;
2198
2199         $d->{template} = is_template($conf);
2200
2201         $res->{$vmid} = $d;
2202     }
2203
2204     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2205     foreach my $dev (keys %$netdev) {
2206         next if $dev !~ m/^tap([1-9]\d*)i/;
2207         my $vmid = $1;
2208         my $d = $res->{$vmid};
2209         next if !$d;
2210
2211         $d->{netout} += $netdev->{$dev}->{receive};
2212         $d->{netin} += $netdev->{$dev}->{transmit};
2213     }
2214
2215     my $ctime = gettimeofday;
2216
2217     foreach my $vmid (keys %$list) {
2218
2219         my $d = $res->{$vmid};
2220         my $pid = $d->{pid};
2221         next if !$pid;
2222
2223         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2224         next if !$pstat; # not running
2225
2226         my $used = $pstat->{utime} + $pstat->{stime};
2227
2228         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2229
2230         if ($pstat->{vsize}) {
2231             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2232         }
2233
2234         my $old = $last_proc_pid_stat->{$pid};
2235         if (!$old) {
2236             $last_proc_pid_stat->{$pid} = {
2237                 time => $ctime,
2238                 used => $used,
2239                 cpu => 0,
2240             };
2241             next;
2242         }
2243
2244         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2245
2246         if ($dtime > 1000) {
2247             my $dutime = $used -  $old->{used};
2248
2249             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2250             $last_proc_pid_stat->{$pid} = {
2251                 time => $ctime,
2252                 used => $used,
2253                 cpu => $d->{cpu},
2254             };
2255         } else {
2256             $d->{cpu} = $old->{cpu};
2257         }
2258     }
2259
2260     return $res if !$full;
2261
2262     my $qmpclient = PVE::QMPClient->new();
2263
2264     my $ballooncb = sub {
2265         my ($vmid, $resp) = @_;
2266
2267         my $info = $resp->{'return'};
2268         return if !$info->{max_mem};
2269
2270         my $d = $res->{$vmid};
2271
2272         # use memory assigned to VM
2273         $d->{maxmem} = $info->{max_mem};
2274         $d->{balloon} = $info->{actual};
2275
2276         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2277             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2278             $d->{freemem} = $info->{free_mem};
2279         }
2280
2281     };
2282
2283     my $blockstatscb = sub {
2284         my ($vmid, $resp) = @_;
2285         my $data = $resp->{'return'} || [];
2286         my $totalrdbytes = 0;
2287         my $totalwrbytes = 0;
2288         for my $blockstat (@$data) {
2289             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2290             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2291         }
2292         $res->{$vmid}->{diskread} = $totalrdbytes;
2293         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2294     };
2295
2296     my $statuscb = sub {
2297         my ($vmid, $resp) = @_;
2298
2299         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2300         # this fails if ballon driver is not loaded, so this must be
2301         # the last commnand (following command are aborted if this fails).
2302         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2303
2304         my $status = 'unknown';
2305         if (!defined($status = $resp->{'return'}->{status})) {
2306             warn "unable to get VM status\n";
2307             return;
2308         }
2309
2310         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2311     };
2312
2313     foreach my $vmid (keys %$list) {
2314         next if $opt_vmid && ($vmid ne $opt_vmid);
2315         next if !$res->{$vmid}->{pid}; # not running
2316         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2317     }
2318
2319     $qmpclient->queue_execute();
2320
2321     foreach my $vmid (keys %$list) {
2322         next if $opt_vmid && ($vmid ne $opt_vmid);
2323         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2324     }
2325
2326     return $res;
2327 }
2328
2329 sub foreach_drive {
2330     my ($conf, $func) = @_;
2331
2332     foreach my $ds (keys %$conf) {
2333         next if !valid_drivename($ds);
2334
2335         my $drive = parse_drive($ds, $conf->{$ds});
2336         next if !$drive;
2337
2338         &$func($ds, $drive);
2339     }
2340 }
2341
2342 sub foreach_volid {
2343     my ($conf, $func) = @_;
2344
2345     my $volhash = {};
2346
2347     my $test_volid = sub {
2348         my ($volid, $is_cdrom) = @_;
2349
2350         return if !$volid;
2351
2352         $volhash->{$volid} = $is_cdrom || 0;
2353     };
2354
2355     foreach_drive($conf, sub {
2356         my ($ds, $drive) = @_;
2357         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2358     });
2359
2360     foreach my $snapname (keys %{$conf->{snapshots}}) {
2361         my $snap = $conf->{snapshots}->{$snapname};
2362         &$test_volid($snap->{vmstate}, 0);
2363         foreach_drive($snap, sub {
2364             my ($ds, $drive) = @_;
2365             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2366         });
2367     }
2368
2369     foreach my $volid (keys %$volhash) {
2370         &$func($volid, $volhash->{$volid});
2371     }
2372 }
2373
2374 sub vga_conf_has_spice {
2375     my ($vga) = @_;
2376
2377     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2378
2379     return $1 || 1;
2380 }
2381
2382 sub config_to_command {
2383     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2384
2385     my $cmd = [];
2386     my $globalFlags = [];
2387     my $machineFlags = [];
2388     my $rtcFlags = [];
2389     my $cpuFlags = [];
2390     my $devices = [];
2391     my $pciaddr = '';
2392     my $bridges = {};
2393     my $kvmver = kvm_user_version();
2394     my $vernum = 0; # unknown
2395     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2396         $vernum = $1*1000000+$2*1000;
2397     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2398         $vernum = $1*1000000+$2*1000+$3;
2399     }
2400
2401     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2402
2403     my $have_ovz = -f '/proc/vz/vestat';
2404
2405     my $q35 = machine_type_is_q35($conf);
2406
2407     push @$cmd, '/usr/bin/kvm';
2408
2409     push @$cmd, '-id', $vmid;
2410
2411     my $use_virtio = 0;
2412
2413     my $qmpsocket = qmp_socket($vmid);
2414     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2415     push @$cmd, '-mon', "chardev=qmp,mode=control";
2416
2417     my $socket = vnc_socket($vmid);
2418     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2419
2420     push @$cmd, '-pidfile' , pidfile_name($vmid);
2421
2422     push @$cmd, '-daemonize';
2423
2424     if ($conf->{smbios1}) {
2425         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2426     }
2427
2428     if ($q35) {
2429         # the q35 chipset support native usb2, so we enable usb controller
2430         # by default for this machine type
2431         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2432     } else {
2433         $pciaddr = print_pci_addr("piix3", $bridges);
2434         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2435
2436         my $use_usb2 = 0;
2437         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2438             next if !$conf->{"usb$i"};
2439             $use_usb2 = 1;
2440         }
2441         # include usb device config
2442         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2443     }
2444
2445     my $vga = $conf->{vga};
2446
2447     my $qxlnum = vga_conf_has_spice($vga);
2448     $vga = 'qxl' if $qxlnum;
2449
2450     if (!$vga) {
2451         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2452                                 $conf->{ostype} eq 'win7' ||
2453                                 $conf->{ostype} eq 'w2k8')) {
2454             $vga = 'std';
2455         } else {
2456             $vga = 'cirrus';
2457         }
2458     }
2459
2460     # enable absolute mouse coordinates (needed by vnc)
2461     my $tablet;
2462     if (defined($conf->{tablet})) {
2463         $tablet = $conf->{tablet};
2464     } else {
2465         $tablet = $defaults->{tablet};
2466         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2467         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2468     }
2469
2470     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2471
2472     # host pci devices
2473     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2474         my $d = parse_hostpci($conf->{"hostpci$i"});
2475         next if !$d;
2476
2477         my $pcie = $d->{pcie};
2478         if($pcie){
2479             die "q35 machine model is not enabled" if !$q35;
2480             $pciaddr = print_pcie_addr("hostpci$i");
2481         }else{
2482             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2483         }
2484
2485         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2486         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2487         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2488         $driver = "vfio-pci" if $xvga ne '';
2489         my $pcidevices = $d->{pciid};
2490         my $multifunction = 1 if @$pcidevices > 1;
2491
2492         my $j=0;
2493         foreach my $pcidevice (@$pcidevices) {
2494
2495             my $id = "hostpci$i";
2496             $id .= ".$j" if $multifunction;
2497             my $addr = $pciaddr;
2498             $addr .= ".$j" if $multifunction;
2499             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2500
2501             if($j == 0){
2502                 $devicestr .= "$rombar$xvga";
2503                 $devicestr .= ",multifunction=on" if $multifunction;
2504             }
2505
2506             push @$devices, '-device', $devicestr;
2507             $j++;
2508         }
2509     }
2510
2511     # usb devices
2512     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2513         my $d = parse_usb_device($conf->{"usb$i"});
2514         next if !$d;
2515         if ($d->{vendorid} && $d->{productid}) {
2516             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2517         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2518             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2519         } elsif ($d->{spice}) {
2520             # usb redir support for spice
2521             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2522             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2523         }
2524     }
2525
2526     # serial devices
2527     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2528         if (my $path = $conf->{"serial$i"}) {
2529             if ($path eq 'socket') {
2530                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2531                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2532                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2533             } else {
2534                 die "no such serial device\n" if ! -c $path;
2535                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2536                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2537             }
2538         }
2539     }
2540
2541     # parallel devices
2542     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2543         if (my $path = $conf->{"parallel$i"}) {
2544             die "no such parallel device\n" if ! -c $path;
2545             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2546             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2547             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2548         }
2549     }
2550
2551     my $vmname = $conf->{name} || "vm$vmid";
2552
2553     push @$cmd, '-name', $vmname;
2554
2555     my $sockets = 1;
2556     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2557     $sockets = $conf->{sockets} if  $conf->{sockets};
2558
2559     my $cores = $conf->{cores} || 1;
2560     my $maxcpus = $conf->{maxcpus} if $conf->{maxcpus};
2561
2562     if ($maxcpus) {
2563         push @$cmd, '-smp', "cpus=$cores,maxcpus=$maxcpus";
2564     } else {
2565         push @$cmd, '-smp', "sockets=$sockets,cores=$cores";
2566     }
2567
2568     push @$cmd, '-nodefaults';
2569
2570     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2571
2572     my $bootindex_hash = {};
2573     my $i = 1;
2574     foreach my $o (split(//, $bootorder)) {
2575         $bootindex_hash->{$o} = $i*100;
2576         $i++;
2577     }
2578
2579     push @$cmd, '-boot', "menu=on";
2580
2581     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2582
2583     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2584
2585     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2586
2587     # time drift fix
2588     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2589
2590     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2591     my $useLocaltime = $conf->{localtime};
2592
2593     if (my $ost = $conf->{ostype}) {
2594         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2595
2596         if ($ost =~ m/^w/) { # windows
2597             $useLocaltime = 1 if !defined($conf->{localtime});
2598
2599             # use time drift fix when acpi is enabled
2600             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2601                 $tdf = 1 if !defined($conf->{tdf});
2602             }
2603         }
2604
2605         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2606             $ost eq 'wvista') {
2607             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2608             push @$cmd, '-no-hpet';
2609             #push @$cpuFlags , 'hv_vapic" if !$nokvm;  #fixme, my win2008R2 hang at boot with this
2610             push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2611         }
2612
2613         if ($ost eq 'win7' || $ost eq 'win8') {
2614             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2615         }
2616     }
2617
2618     push @$rtcFlags, 'driftfix=slew' if $tdf;
2619
2620     if ($nokvm) {
2621         push @$machineFlags, 'accel=tcg';
2622     } else {
2623         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2624     }
2625
2626     my $machine_type = $forcemachine || $conf->{machine};
2627     if ($machine_type) {
2628         push @$machineFlags, "type=${machine_type}";
2629     }
2630
2631     if ($conf->{startdate}) {
2632         push @$rtcFlags, "base=$conf->{startdate}";
2633     } elsif ($useLocaltime) {
2634         push @$rtcFlags, 'base=localtime';
2635     }
2636
2637     my $cpu = $nokvm ? "qemu64" : "kvm64";
2638     $cpu = $conf->{cpu} if $conf->{cpu};
2639
2640     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2641
2642     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2643
2644     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2645
2646     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2647
2648     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2649
2650     # Note: enforce needs kernel 3.10, so we do not use it for now
2651     # push @$cmd, '-cpu', "$cpu,enforce";
2652     push @$cmd, '-cpu', $cpu;
2653
2654     push @$cmd, '-S' if $conf->{freeze};
2655
2656     # set keyboard layout
2657     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2658     push @$cmd, '-k', $kb if $kb;
2659
2660     # enable sound
2661     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2662     #push @$cmd, '-soundhw', 'es1370';
2663     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2664
2665     if($conf->{agent}) {
2666         my $qgasocket = qga_socket($vmid);
2667         my $pciaddr = print_pci_addr("qga0", $bridges);
2668         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2669         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2670         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2671     }
2672
2673     my $spice_port;
2674
2675     if ($qxlnum) {
2676         if ($qxlnum > 1) {
2677             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2678                 for(my $i = 1; $i < $qxlnum; $i++){
2679                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2680                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2681                 }
2682             } else {
2683                 # assume other OS works like Linux
2684                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2685                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2686             }
2687         }
2688
2689         my $pciaddr = print_pci_addr("spice", $bridges);
2690
2691         $spice_port = PVE::Tools::next_spice_port();
2692
2693         push @$cmd, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2694
2695         push @$cmd, '-device', "virtio-serial,id=spice$pciaddr";
2696         push @$cmd, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2697         push @$cmd, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2698     }
2699
2700     # enable balloon by default, unless explicitly disabled
2701     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2702         $pciaddr = print_pci_addr("balloon0", $bridges);
2703         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2704     }
2705
2706     if ($conf->{watchdog}) {
2707         my $wdopts = parse_watchdog($conf->{watchdog});
2708         $pciaddr = print_pci_addr("watchdog", $bridges);
2709         my $watchdog = $wdopts->{model} || 'i6300esb';
2710         push @$devices, '-device', "$watchdog$pciaddr";
2711         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
2712     }
2713
2714     my $vollist = [];
2715     my $scsicontroller = {};
2716     my $ahcicontroller = {};
2717     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
2718
2719     # Add iscsi initiator name if available
2720     if (my $initiator = get_initiator_name()) {
2721         push @$devices, '-iscsi', "initiator-name=$initiator";
2722     }
2723
2724     foreach_drive($conf, sub {
2725         my ($ds, $drive) = @_;
2726
2727         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
2728             push @$vollist, $drive->{file};
2729         }
2730
2731         $use_virtio = 1 if $ds =~ m/^virtio/;
2732
2733         if (drive_is_cdrom ($drive)) {
2734             if ($bootindex_hash->{d}) {
2735                 $drive->{bootindex} = $bootindex_hash->{d};
2736                 $bootindex_hash->{d} += 1;
2737             }
2738         } else {
2739             if ($bootindex_hash->{c}) {
2740                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
2741                 $bootindex_hash->{c} += 1;
2742             }
2743         }
2744
2745         if ($drive->{interface} eq 'scsi') {
2746
2747             my $maxdev = ($scsihw !~ m/^lsi/) ? 256 : 7;
2748             my $controller = int($drive->{index} / $maxdev);
2749             $pciaddr = print_pci_addr("scsihw$controller", $bridges);
2750             push @$devices, '-device', "$scsihw,id=scsihw$controller$pciaddr" if !$scsicontroller->{$controller};
2751             $scsicontroller->{$controller}=1;
2752         }
2753
2754         if ($drive->{interface} eq 'sata') {
2755            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
2756            $pciaddr = print_pci_addr("ahci$controller", $bridges);
2757            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
2758            $ahcicontroller->{$controller}=1;
2759         }
2760
2761         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
2762         push @$devices, '-drive',$drive_cmd;
2763         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
2764     });
2765
2766     push @$cmd, '-m', $conf->{memory} || $defaults->{memory};
2767
2768     for (my $i = 0; $i < $MAX_NETS; $i++) {
2769          next if !$conf->{"net$i"};
2770          my $d = parse_net($conf->{"net$i"});
2771          next if !$d;
2772
2773          $use_virtio = 1 if $d->{model} eq 'virtio';
2774
2775          if ($bootindex_hash->{n}) {
2776             $d->{bootindex} = $bootindex_hash->{n};
2777             $bootindex_hash->{n} += 1;
2778          }
2779
2780          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
2781          push @$devices, '-netdev', $netdevfull;
2782
2783          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
2784          push @$devices, '-device', $netdevicefull;
2785     }
2786
2787     if (!$q35) {
2788         # add pci bridges
2789         while (my ($k, $v) = each %$bridges) {
2790             $pciaddr = print_pci_addr("pci.$k");
2791             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
2792         }
2793     }
2794
2795     # hack: virtio with fairsched is unreliable, so we do not use fairsched
2796     # when the VM uses virtio devices.
2797     if (!$use_virtio && $have_ovz) {
2798
2799         my $cpuunits = defined($conf->{cpuunits}) ?
2800             $conf->{cpuunits} : $defaults->{cpuunits};
2801
2802         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
2803
2804         # fixme: cpulimit is currently ignored
2805         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
2806     }
2807
2808     # add custom args
2809     if ($conf->{args}) {
2810         my $aa = PVE::Tools::split_args($conf->{args});
2811         push @$cmd, @$aa;
2812     }
2813
2814     push @$cmd, @$devices;
2815     push @$cmd, '-rtc', join(',', @$rtcFlags)
2816         if scalar(@$rtcFlags);
2817     push @$cmd, '-machine', join(',', @$machineFlags)
2818         if scalar(@$machineFlags);
2819     push @$cmd, '-global', join(',', @$globalFlags)
2820         if scalar(@$globalFlags);
2821
2822     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
2823 }
2824
2825 sub vnc_socket {
2826     my ($vmid) = @_;
2827     return "${var_run_tmpdir}/$vmid.vnc";
2828 }
2829
2830 sub spice_port {
2831     my ($vmid) = @_;
2832
2833     my $res = vm_mon_cmd($vmid, 'query-spice');
2834
2835     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
2836 }
2837
2838 sub qmp_socket {
2839     my ($vmid) = @_;
2840     return "${var_run_tmpdir}/$vmid.qmp";
2841 }
2842
2843 sub qga_socket {
2844     my ($vmid) = @_;
2845     return "${var_run_tmpdir}/$vmid.qga";
2846 }
2847
2848 sub pidfile_name {
2849     my ($vmid) = @_;
2850     return "${var_run_tmpdir}/$vmid.pid";
2851 }
2852
2853 sub vm_devices_list {
2854     my ($vmid) = @_;
2855
2856     my $res = vm_mon_cmd($vmid, 'query-pci');
2857
2858     my $devices = {};
2859     foreach my $pcibus (@$res) {
2860         foreach my $device (@{$pcibus->{devices}}) {
2861             next if !$device->{'qdev_id'};
2862             $devices->{$device->{'qdev_id'}} = $device;
2863         }
2864     }
2865
2866     return $devices;
2867 }
2868
2869 sub vm_deviceplug {
2870     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
2871
2872     return 1 if !check_running($vmid);
2873
2874     my $q35 = machine_type_is_q35($conf);
2875
2876     if ($deviceid eq 'tablet') {
2877         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
2878         return 1;
2879     }
2880
2881     return 1 if !$conf->{hotplug};
2882
2883     my $devices_list = vm_devices_list($vmid);
2884     return 1 if defined($devices_list->{$deviceid});
2885
2886     qemu_bridgeadd($storecfg, $conf, $vmid, $deviceid); #add bridge if we need it for the device
2887
2888     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2889         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2890         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2891         qemu_deviceadd($vmid, $devicefull);
2892         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2893            qemu_drivedel($vmid, $deviceid);
2894            return undef;
2895         }
2896     }
2897
2898     if ($deviceid =~ m/^(scsihw)(\d+)$/) {
2899         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
2900         my $pciaddr = print_pci_addr($deviceid);
2901         my $devicefull = "$scsihw,id=$deviceid$pciaddr";
2902         qemu_deviceadd($vmid, $devicefull);
2903         return undef if(!qemu_deviceaddverify($vmid, $deviceid));
2904     }
2905
2906     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2907         return 1 if ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)); #virtio-scsi not yet support hotplug
2908         return undef if !qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
2909         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2910         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2911         if(!qemu_deviceadd($vmid, $devicefull)) {
2912            qemu_drivedel($vmid, $deviceid);
2913            return undef;
2914         }
2915     }
2916
2917     if ($deviceid =~ m/^(net)(\d+)$/) {
2918         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
2919         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
2920         qemu_deviceadd($vmid, $netdevicefull);
2921         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2922            qemu_netdevdel($vmid, $deviceid);
2923            return undef;
2924         }
2925     }
2926
2927
2928     if (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
2929         my $bridgeid = $2;
2930         my $pciaddr = print_pci_addr($deviceid);
2931         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
2932         qemu_deviceadd($vmid, $devicefull);
2933         return undef if !qemu_deviceaddverify($vmid, $deviceid);
2934     }
2935
2936     return 1;
2937 }
2938
2939 sub vm_deviceunplug {
2940     my ($vmid, $conf, $deviceid) = @_;
2941
2942     return 1 if !check_running ($vmid);
2943
2944     if ($deviceid eq 'tablet') {
2945         qemu_devicedel($vmid, $deviceid);
2946         return 1;
2947     }
2948
2949     return 1 if !$conf->{hotplug};
2950
2951     my $devices_list = vm_devices_list($vmid);
2952     return 1 if !defined($devices_list->{$deviceid});
2953
2954     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
2955
2956     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2957         qemu_devicedel($vmid, $deviceid);
2958         return undef if !qemu_devicedelverify($vmid, $deviceid);
2959         return undef if !qemu_drivedel($vmid, $deviceid);
2960     }
2961
2962     if ($deviceid =~ m/^(lsi)(\d+)$/) {
2963         return undef if !qemu_devicedel($vmid, $deviceid);
2964     }
2965
2966     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2967         return undef if !qemu_devicedel($vmid, $deviceid);
2968         return undef if !qemu_drivedel($vmid, $deviceid);
2969     }
2970
2971     if ($deviceid =~ m/^(net)(\d+)$/) {
2972         qemu_devicedel($vmid, $deviceid);
2973         return undef if !qemu_devicedelverify($vmid, $deviceid);
2974         return undef if !qemu_netdevdel($vmid, $deviceid);
2975     }
2976
2977     return 1;
2978 }
2979
2980 sub qemu_deviceadd {
2981     my ($vmid, $devicefull) = @_;
2982
2983     $devicefull = "driver=".$devicefull;
2984     my %options =  split(/[=,]/, $devicefull);
2985
2986     vm_mon_cmd($vmid, "device_add" , %options);
2987     return 1;
2988 }
2989
2990 sub qemu_devicedel {
2991     my($vmid, $deviceid) = @_;
2992     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
2993     return 1;
2994 }
2995
2996 sub qemu_driveadd {
2997     my($storecfg, $vmid, $device) = @_;
2998
2999     my $drive = print_drive_full($storecfg, $vmid, $device);
3000     my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
3001     # If the command succeeds qemu prints: "OK"
3002     if ($ret !~ m/OK/s) {
3003         syslog("err", "adding drive failed: $ret");
3004         return undef;
3005     }
3006     return 1;
3007 }
3008
3009 sub qemu_drivedel {
3010     my($vmid, $deviceid) = @_;
3011
3012     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3013     $ret =~ s/^\s+//;
3014     if ($ret =~ m/Device \'.*?\' not found/s) {
3015         # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3016     }
3017     elsif ($ret ne "") {
3018       syslog("err", "deleting drive $deviceid failed : $ret");
3019       return undef;
3020     }
3021     return 1;
3022 }
3023
3024 sub qemu_deviceaddverify {
3025     my ($vmid,$deviceid) = @_;
3026
3027     for (my $i = 0; $i <= 5; $i++) {
3028          my $devices_list = vm_devices_list($vmid);
3029          return 1 if defined($devices_list->{$deviceid});
3030          sleep 1;
3031     }
3032     syslog("err", "error on hotplug device $deviceid");
3033     return undef;
3034 }
3035
3036
3037 sub qemu_devicedelverify {
3038     my ($vmid,$deviceid) = @_;
3039
3040     #need to verify the device is correctly remove as device_del is async and empty return is not reliable
3041     for (my $i = 0; $i <= 5; $i++) {
3042          my $devices_list = vm_devices_list($vmid);
3043          return 1 if !defined($devices_list->{$deviceid});
3044          sleep 1;
3045     }
3046     syslog("err", "error on hot-unplugging device $deviceid");
3047     return undef;
3048 }
3049
3050 sub qemu_findorcreatescsihw {
3051     my ($storecfg, $conf, $vmid, $device) = @_;
3052
3053     my $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
3054     my $controller = int($device->{index} / $maxdev);
3055     my $scsihwid="scsihw$controller";
3056     my $devices_list = vm_devices_list($vmid);
3057
3058     if(!defined($devices_list->{$scsihwid})) {
3059        return undef if !vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
3060     }
3061     return 1;
3062 }
3063
3064 sub qemu_bridgeadd {
3065     my ($storecfg, $conf, $vmid, $device) = @_;
3066
3067     my $bridges = {};
3068     my $bridgeid = undef;
3069     print_pci_addr($device, $bridges);
3070
3071     while (my ($k, $v) = each %$bridges) {
3072         $bridgeid = $k;
3073     }
3074     return if !$bridgeid || $bridgeid < 1;
3075     my $bridge = "pci.$bridgeid";
3076     my $devices_list = vm_devices_list($vmid);
3077
3078     if(!defined($devices_list->{$bridge})) {
3079         return undef if !vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3080     }
3081     return 1;
3082 }
3083
3084 sub qemu_netdevadd {
3085     my ($vmid, $conf, $device, $deviceid) = @_;
3086
3087     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3088     my %options =  split(/[=,]/, $netdev);
3089
3090     vm_mon_cmd($vmid, "netdev_add",  %options);
3091     return 1;
3092 }
3093
3094 sub qemu_netdevdel {
3095     my ($vmid, $deviceid) = @_;
3096
3097     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3098     return 1;
3099 }
3100
3101 sub qemu_cpu_hotplug {
3102     my ($vmid, $conf, $cores) = @_;
3103
3104     die "new cores config is not defined" if !$cores;
3105     die "you can't add more cores than maxcpus"
3106         if $conf->{maxcpus} && ($cores > $conf->{maxcpus});
3107     return if !check_running($vmid);
3108
3109     my $currentcores = $conf->{cores} if $conf->{cores};
3110     die "current cores is not defined" if !$currentcores;
3111     die "maxcpus is not defined" if !$conf->{maxcpus};
3112     raise_param_exc({ 'cores' => "online cpu unplug is not yet possible" })
3113         if($cores < $currentcores);
3114
3115     my $currentrunningcores = vm_mon_cmd($vmid, "query-cpus");
3116     raise_param_exc({ 'cores' => "cores number if running vm is different than configuration" })
3117         if scalar (@{$currentrunningcores}) != $currentcores;
3118
3119     for(my $i = $currentcores; $i < $cores; $i++) {
3120         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3121     }
3122 }
3123
3124 sub qemu_block_set_io_throttle {
3125     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3126
3127     return if !check_running($vmid) ;
3128
3129     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3130
3131 }
3132
3133 # old code, only used to shutdown old VM after update
3134 sub __read_avail {
3135     my ($fh, $timeout) = @_;
3136
3137     my $sel = new IO::Select;
3138     $sel->add($fh);
3139
3140     my $res = '';
3141     my $buf;
3142
3143     my @ready;
3144     while (scalar (@ready = $sel->can_read($timeout))) {
3145         my $count;
3146         if ($count = $fh->sysread($buf, 8192)) {
3147             if ($buf =~ /^(.*)\(qemu\) $/s) {
3148                 $res .= $1;
3149                 last;
3150             } else {
3151                 $res .= $buf;
3152             }
3153         } else {
3154             if (!defined($count)) {
3155                 die "$!\n";
3156             }
3157             last;
3158         }
3159     }
3160
3161     die "monitor read timeout\n" if !scalar(@ready);
3162
3163     return $res;
3164 }
3165
3166 # old code, only used to shutdown old VM after update
3167 sub vm_monitor_command {
3168     my ($vmid, $cmdstr, $nocheck) = @_;
3169
3170     my $res;
3171
3172     eval {
3173         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3174
3175         my $sname = "${var_run_tmpdir}/$vmid.mon";
3176
3177         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3178             die "unable to connect to VM $vmid socket - $!\n";
3179
3180         my $timeout = 3;
3181
3182         # hack: migrate sometime blocks the monitor (when migrate_downtime
3183         # is set)
3184         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3185             $timeout = 60*60; # 1 hour
3186         }
3187
3188         # read banner;
3189         my $data = __read_avail($sock, $timeout);
3190
3191         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3192             die "got unexpected qemu monitor banner\n";
3193         }
3194
3195         my $sel = new IO::Select;
3196         $sel->add($sock);
3197
3198         if (!scalar(my @ready = $sel->can_write($timeout))) {
3199             die "monitor write error - timeout";
3200         }
3201
3202         my $fullcmd = "$cmdstr\r";
3203
3204         # syslog('info', "VM $vmid monitor command: $cmdstr");
3205
3206         my $b;
3207         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3208             die "monitor write error - $!";
3209         }
3210
3211         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3212
3213         $timeout = 20;
3214
3215         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3216             $timeout = 60*60; # 1 hour
3217         } elsif ($cmdstr =~ m/^(eject|change)/) {
3218             $timeout = 60; # note: cdrom mount command is slow
3219         }
3220         if ($res = __read_avail($sock, $timeout)) {
3221
3222             my @lines = split("\r?\n", $res);
3223
3224             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3225
3226             $res = join("\n", @lines);
3227             $res .= "\n";
3228         }
3229     };
3230
3231     my $err = $@;
3232
3233     if ($err) {
3234         syslog("err", "VM $vmid monitor command failed - $err");
3235         die $err;
3236     }
3237
3238     return $res;
3239 }
3240
3241 sub qemu_block_resize {
3242     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3243
3244     my $running = check_running($vmid);
3245
3246     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3247
3248     return if !$running;
3249
3250     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3251
3252 }
3253
3254 sub qemu_volume_snapshot {
3255     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3256
3257     my $running = check_running($vmid);
3258
3259     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3260
3261     return if !$running;
3262
3263     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3264
3265 }
3266
3267 sub qemu_volume_snapshot_delete {
3268     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3269
3270     my $running = check_running($vmid);
3271
3272     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3273
3274     return if !$running;
3275
3276     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3277 }
3278
3279 sub qga_freezefs {
3280     my ($vmid) = @_;
3281
3282     #need to impplement call to qemu-ga
3283 }
3284
3285 sub qga_unfreezefs {
3286     my ($vmid) = @_;
3287
3288     #need to impplement call to qemu-ga
3289 }
3290
3291 sub set_migration_caps {
3292     my ($vmid) = @_;
3293
3294     my $cap_ref = [];
3295
3296     my $enabled_cap = {
3297         "auto-converge" => 1,
3298         "xbzrle" => 0,
3299         "x-rdma-pin-all" => 0,
3300         "zero-blocks" => 0,
3301     };
3302
3303     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3304
3305     for my $supported_capability (@$supported_capabilities) {
3306         push @$cap_ref, {
3307             capability => $supported_capability->{capability},
3308             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3309         };
3310     }
3311
3312     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3313 }
3314
3315 sub vm_start {
3316     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
3317
3318     lock_config($vmid, sub {
3319         my $conf = load_config($vmid, $migratedfrom);
3320
3321         die "you can't start a vm if it's a template\n" if is_template($conf);
3322
3323         check_lock($conf) if !$skiplock;
3324
3325         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
3326
3327         my $defaults = load_defaults();
3328
3329         # set environment variable useful inside network script
3330         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
3331
3332         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
3333
3334         my $migrate_port = 0;
3335         my $migrate_uri;
3336         if ($statefile) {
3337             if ($statefile eq 'tcp') {
3338                 my $localip = "localhost";
3339                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
3340                 if ($datacenterconf->{migration_unsecure}) {
3341                         my $nodename = PVE::INotify::nodename();
3342                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
3343                 }
3344                 $migrate_port = PVE::Tools::next_migrate_port();
3345                 $migrate_uri = "tcp:${localip}:${migrate_port}";
3346                 push @$cmd, '-incoming', $migrate_uri;
3347                 push @$cmd, '-S';
3348             } else {
3349                 push @$cmd, '-loadstate', $statefile;
3350             }
3351         } elsif ($paused) {
3352             push @$cmd, '-S';
3353         }
3354
3355         # host pci devices
3356         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3357           my $d = parse_hostpci($conf->{"hostpci$i"});
3358           next if !$d;
3359           my $pcidevices = $d->{pciid};
3360           foreach my $pcidevice (@$pcidevices) {
3361                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
3362
3363                 my $info = pci_device_info("0000:$pciid");
3364                 die "IOMMU not present\n" if !check_iommu_support();
3365                 die "no pci device info for device '$pciid'\n" if !$info;
3366
3367                 if ($d->{driver} && $d->{driver} eq "vfio") {
3368                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
3369                 } else {
3370                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
3371                 }
3372
3373                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
3374           }
3375         }
3376
3377         PVE::Storage::activate_volumes($storecfg, $vollist);
3378
3379         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
3380                     umask => 0077); };
3381         my $err = $@;
3382         die "start failed: $err" if $err;
3383
3384         print "migration listens on $migrate_uri\n" if $migrate_uri;
3385
3386         if ($statefile && $statefile ne 'tcp')  {
3387             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
3388             warn $@ if $@;
3389         }
3390
3391         if ($migratedfrom) {
3392
3393             eval {
3394                 PVE::QemuServer::set_migration_caps($vmid);
3395             };
3396             warn $@ if $@;
3397
3398             if ($spice_port) {
3399                 print "spice listens on port $spice_port\n";
3400                 if ($spice_ticket) {
3401                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
3402                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
3403                 }
3404             }
3405
3406         } else {
3407
3408             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
3409                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
3410                     if $conf->{balloon};
3411                 vm_mon_cmd_nocheck($vmid, 'qom-set',
3412                             path => "machine/peripheral/balloon0",
3413                             property => "guest-stats-polling-interval",
3414                             value => 2);
3415             }
3416         }
3417     });
3418 }
3419
3420 sub vm_mon_cmd {
3421     my ($vmid, $execute, %params) = @_;
3422
3423     my $cmd = { execute => $execute, arguments => \%params };
3424     vm_qmp_command($vmid, $cmd);
3425 }
3426
3427 sub vm_mon_cmd_nocheck {
3428     my ($vmid, $execute, %params) = @_;
3429
3430     my $cmd = { execute => $execute, arguments => \%params };
3431     vm_qmp_command($vmid, $cmd, 1);
3432 }
3433
3434 sub vm_qmp_command {
3435     my ($vmid, $cmd, $nocheck) = @_;
3436
3437     my $res;
3438
3439     my $timeout;
3440     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
3441         $timeout = $cmd->{arguments}->{timeout};
3442         delete $cmd->{arguments}->{timeout};
3443     }
3444
3445     eval {
3446         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3447         my $sname = qmp_socket($vmid);
3448         if (-e $sname) {
3449             my $qmpclient = PVE::QMPClient->new();
3450
3451             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
3452         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
3453             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
3454                 if scalar(%{$cmd->{arguments}});
3455             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
3456         } else {
3457             die "unable to open monitor socket\n";
3458         }
3459     };
3460     if (my $err = $@) {
3461         syslog("err", "VM $vmid qmp command failed - $err");
3462         die $err;
3463     }
3464
3465     return $res;
3466 }
3467
3468 sub vm_human_monitor_command {
3469     my ($vmid, $cmdline) = @_;
3470
3471     my $res;
3472
3473     my $cmd = {
3474         execute => 'human-monitor-command',
3475         arguments => { 'command-line' => $cmdline},
3476     };
3477
3478     return vm_qmp_command($vmid, $cmd);
3479 }
3480
3481 sub vm_commandline {
3482     my ($storecfg, $vmid) = @_;
3483
3484     my $conf = load_config($vmid);
3485
3486     my $defaults = load_defaults();
3487
3488     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
3489
3490     return join(' ', @$cmd);
3491 }
3492
3493 sub vm_reset {
3494     my ($vmid, $skiplock) = @_;
3495
3496     lock_config($vmid, sub {
3497
3498         my $conf = load_config($vmid);
3499
3500         check_lock($conf) if !$skiplock;
3501
3502         vm_mon_cmd($vmid, "system_reset");
3503     });
3504 }
3505
3506 sub get_vm_volumes {
3507     my ($conf) = @_;
3508
3509     my $vollist = [];
3510     foreach_volid($conf, sub {
3511         my ($volid, $is_cdrom) = @_;
3512
3513         return if $volid =~ m|^/|;
3514
3515         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3516         return if !$sid;
3517
3518         push @$vollist, $volid;
3519     });
3520
3521     return $vollist;
3522 }
3523
3524 sub vm_stop_cleanup {
3525     my ($storecfg, $vmid, $conf, $keepActive) = @_;
3526
3527     eval {
3528         fairsched_rmnod($vmid); # try to destroy group
3529
3530         if (!$keepActive) {
3531             my $vollist = get_vm_volumes($conf);
3532             PVE::Storage::deactivate_volumes($storecfg, $vollist);
3533         }
3534
3535         foreach my $ext (qw(mon qmp pid vnc qga)) {
3536             unlink "/var/run/qemu-server/${vmid}.$ext";
3537         }
3538     };
3539     warn $@ if $@; # avoid errors - just warn
3540 }
3541
3542 # Note: use $nockeck to skip tests if VM configuration file exists.
3543 # We need that when migration VMs to other nodes (files already moved)
3544 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
3545 sub vm_stop {
3546     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
3547
3548     $force = 1 if !defined($force) && !$shutdown;
3549
3550     if ($migratedfrom){
3551         my $pid = check_running($vmid, $nocheck, $migratedfrom);
3552         kill 15, $pid if $pid;
3553         my $conf = load_config($vmid, $migratedfrom);
3554         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive);
3555         return;
3556     }
3557
3558     lock_config($vmid, sub {
3559
3560         my $pid = check_running($vmid, $nocheck);
3561         return if !$pid;
3562
3563         my $conf;
3564         if (!$nocheck) {
3565             $conf = load_config($vmid);
3566             check_lock($conf) if !$skiplock;
3567             if (!defined($timeout) && $shutdown && $conf->{startup}) {
3568                 my $opts = parse_startup($conf->{startup});
3569                 $timeout = $opts->{down} if $opts->{down};
3570             }
3571         }
3572
3573         $timeout = 60 if !defined($timeout);
3574
3575         eval {
3576             if ($shutdown) {
3577                 $nocheck ? vm_mon_cmd_nocheck($vmid, "system_powerdown") : vm_mon_cmd($vmid, "system_powerdown");
3578
3579             } else {
3580                 $nocheck ? vm_mon_cmd_nocheck($vmid, "quit") : vm_mon_cmd($vmid, "quit");
3581             }
3582         };
3583         my $err = $@;
3584
3585         if (!$err) {
3586             my $count = 0;
3587             while (($count < $timeout) && check_running($vmid, $nocheck)) {
3588                 $count++;
3589                 sleep 1;
3590             }
3591
3592             if ($count >= $timeout) {
3593                 if ($force) {
3594                     warn "VM still running - terminating now with SIGTERM\n";
3595                     kill 15, $pid;
3596                 } else {
3597                     die "VM quit/powerdown failed - got timeout\n";
3598                 }
3599             } else {
3600                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3601                 return;
3602             }
3603         } else {
3604             if ($force) {
3605                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
3606                 kill 15, $pid;
3607             } else {
3608                 die "VM quit/powerdown failed\n";
3609             }
3610         }
3611
3612         # wait again
3613         $timeout = 10;
3614
3615         my $count = 0;
3616         while (($count < $timeout) && check_running($vmid, $nocheck)) {
3617             $count++;
3618             sleep 1;
3619         }
3620
3621         if ($count >= $timeout) {
3622             warn "VM still running - terminating now with SIGKILL\n";
3623             kill 9, $pid;
3624             sleep 1;
3625         }
3626
3627         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3628    });
3629 }
3630
3631 sub vm_suspend {
3632     my ($vmid, $skiplock) = @_;
3633
3634     lock_config($vmid, sub {
3635
3636         my $conf = load_config($vmid);
3637
3638         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3639
3640         vm_mon_cmd($vmid, "stop");
3641     });
3642 }
3643
3644 sub vm_resume {
3645     my ($vmid, $skiplock) = @_;
3646
3647     lock_config($vmid, sub {
3648
3649         my $conf = load_config($vmid);
3650
3651         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3652
3653         vm_mon_cmd($vmid, "cont");
3654     });
3655 }
3656
3657 sub vm_sendkey {
3658     my ($vmid, $skiplock, $key) = @_;
3659
3660     lock_config($vmid, sub {
3661
3662         my $conf = load_config($vmid);
3663
3664         # there is no qmp command, so we use the human monitor command
3665         vm_human_monitor_command($vmid, "sendkey $key");
3666     });
3667 }
3668
3669 sub vm_destroy {
3670     my ($storecfg, $vmid, $skiplock) = @_;
3671
3672     lock_config($vmid, sub {
3673
3674         my $conf = load_config($vmid);
3675
3676         check_lock($conf) if !$skiplock;
3677
3678         if (!check_running($vmid)) {
3679             fairsched_rmnod($vmid); # try to destroy group
3680             destroy_vm($storecfg, $vmid);
3681         } else {
3682             die "VM $vmid is running - destroy failed\n";
3683         }
3684     });
3685 }
3686
3687 # pci helpers
3688
3689 sub file_write {
3690     my ($filename, $buf) = @_;
3691
3692     my $fh = IO::File->new($filename, "w");
3693     return undef if !$fh;
3694
3695     my $res = print $fh $buf;
3696
3697     $fh->close();
3698
3699     return $res;
3700 }
3701
3702 sub pci_device_info {
3703     my ($name) = @_;
3704
3705     my $res;
3706
3707     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
3708     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
3709
3710     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
3711     return undef if !defined($irq) || $irq !~ m/^\d+$/;
3712
3713     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
3714     return undef if !defined($vendor) || $vendor !~ s/^0x//;
3715
3716     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
3717     return undef if !defined($product) || $product !~ s/^0x//;
3718
3719     $res = {
3720         name => $name,
3721         vendor => $vendor,
3722         product => $product,
3723         domain => $domain,
3724         bus => $bus,
3725         slot => $slot,
3726         func => $func,
3727         irq => $irq,
3728         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
3729     };
3730
3731     return $res;
3732 }
3733
3734 sub pci_dev_reset {
3735     my ($dev) = @_;
3736
3737     my $name = $dev->{name};
3738
3739     my $fn = "$pcisysfs/devices/$name/reset";
3740
3741     return file_write($fn, "1");
3742 }
3743
3744 sub pci_dev_bind_to_stub {
3745     my ($dev) = @_;
3746
3747     my $name = $dev->{name};
3748
3749     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
3750     return 1 if -d $testdir;
3751
3752     my $data = "$dev->{vendor} $dev->{product}";
3753     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
3754
3755     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3756     if (!file_write($fn, $name)) {
3757         return undef if -f $fn;
3758     }
3759
3760     $fn = "$pcisysfs/drivers/pci-stub/bind";
3761     if (! -d $testdir) {
3762         return undef if !file_write($fn, $name);
3763     }
3764
3765     return -d $testdir;
3766 }
3767
3768 sub pci_dev_bind_to_vfio {
3769     my ($dev) = @_;
3770
3771     my $name = $dev->{name};
3772
3773     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
3774
3775     if (!-d $vfio_basedir) {
3776         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
3777     }
3778     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
3779
3780     my $testdir = "$vfio_basedir/$name";
3781     return 1 if -d $testdir;
3782
3783     my $data = "$dev->{vendor} $dev->{product}";
3784     return undef if !file_write("$vfio_basedir/new_id", $data);
3785
3786     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3787     if (!file_write($fn, $name)) {
3788         return undef if -f $fn;
3789     }
3790
3791     $fn = "$vfio_basedir/bind";
3792     if (! -d $testdir) {
3793         return undef if !file_write($fn, $name);
3794     }
3795
3796     return -d $testdir;
3797 }
3798
3799 sub pci_dev_group_bind_to_vfio {
3800     my ($pciid) = @_;
3801
3802     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
3803
3804     if (!-d $vfio_basedir) {
3805         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
3806     }
3807     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
3808
3809     # get IOMMU group devices
3810     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
3811       my @devs = grep /^0000:/, readdir($D);
3812     closedir($D);
3813
3814     foreach my $pciid (@devs) {
3815         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
3816         my $info = pci_device_info($1);
3817         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
3818     }
3819
3820     return 1;
3821 }
3822
3823 sub print_pci_addr {
3824     my ($id, $bridges) = @_;
3825
3826     my $res = '';
3827     my $devices = {
3828         piix3 => { bus => 0, addr => 1 },
3829         #addr2 : first videocard
3830         balloon0 => { bus => 0, addr => 3 },
3831         watchdog => { bus => 0, addr => 4 },
3832         scsihw0 => { bus => 0, addr => 5 },
3833         scsihw1 => { bus => 0, addr => 6 },
3834         ahci0 => { bus => 0, addr => 7 },
3835         qga0 => { bus => 0, addr => 8 },
3836         spice => { bus => 0, addr => 9 },
3837         virtio0 => { bus => 0, addr => 10 },
3838         virtio1 => { bus => 0, addr => 11 },
3839         virtio2 => { bus => 0, addr => 12 },
3840         virtio3 => { bus => 0, addr => 13 },
3841         virtio4 => { bus => 0, addr => 14 },
3842         virtio5 => { bus => 0, addr => 15 },
3843         hostpci0 => { bus => 0, addr => 16 },
3844         hostpci1 => { bus => 0, addr => 17 },
3845         net0 => { bus => 0, addr => 18 },
3846         net1 => { bus => 0, addr => 19 },
3847         net2 => { bus => 0, addr => 20 },
3848         net3 => { bus => 0, addr => 21 },
3849         net4 => { bus => 0, addr => 22 },
3850         net5 => { bus => 0, addr => 23 },
3851         vga1 => { bus => 0, addr => 24 },
3852         vga2 => { bus => 0, addr => 25 },
3853         vga3 => { bus => 0, addr => 26 },
3854         #addr29 : usb-host (pve-usb.cfg)
3855         'pci.1' => { bus => 0, addr => 30 },
3856         'pci.2' => { bus => 0, addr => 31 },
3857         'net6' => { bus => 1, addr => 1 },
3858         'net7' => { bus => 1, addr => 2 },
3859         'net8' => { bus => 1, addr => 3 },
3860         'net9' => { bus => 1, addr => 4 },
3861         'net10' => { bus => 1, addr => 5 },
3862         'net11' => { bus => 1, addr => 6 },
3863         'net12' => { bus => 1, addr => 7 },
3864         'net13' => { bus => 1, addr => 8 },
3865         'net14' => { bus => 1, addr => 9 },
3866         'net15' => { bus => 1, addr => 10 },
3867         'net16' => { bus => 1, addr => 11 },
3868         'net17' => { bus => 1, addr => 12 },
3869         'net18' => { bus => 1, addr => 13 },
3870         'net19' => { bus => 1, addr => 14 },
3871         'net20' => { bus => 1, addr => 15 },
3872         'net21' => { bus => 1, addr => 16 },
3873         'net22' => { bus => 1, addr => 17 },
3874         'net23' => { bus => 1, addr => 18 },
3875         'net24' => { bus => 1, addr => 19 },
3876         'net25' => { bus => 1, addr => 20 },
3877         'net26' => { bus => 1, addr => 21 },
3878         'net27' => { bus => 1, addr => 22 },
3879         'net28' => { bus => 1, addr => 23 },
3880         'net29' => { bus => 1, addr => 24 },
3881         'net30' => { bus => 1, addr => 25 },
3882         'net31' => { bus => 1, addr => 26 },
3883         'virtio6' => { bus => 2, addr => 1 },
3884         'virtio7' => { bus => 2, addr => 2 },
3885         'virtio8' => { bus => 2, addr => 3 },
3886         'virtio9' => { bus => 2, addr => 4 },
3887         'virtio10' => { bus => 2, addr => 5 },
3888         'virtio11' => { bus => 2, addr => 6 },
3889         'virtio12' => { bus => 2, addr => 7 },
3890         'virtio13' => { bus => 2, addr => 8 },
3891         'virtio14' => { bus => 2, addr => 9 },
3892         'virtio15' => { bus => 2, addr => 10 },
3893     };
3894
3895     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3896            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3897            my $bus = $devices->{$id}->{bus};
3898            $res = ",bus=pci.$bus,addr=$addr";
3899            $bridges->{$bus} = 1 if $bridges;
3900     }
3901     return $res;
3902
3903 }
3904
3905 sub print_pcie_addr {
3906     my ($id) = @_;
3907
3908     my $res = '';
3909     my $devices = {
3910         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
3911         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
3912         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
3913         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
3914     };
3915
3916     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3917            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3918            my $bus = $devices->{$id}->{bus};
3919            $res = ",bus=$bus,addr=$addr";
3920     }
3921     return $res;
3922
3923 }
3924
3925 # vzdump restore implementaion
3926
3927 sub tar_archive_read_firstfile {
3928     my $archive = shift;
3929
3930     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
3931
3932     # try to detect archive type first
3933     my $pid = open (TMP, "tar tf '$archive'|") ||
3934         die "unable to open file '$archive'\n";
3935     my $firstfile = <TMP>;
3936     kill 15, $pid;
3937     close TMP;
3938
3939     die "ERROR: archive contaions no data\n" if !$firstfile;
3940     chomp $firstfile;
3941
3942     return $firstfile;
3943 }
3944
3945 sub tar_restore_cleanup {
3946     my ($storecfg, $statfile) = @_;
3947
3948     print STDERR "starting cleanup\n";
3949
3950     if (my $fd = IO::File->new($statfile, "r")) {
3951         while (defined(my $line = <$fd>)) {
3952             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
3953                 my $volid = $2;
3954                 eval {
3955                     if ($volid =~ m|^/|) {
3956                         unlink $volid || die 'unlink failed\n';
3957                     } else {
3958                         PVE::Storage::vdisk_free($storecfg, $volid);
3959                     }
3960                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
3961                 };
3962                 print STDERR "unable to cleanup '$volid' - $@" if $@;
3963             } else {
3964                 print STDERR "unable to parse line in statfile - $line";
3965             }
3966         }
3967         $fd->close();
3968     }
3969 }
3970
3971 sub restore_archive {
3972     my ($archive, $vmid, $user, $opts) = @_;
3973
3974     my $format = $opts->{format};
3975     my $comp;
3976
3977     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
3978         $format = 'tar' if !$format;
3979         $comp = 'gzip';
3980     } elsif ($archive =~ m/\.tar$/) {
3981         $format = 'tar' if !$format;
3982     } elsif ($archive =~ m/.tar.lzo$/) {
3983         $format = 'tar' if !$format;
3984         $comp = 'lzop';
3985     } elsif ($archive =~ m/\.vma$/) {
3986         $format = 'vma' if !$format;
3987     } elsif ($archive =~ m/\.vma\.gz$/) {
3988         $format = 'vma' if !$format;
3989         $comp = 'gzip';
3990     } elsif ($archive =~ m/\.vma\.lzo$/) {
3991         $format = 'vma' if !$format;
3992         $comp = 'lzop';
3993     } else {
3994         $format = 'vma' if !$format; # default
3995     }
3996
3997     # try to detect archive format
3998     if ($format eq 'tar') {
3999         return restore_tar_archive($archive, $vmid, $user, $opts);
4000     } else {
4001         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4002     }
4003 }
4004
4005 sub restore_update_config_line {
4006     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4007
4008     return if $line =~ m/^\#qmdump\#/;
4009     return if $line =~ m/^\#vzdump\#/;
4010     return if $line =~ m/^lock:/;
4011     return if $line =~ m/^unused\d+:/;
4012     return if $line =~ m/^parent:/;
4013     return if $line =~ m/^template:/; # restored VM is never a template
4014
4015     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4016         # try to convert old 1.X settings
4017         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4018         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4019             my ($model, $macaddr) = split(/\=/, $devconfig);
4020             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4021             my $net = {
4022                 model => $model,
4023                 bridge => "vmbr$ind",
4024                 macaddr => $macaddr,
4025             };
4026             my $netstr = print_net($net);
4027
4028             print $outfd "net$cookie->{netcount}: $netstr\n";
4029             $cookie->{netcount}++;
4030         }
4031     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4032         my ($id, $netstr) = ($1, $2);
4033         my $net = parse_net($netstr);
4034         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4035         $netstr = print_net($net);
4036         print $outfd "$id: $netstr\n";
4037     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4038         my $virtdev = $1;
4039         my $value = $3;
4040         if ($line =~ m/backup=no/) {
4041             print $outfd "#$line";
4042         } elsif ($virtdev && $map->{$virtdev}) {
4043             my $di = parse_drive($virtdev, $value);
4044             delete $di->{format}; # format can change on restore
4045             $di->{file} = $map->{$virtdev};
4046             $value = print_drive($vmid, $di);
4047             print $outfd "$virtdev: $value\n";
4048         } else {
4049             print $outfd $line;
4050         }
4051     } else {
4052         print $outfd $line;
4053     }
4054 }
4055
4056 sub scan_volids {
4057     my ($cfg, $vmid) = @_;
4058
4059     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4060
4061     my $volid_hash = {};
4062     foreach my $storeid (keys %$info) {
4063         foreach my $item (@{$info->{$storeid}}) {
4064             next if !($item->{volid} && $item->{size});
4065             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4066             $volid_hash->{$item->{volid}} = $item;
4067         }
4068     }
4069
4070     return $volid_hash;
4071 }
4072
4073 sub get_used_paths {
4074     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4075
4076     my $used_path = {};
4077
4078     my $scan_config = sub {
4079         my ($cref, $snapname) = @_;
4080
4081         foreach my $key (keys %$cref) {
4082             my $value = $cref->{$key};
4083             if (valid_drivename($key)) {
4084                 next if $skip_drive && $key eq $skip_drive;
4085                 my $drive = parse_drive($key, $value);
4086                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4087                 if ($drive->{file} =~ m!^/!) {
4088                     $used_path->{$drive->{file}}++; # = 1;
4089                 } else {
4090                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4091                     next if !$storeid;
4092                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4093                     next if !$scfg;
4094                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4095                     $used_path->{$path}++; # = 1;
4096                 }
4097             }
4098         }
4099     };
4100
4101     &$scan_config($conf);
4102
4103     undef $skip_drive;
4104
4105     if ($scan_snapshots) {
4106         foreach my $snapname (keys %{$conf->{snapshots}}) {
4107             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4108         }
4109     }
4110
4111     return $used_path;
4112 }
4113
4114 sub update_disksize {
4115     my ($vmid, $conf, $volid_hash) = @_;
4116
4117     my $changes;
4118
4119     my $used = {};
4120
4121     # Note: it is allowed to define multiple storages with same path (alias), so
4122     # we need to check both 'volid' and real 'path' (two different volid can point
4123     # to the same path).
4124
4125     my $usedpath = {};
4126
4127     # update size info
4128     foreach my $opt (keys %$conf) {
4129         if (valid_drivename($opt)) {
4130             my $drive = parse_drive($opt, $conf->{$opt});
4131             my $volid = $drive->{file};
4132             next if !$volid;
4133
4134             $used->{$volid} = 1;
4135             if ($volid_hash->{$volid} &&
4136                 (my $path = $volid_hash->{$volid}->{path})) {
4137                 $usedpath->{$path} = 1;
4138             }
4139
4140             next if drive_is_cdrom($drive);
4141             next if !$volid_hash->{$volid};
4142
4143             $drive->{size} = $volid_hash->{$volid}->{size};
4144             my $new = print_drive($vmid, $drive);
4145             if ($new ne $conf->{$opt}) {
4146                 $changes = 1;
4147                 $conf->{$opt} = $new;
4148             }
4149         }
4150     }
4151
4152     # remove 'unusedX' entry if volume is used
4153     foreach my $opt (keys %$conf) {
4154         next if $opt !~ m/^unused\d+$/;
4155         my $volid = $conf->{$opt};
4156         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
4157         if ($used->{$volid} || ($path && $usedpath->{$path})) {
4158             $changes = 1;
4159             delete $conf->{$opt};
4160         }
4161     }
4162
4163     foreach my $volid (sort keys %$volid_hash) {
4164         next if $volid =~ m/vm-$vmid-state-/;
4165         next if $used->{$volid};
4166         my $path = $volid_hash->{$volid}->{path};
4167         next if !$path; # just to be sure
4168         next if $usedpath->{$path};
4169         $changes = 1;
4170         add_unused_volume($conf, $volid);
4171         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
4172     }
4173
4174     return $changes;
4175 }
4176
4177 sub rescan {
4178     my ($vmid, $nolock) = @_;
4179
4180     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
4181
4182     my $volid_hash = scan_volids($cfg, $vmid);
4183
4184     my $updatefn =  sub {
4185         my ($vmid) = @_;
4186
4187         my $conf = load_config($vmid);
4188
4189         check_lock($conf);
4190
4191         my $vm_volids = {};
4192         foreach my $volid (keys %$volid_hash) {
4193             my $info = $volid_hash->{$volid};
4194             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
4195         }
4196
4197         my $changes = update_disksize($vmid, $conf, $vm_volids);
4198
4199         update_config_nolock($vmid, $conf, 1) if $changes;
4200     };
4201
4202     if (defined($vmid)) {
4203         if ($nolock) {
4204             &$updatefn($vmid);
4205         } else {
4206             lock_config($vmid, $updatefn, $vmid);
4207         }
4208     } else {
4209         my $vmlist = config_list();
4210         foreach my $vmid (keys %$vmlist) {
4211             if ($nolock) {
4212                 &$updatefn($vmid);
4213             } else {
4214                 lock_config($vmid, $updatefn, $vmid);
4215             }
4216         }
4217     }
4218 }
4219
4220 sub restore_vma_archive {
4221     my ($archive, $vmid, $user, $opts, $comp) = @_;
4222
4223     my $input = $archive eq '-' ? "<&STDIN" : undef;
4224     my $readfrom = $archive;
4225
4226     my $uncomp = '';
4227     if ($comp) {
4228         $readfrom = '-';
4229         my $qarchive = PVE::Tools::shellquote($archive);
4230         if ($comp eq 'gzip') {
4231             $uncomp = "zcat $qarchive|";
4232         } elsif ($comp eq 'lzop') {
4233             $uncomp = "lzop -d -c $qarchive|";
4234         } else {
4235             die "unknown compression method '$comp'\n";
4236         }
4237
4238     }
4239
4240     my $tmpdir = "/var/tmp/vzdumptmp$$";
4241     rmtree $tmpdir;
4242
4243     # disable interrupts (always do cleanups)
4244     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4245         warn "got interrupt - ignored\n";
4246     };
4247
4248     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
4249     POSIX::mkfifo($mapfifo, 0600);
4250     my $fifofh;
4251
4252     my $openfifo = sub {
4253         open($fifofh, '>', $mapfifo) || die $!;
4254     };
4255
4256     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
4257
4258     my $oldtimeout;
4259     my $timeout = 5;
4260
4261     my $devinfo = {};
4262
4263     my $rpcenv = PVE::RPCEnvironment::get();
4264
4265     my $conffile = config_file($vmid);
4266     my $tmpfn = "$conffile.$$.tmp";
4267
4268     # Note: $oldconf is undef if VM does not exists
4269     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
4270
4271     my $print_devmap = sub {
4272         my $virtdev_hash = {};
4273
4274         my $cfgfn = "$tmpdir/qemu-server.conf";
4275
4276         # we can read the config - that is already extracted
4277         my $fh = IO::File->new($cfgfn, "r") ||
4278             "unable to read qemu-server.conf - $!\n";
4279
4280         while (defined(my $line = <$fh>)) {
4281             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
4282                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
4283                 die "archive does not contain data for drive '$virtdev'\n"
4284                     if !$devinfo->{$devname};
4285                 if (defined($opts->{storage})) {
4286                     $storeid = $opts->{storage} || 'local';
4287                 } elsif (!$storeid) {
4288                     $storeid = 'local';
4289                 }
4290                 $format = 'raw' if !$format;
4291                 $devinfo->{$devname}->{devname} = $devname;
4292                 $devinfo->{$devname}->{virtdev} = $virtdev;
4293                 $devinfo->{$devname}->{format} = $format;
4294                 $devinfo->{$devname}->{storeid} = $storeid;
4295
4296                 # check permission on storage
4297                 my $pool = $opts->{pool}; # todo: do we need that?
4298                 if ($user ne 'root@pam') {
4299                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
4300                 }
4301
4302                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
4303             }
4304         }
4305
4306         foreach my $devname (keys %$devinfo) {
4307             die "found no device mapping information for device '$devname'\n"
4308                 if !$devinfo->{$devname}->{virtdev};
4309         }
4310
4311         my $cfg = cfs_read_file('storage.cfg');
4312
4313         # create empty/temp config
4314         if ($oldconf) {
4315             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
4316             foreach_drive($oldconf, sub {
4317                 my ($ds, $drive) = @_;
4318
4319                 return if drive_is_cdrom($drive);
4320
4321                 my $volid = $drive->{file};
4322
4323                 return if !$volid || $volid =~ m|^/|;
4324
4325                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
4326                 return if !$path || !$owner || ($owner != $vmid);
4327
4328                 # Note: only delete disk we want to restore
4329                 # other volumes will become unused
4330                 if ($virtdev_hash->{$ds}) {
4331                     PVE::Storage::vdisk_free($cfg, $volid);
4332                 }
4333             });
4334         }
4335
4336         my $map = {};
4337         foreach my $virtdev (sort keys %$virtdev_hash) {
4338             my $d = $virtdev_hash->{$virtdev};
4339             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
4340             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
4341
4342             # test if requested format is supported
4343             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
4344             my $supported = grep { $_ eq $d->{format} } @$validFormats;
4345             $d->{format} = $defFormat if !$supported;
4346
4347             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
4348                                                   $d->{format}, undef, $alloc_size);
4349             print STDERR "new volume ID is '$volid'\n";
4350             $d->{volid} = $volid;
4351             my $path = PVE::Storage::path($cfg, $volid);
4352
4353             my $write_zeros = 1;
4354             # fixme: what other storages types initialize volumes with zero?
4355             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
4356                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
4357                 $write_zeros = 0;
4358             }
4359
4360             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
4361
4362             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
4363             $map->{$virtdev} = $volid;
4364         }
4365
4366         $fh->seek(0, 0) || die "seek failed - $!\n";
4367
4368         my $outfd = new IO::File ($tmpfn, "w") ||
4369             die "unable to write config for VM $vmid\n";
4370
4371         my $cookie = { netcount => 0 };
4372         while (defined(my $line = <$fh>)) {
4373             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4374         }
4375
4376         $fh->close();
4377         $outfd->close();
4378     };
4379
4380     eval {
4381         # enable interrupts
4382         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4383             die "interrupted by signal\n";
4384         };
4385         local $SIG{ALRM} = sub { die "got timeout\n"; };
4386
4387         $oldtimeout = alarm($timeout);
4388
4389         my $parser = sub {
4390             my $line = shift;
4391
4392             print "$line\n";
4393
4394             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
4395                 my ($dev_id, $size, $devname) = ($1, $2, $3);
4396                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
4397             } elsif ($line =~ m/^CTIME: /) {
4398                 # we correctly received the vma config, so we can disable
4399                 # the timeout now for disk allocation (set to 10 minutes, so
4400                 # that we always timeout if something goes wrong)
4401                 alarm(600);
4402                 &$print_devmap();
4403                 print $fifofh "done\n";
4404                 my $tmp = $oldtimeout || 0;
4405                 $oldtimeout = undef;
4406                 alarm($tmp);
4407                 close($fifofh);
4408             }
4409         };
4410
4411         print "restore vma archive: $cmd\n";
4412         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
4413     };
4414     my $err = $@;
4415
4416     alarm($oldtimeout) if $oldtimeout;
4417
4418     unlink $mapfifo;
4419
4420     if ($err) {
4421         rmtree $tmpdir;
4422         unlink $tmpfn;
4423
4424         my $cfg = cfs_read_file('storage.cfg');
4425         foreach my $devname (keys %$devinfo) {
4426             my $volid = $devinfo->{$devname}->{volid};
4427             next if !$volid;
4428             eval {
4429                 if ($volid =~ m|^/|) {
4430                     unlink $volid || die 'unlink failed\n';
4431                 } else {
4432                     PVE::Storage::vdisk_free($cfg, $volid);
4433                 }
4434                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
4435             };
4436             print STDERR "unable to cleanup '$volid' - $@" if $@;
4437         }
4438         die $err;
4439     }
4440
4441     rmtree $tmpdir;
4442
4443     rename($tmpfn, $conffile) ||
4444         die "unable to commit configuration file '$conffile'\n";
4445
4446     PVE::Cluster::cfs_update(); # make sure we read new file
4447
4448     eval { rescan($vmid, 1); };
4449     warn $@ if $@;
4450 }
4451
4452 sub restore_tar_archive {
4453     my ($archive, $vmid, $user, $opts) = @_;
4454
4455     if ($archive ne '-') {
4456         my $firstfile = tar_archive_read_firstfile($archive);
4457         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
4458             if $firstfile ne 'qemu-server.conf';
4459     }
4460
4461     my $storecfg = cfs_read_file('storage.cfg');
4462
4463     # destroy existing data - keep empty config
4464     my $vmcfgfn = PVE::QemuServer::config_file($vmid);
4465     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
4466
4467     my $tocmd = "/usr/lib/qemu-server/qmextract";
4468
4469     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
4470     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
4471     $tocmd .= ' --prealloc' if $opts->{prealloc};
4472     $tocmd .= ' --info' if $opts->{info};
4473
4474     # tar option "xf" does not autodetect compression when read from STDIN,
4475     # so we pipe to zcat
4476     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
4477         PVE::Tools::shellquote("--to-command=$tocmd");
4478
4479     my $tmpdir = "/var/tmp/vzdumptmp$$";
4480     mkpath $tmpdir;
4481
4482     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
4483     local $ENV{VZDUMP_VMID} = $vmid;
4484     local $ENV{VZDUMP_USER} = $user;
4485
4486     my $conffile = config_file($vmid);
4487     my $tmpfn = "$conffile.$$.tmp";
4488
4489     # disable interrupts (always do cleanups)
4490     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4491         print STDERR "got interrupt - ignored\n";
4492     };
4493
4494     eval {
4495         # enable interrupts
4496         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4497             die "interrupted by signal\n";
4498         };
4499
4500         if ($archive eq '-') {
4501             print "extracting archive from STDIN\n";
4502             run_command($cmd, input => "<&STDIN");
4503         } else {
4504             print "extracting archive '$archive'\n";
4505             run_command($cmd);
4506         }
4507
4508         return if $opts->{info};
4509
4510         # read new mapping
4511         my $map = {};
4512         my $statfile = "$tmpdir/qmrestore.stat";
4513         if (my $fd = IO::File->new($statfile, "r")) {
4514             while (defined (my $line = <$fd>)) {
4515                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4516                     $map->{$1} = $2 if $1;
4517                 } else {
4518                     print STDERR "unable to parse line in statfile - $line\n";
4519                 }
4520             }
4521             $fd->close();
4522         }
4523
4524         my $confsrc = "$tmpdir/qemu-server.conf";
4525
4526         my $srcfd = new IO::File($confsrc, "r") ||
4527             die "unable to open file '$confsrc'\n";
4528
4529         my $outfd = new IO::File ($tmpfn, "w") ||
4530             die "unable to write config for VM $vmid\n";
4531
4532         my $cookie = { netcount => 0 };
4533         while (defined (my $line = <$srcfd>)) {
4534             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4535         }
4536
4537         $srcfd->close();
4538         $outfd->close();
4539     };
4540     my $err = $@;
4541
4542     if ($err) {
4543
4544         unlink $tmpfn;
4545
4546         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
4547
4548         die $err;
4549     }
4550
4551     rmtree $tmpdir;
4552
4553     rename $tmpfn, $conffile ||
4554         die "unable to commit configuration file '$conffile'\n";
4555
4556     PVE::Cluster::cfs_update(); # make sure we read new file
4557
4558     eval { rescan($vmid, 1); };
4559     warn $@ if $@;
4560 };
4561
4562
4563 # Internal snapshots
4564
4565 # NOTE: Snapshot create/delete involves several non-atomic
4566 # action, and can take a long time.
4567 # So we try to avoid locking the file and use 'lock' variable
4568 # inside the config file instead.
4569
4570 my $snapshot_copy_config = sub {
4571     my ($source, $dest) = @_;
4572
4573     foreach my $k (keys %$source) {
4574         next if $k eq 'snapshots';
4575         next if $k eq 'snapstate';
4576         next if $k eq 'snaptime';
4577         next if $k eq 'vmstate';
4578         next if $k eq 'lock';
4579         next if $k eq 'digest';
4580         next if $k eq 'description';
4581         next if $k =~ m/^unused\d+$/;
4582
4583         $dest->{$k} = $source->{$k};
4584     }
4585 };
4586
4587 my $snapshot_apply_config = sub {
4588     my ($conf, $snap) = @_;
4589
4590     # copy snapshot list
4591     my $newconf = {
4592         snapshots => $conf->{snapshots},
4593     };
4594
4595     # keep description and list of unused disks
4596     foreach my $k (keys %$conf) {
4597         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
4598         $newconf->{$k} = $conf->{$k};
4599     }
4600
4601     &$snapshot_copy_config($snap, $newconf);
4602
4603     return $newconf;
4604 };
4605
4606 sub foreach_writable_storage {
4607     my ($conf, $func) = @_;
4608
4609     my $sidhash = {};
4610
4611     foreach my $ds (keys %$conf) {
4612         next if !valid_drivename($ds);
4613
4614         my $drive = parse_drive($ds, $conf->{$ds});
4615         next if !$drive;
4616         next if drive_is_cdrom($drive);
4617
4618         my $volid = $drive->{file};
4619
4620         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4621         $sidhash->{$sid} = $sid if $sid;
4622     }
4623
4624     foreach my $sid (sort keys %$sidhash) {
4625         &$func($sid);
4626     }
4627 }
4628
4629 my $alloc_vmstate_volid = sub {
4630     my ($storecfg, $vmid, $conf, $snapname) = @_;
4631
4632     # Note: we try to be smart when selecting a $target storage
4633
4634     my $target;
4635
4636     # search shared storage first
4637     foreach_writable_storage($conf, sub {
4638         my ($sid) = @_;
4639         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4640         return if !$scfg->{shared};
4641
4642         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
4643     });
4644
4645     if (!$target) {
4646         # now search local storage
4647         foreach_writable_storage($conf, sub {
4648             my ($sid) = @_;
4649             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4650             return if $scfg->{shared};
4651
4652             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
4653         });
4654     }
4655
4656     $target = 'local' if !$target;
4657
4658     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
4659     # we abort live save after $conf->{memory}, so we need at max twice that space
4660     my $size = $conf->{memory}*2 + $driver_state_size;
4661
4662     my $name = "vm-$vmid-state-$snapname";
4663     my $scfg = PVE::Storage::storage_config($storecfg, $target);
4664     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
4665     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
4666
4667     return $volid;
4668 };
4669
4670 my $snapshot_prepare = sub {
4671     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
4672
4673     my $snap;
4674
4675     my $updatefn =  sub {
4676
4677         my $conf = load_config($vmid);
4678
4679         die "you can't take a snapshot if it's a template\n"
4680             if is_template($conf);
4681
4682         check_lock($conf);
4683
4684         $conf->{lock} = 'snapshot';
4685
4686         die "snapshot name '$snapname' already used\n"
4687             if defined($conf->{snapshots}->{$snapname});
4688
4689         my $storecfg = PVE::Storage::config();
4690         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
4691
4692         $snap = $conf->{snapshots}->{$snapname} = {};
4693
4694         if ($save_vmstate && check_running($vmid)) {
4695             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
4696         }
4697
4698         &$snapshot_copy_config($conf, $snap);
4699
4700         $snap->{snapstate} = "prepare";
4701         $snap->{snaptime} = time();
4702         $snap->{description} = $comment if $comment;
4703
4704         # always overwrite machine if we save vmstate. This makes sure we
4705         # can restore it later using correct machine type
4706         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
4707
4708         update_config_nolock($vmid, $conf, 1);
4709     };
4710
4711     lock_config($vmid, $updatefn);
4712
4713     return $snap;
4714 };
4715
4716 my $snapshot_commit = sub {
4717     my ($vmid, $snapname) = @_;
4718
4719     my $updatefn = sub {
4720
4721         my $conf = load_config($vmid);
4722
4723         die "missing snapshot lock\n"
4724             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
4725
4726         my $snap = $conf->{snapshots}->{$snapname};
4727
4728         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4729
4730         die "wrong snapshot state\n"
4731             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
4732
4733         delete $snap->{snapstate};
4734         delete $conf->{lock};
4735
4736         my $newconf = &$snapshot_apply_config($conf, $snap);
4737
4738         $newconf->{parent} = $snapname;
4739
4740         update_config_nolock($vmid, $newconf, 1);
4741     };
4742
4743     lock_config($vmid, $updatefn);
4744 };
4745
4746 sub snapshot_rollback {
4747     my ($vmid, $snapname) = @_;
4748
4749     my $snap;
4750
4751     my $prepare = 1;
4752
4753     my $storecfg = PVE::Storage::config();
4754
4755     my $updatefn = sub {
4756
4757         my $conf = load_config($vmid);
4758
4759         die "you can't rollback if vm is a template\n" if is_template($conf);
4760
4761         $snap = $conf->{snapshots}->{$snapname};
4762
4763         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4764
4765         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
4766             if $snap->{snapstate};
4767
4768         if ($prepare) {
4769             check_lock($conf);
4770             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
4771         }
4772
4773         die "unable to rollback vm $vmid: vm is running\n"
4774             if check_running($vmid);
4775
4776         if ($prepare) {
4777             $conf->{lock} = 'rollback';
4778         } else {
4779             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
4780             delete $conf->{lock};
4781         }
4782
4783         my $forcemachine;
4784
4785         if (!$prepare) {
4786             my $has_machine_config = defined($conf->{machine});
4787
4788             # copy snapshot config to current config
4789             $conf = &$snapshot_apply_config($conf, $snap);
4790             $conf->{parent} = $snapname;
4791
4792             # Note: old code did not store 'machine', so we try to be smart
4793             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
4794             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
4795             # we remove the 'machine' configuration if not explicitly specified
4796             # in the original config.
4797             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
4798         }
4799
4800         update_config_nolock($vmid, $conf, 1);
4801
4802         if (!$prepare && $snap->{vmstate}) {
4803             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
4804             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
4805         }
4806     };
4807
4808     lock_config($vmid, $updatefn);
4809
4810     foreach_drive($snap, sub {
4811         my ($ds, $drive) = @_;
4812
4813         return if drive_is_cdrom($drive);
4814
4815         my $volid = $drive->{file};
4816         my $device = "drive-$ds";
4817
4818         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
4819     });
4820
4821     $prepare = 0;
4822     lock_config($vmid, $updatefn);
4823 }
4824
4825 my $savevm_wait = sub {
4826     my ($vmid) = @_;
4827
4828     for(;;) {
4829         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
4830         if (!$stat->{status}) {
4831             die "savevm not active\n";
4832         } elsif ($stat->{status} eq 'active') {
4833             sleep(1);
4834             next;
4835         } elsif ($stat->{status} eq 'completed') {
4836             last;
4837         } else {
4838             die "query-savevm returned status '$stat->{status}'\n";
4839         }
4840     }
4841 };
4842
4843 sub snapshot_create {
4844     my ($vmid, $snapname, $save_vmstate, $freezefs, $comment) = @_;
4845
4846     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
4847
4848     $freezefs = $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
4849
4850     my $drivehash = {};
4851
4852     my $running = check_running($vmid);
4853
4854     eval {
4855         # create internal snapshots of all drives
4856
4857         my $storecfg = PVE::Storage::config();
4858
4859         if ($running) {
4860             if ($snap->{vmstate}) {
4861                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
4862                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
4863                 &$savevm_wait($vmid);
4864             } else {
4865                 vm_mon_cmd($vmid, "savevm-start");
4866             }
4867         };
4868
4869         qga_freezefs($vmid) if $running && $freezefs;
4870
4871         foreach_drive($snap, sub {
4872             my ($ds, $drive) = @_;
4873
4874             return if drive_is_cdrom($drive);
4875
4876             my $volid = $drive->{file};
4877             my $device = "drive-$ds";
4878
4879             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
4880             $drivehash->{$ds} = 1;
4881        });
4882     };
4883     my $err = $@;
4884
4885     eval { qga_unfreezefs($vmid) if $running && $freezefs; };
4886     warn $@ if $@;
4887
4888     eval { vm_mon_cmd($vmid, "savevm-end") if $running; };
4889     warn $@ if $@;
4890
4891     if ($err) {
4892         warn "snapshot create failed: starting cleanup\n";
4893         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
4894         warn $@ if $@;
4895         die $err;
4896     }
4897
4898     &$snapshot_commit($vmid, $snapname);
4899 }
4900
4901 # Note: $drivehash is only set when called from snapshot_create.
4902 sub snapshot_delete {
4903     my ($vmid, $snapname, $force, $drivehash) = @_;
4904
4905     my $prepare = 1;
4906
4907     my $snap;
4908     my $unused = [];
4909
4910     my $unlink_parent = sub {
4911         my ($confref, $new_parent) = @_;
4912
4913         if ($confref->{parent} && $confref->{parent} eq $snapname) {
4914             if ($new_parent) {
4915                 $confref->{parent} = $new_parent;
4916             } else {
4917                 delete $confref->{parent};
4918             }
4919         }
4920     };
4921
4922     my $updatefn =  sub {
4923         my ($remove_drive) = @_;
4924
4925         my $conf = load_config($vmid);
4926
4927         if (!$drivehash) {
4928             check_lock($conf);
4929             die "you can't delete a snapshot if vm is a template\n"
4930                 if is_template($conf);
4931         }
4932
4933         $snap = $conf->{snapshots}->{$snapname};
4934
4935         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4936
4937         # remove parent refs
4938         if (!$prepare) {
4939             &$unlink_parent($conf, $snap->{parent});
4940             foreach my $sn (keys %{$conf->{snapshots}}) {
4941                 next if $sn eq $snapname;
4942                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
4943             }
4944         }
4945
4946         if ($remove_drive) {
4947             if ($remove_drive eq 'vmstate') {
4948                 delete $snap->{$remove_drive};
4949             } else {
4950                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
4951                 my $volid = $drive->{file};
4952                 delete $snap->{$remove_drive};
4953                 add_unused_volume($conf, $volid);
4954             }
4955         }
4956
4957         if ($prepare) {
4958             $snap->{snapstate} = 'delete';
4959         } else {
4960             delete $conf->{snapshots}->{$snapname};
4961             delete $conf->{lock} if $drivehash;
4962             foreach my $volid (@$unused) {
4963                 add_unused_volume($conf, $volid);
4964             }
4965         }
4966
4967         update_config_nolock($vmid, $conf, 1);
4968     };
4969
4970     lock_config($vmid, $updatefn);
4971
4972     # now remove vmstate file
4973
4974     my $storecfg = PVE::Storage::config();
4975
4976     if ($snap->{vmstate}) {
4977         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
4978         if (my $err = $@) {
4979             die $err if !$force;
4980             warn $err;
4981         }
4982         # save changes (remove vmstate from snapshot)
4983         lock_config($vmid, $updatefn, 'vmstate') if !$force;
4984     };
4985
4986     # now remove all internal snapshots
4987     foreach_drive($snap, sub {
4988         my ($ds, $drive) = @_;
4989
4990         return if drive_is_cdrom($drive);
4991
4992         my $volid = $drive->{file};
4993         my $device = "drive-$ds";
4994
4995         if (!$drivehash || $drivehash->{$ds}) {
4996             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
4997             if (my $err = $@) {
4998                 die $err if !$force;
4999                 warn $err;
5000             }
5001         }
5002
5003         # save changes (remove drive fron snapshot)
5004         lock_config($vmid, $updatefn, $ds) if !$force;
5005         push @$unused, $volid;
5006     });
5007
5008     # now cleanup config
5009     $prepare = 0;
5010     lock_config($vmid, $updatefn);
5011 }
5012
5013 sub has_feature {
5014     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5015
5016     my $err;
5017     foreach_drive($conf, sub {
5018         my ($ds, $drive) = @_;
5019
5020         return if drive_is_cdrom($drive);
5021         my $volid = $drive->{file};
5022         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5023     });
5024
5025     return $err ? 0 : 1;
5026 }
5027
5028 sub template_create {
5029     my ($vmid, $conf, $disk) = @_;
5030
5031     my $storecfg = PVE::Storage::config();
5032
5033     foreach_drive($conf, sub {
5034         my ($ds, $drive) = @_;
5035
5036         return if drive_is_cdrom($drive);
5037         return if $disk && $ds ne $disk;
5038
5039         my $volid = $drive->{file};
5040         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5041
5042         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5043         $drive->{file} = $voliddst;
5044         $conf->{$ds} = print_drive($vmid, $drive);
5045         update_config_nolock($vmid, $conf, 1);
5046     });
5047 }
5048
5049 sub is_template {
5050     my ($conf) = @_;
5051
5052     return 1 if defined $conf->{template} && $conf->{template} == 1;
5053 }
5054
5055 sub qemu_img_convert {
5056     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5057
5058     my $storecfg = PVE::Storage::config();
5059     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5060     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5061
5062     if ($src_storeid && $dst_storeid) {
5063         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5064         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5065
5066         my $src_format = qemu_img_format($src_scfg, $src_volname);
5067         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5068
5069         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5070         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5071
5072         my $cmd = [];
5073         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5074         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5075         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5076
5077         my $parser = sub {
5078             my $line = shift;
5079             if($line =~ m/\((\S+)\/100\%\)/){
5080                 my $percent = $1;
5081                 my $transferred = int($size * $percent / 100);
5082                 my $remaining = $size - $transferred;
5083
5084                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5085             }
5086
5087         };
5088
5089         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5090         my $err = $@;
5091         die "copy failed: $err" if $err;
5092     }
5093 }
5094
5095 sub qemu_img_format {
5096     my ($scfg, $volname) = @_;
5097
5098     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
5099         return $1;
5100     } elsif ($scfg->{type} eq 'iscsi') {
5101         return "host_device";
5102     } else {
5103         return "raw";
5104     }
5105 }
5106
5107 sub qemu_drive_mirror {
5108     my ($vmid, $drive, $dst_volid, $vmiddst, $maxwait) = @_;
5109
5110     my $count = 1;
5111     my $old_len = 0;
5112     my $frozen = undef;
5113
5114     my $storecfg = PVE::Storage::config();
5115     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5116
5117     if ($dst_storeid) {
5118         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5119
5120         my $format;
5121         if ($dst_volname =~ m/\.(raw|qcow2)$/){
5122             $format = $1;
5123         }
5124
5125         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5126
5127         if ($format) {
5128             #fixme : sometime drive-mirror timeout, but works fine after.
5129             # (I have see the problem with big volume > 200GB), so we need to eval
5130             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
5131                               sync => "full", target => $dst_path, format => $format); };
5132         } else {
5133             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
5134                               sync => "full", target => $dst_path); };
5135         }
5136
5137         eval {
5138             while (1) {
5139                 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5140                 my $stat = @$stats[0];
5141                 die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5142                 die "error job is not mirroring" if $stat->{type} ne "mirror";
5143
5144                 my $transferred = $stat->{offset};
5145                 my $total = $stat->{len};
5146                 my $remaining = $total - $transferred;
5147                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5148
5149                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent %\n";
5150
5151                 last if ($stat->{len} == $stat->{offset});
5152                 if ($old_len == $stat->{offset}) {
5153                     if ($maxwait && $count > $maxwait) {
5154                     # if writes to disk occurs the disk needs to be freezed
5155                     # to be able to complete the migration
5156                         vm_suspend($vmid,1);
5157                         $count = 0;
5158                         $frozen = 1;
5159                     } else {
5160                         $count++ unless $frozen;
5161                     }
5162                 } elsif ($frozen) {
5163                     vm_resume($vmid,1);
5164                     $count = 0;
5165                 }
5166                 $old_len = $stat->{offset};
5167                 sleep 1;
5168             }
5169
5170             if ($vmiddst == $vmid) {
5171                 # switch the disk if source and destination are on the same guest
5172                 vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive");
5173             }
5174         };
5175         if (my $err = $@) {
5176             eval { vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive"); };
5177             die "mirroring error: $err";
5178         }
5179
5180         if ($vmiddst != $vmid) {
5181             # if we clone a disk for a new target vm, we don't switch the disk
5182             vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5183         }
5184     }
5185 }
5186
5187 sub clone_disk {
5188     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5189         $newvmid, $storage, $format, $full, $newvollist) = @_;
5190
5191     my $newvolid;
5192
5193     if (!$full) {
5194         print "create linked clone of drive $drivename ($drive->{file})\n";
5195         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5196         push @$newvollist, $newvolid;
5197     } else {
5198         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5199         $storeid = $storage if $storage;
5200
5201         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5202         if (!$format) {
5203             $format = $drive->{format} || $defFormat;
5204         }
5205
5206         # test if requested format is supported - else use default
5207         my $supported = grep { $_ eq $format } @$validFormats;
5208         $format = $defFormat if !$supported;
5209
5210         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5211
5212         print "create full clone of drive $drivename ($drive->{file})\n";
5213         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5214         push @$newvollist, $newvolid;
5215
5216         if (!$running || $snapname) {
5217             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5218         } else {
5219             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5220         }
5221     }
5222
5223     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5224
5225     my $disk = $drive;
5226     $disk->{format} = undef;
5227     $disk->{file} = $newvolid;
5228     $disk->{size} = $size;
5229
5230     return $disk;
5231 }
5232
5233 # this only works if VM is running
5234 sub get_current_qemu_machine {
5235     my ($vmid) = @_;
5236
5237     my $cmd = { execute => 'query-machines', arguments => {} };
5238     my $res = PVE::QemuServer::vm_qmp_command($vmid, $cmd);
5239
5240     my ($current, $default);
5241     foreach my $e (@$res) {
5242         $default = $e->{name} if $e->{'is-default'};
5243         $current = $e->{name} if $e->{'is-current'};
5244     }
5245
5246     # fallback to the default machine if current is not supported by qemu
5247     return $current || $default || 'pc';
5248 }
5249
5250 sub lspci {
5251
5252     my $devices = {};
5253
5254     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5255             my (undef, $id, $function) = @_;
5256             my $res = { id => $id, function => $function};
5257             push @{$devices->{$id}}, $res;
5258     });
5259
5260     return $devices;
5261 }
5262
5263 1;