PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     onboot => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Specifies whether a VM will be started during system bootup.",
 159         default => 0,
 160     },
 161     autostart => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Automatic restart after crash (currently ignored).",
 165         default => 0,
 166     },
 167     hotplug => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Allow hotplug for disk and network device",
 171         default => 0,
 172     },
 173     reboot => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 177         default => 1,
 178     },
 179     lock => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Lock/unlock the VM.",
 183         enum => [qw(migrate backup snapshot rollback)],
 184     },
 185     cpulimit => {
 186         optional => 1,
 187         type => 'integer',
 188         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 189         minimum => 0,
 190         default => 0,
 191     },
 192     cpuunits => {
 193         optional => 1,
 194         type => 'integer',
 195         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 196         minimum => 0,
 197         maximum => 500000,
 198         default => 1000,
 199     },
 200     memory => {
 201         optional => 1,
 202         type => 'integer',
 203         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 204         minimum => 16,
 205         default => 512,
 206     },
 207     balloon => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 211         minimum => 0,
 212     },
 213     shares => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 217         minimum => 0,
 218         maximum => 50000,
 219         default => 1000,
 220     },
 221     keyboard => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 225         enum => PVE::Tools::kvmkeymaplist(),
 226         default => 'en-us',
 227     },
 228     name => {
 229         optional => 1,
 230         type => 'string', format => 'dns-name',
 231         description => "Set a name for the VM. Only used on the configuration web interface.",
 232     },
 233     scsihw => {
 234         optional => 1,
 235         type => 'string',
 236         description => "scsi controller model",
 237         enum => [qw(lsi lsi53c810 virtio-scsi-pci megasas pvscsi)],
 238         default => 'lsi',
 239     },
 240     description => {
 241         optional => 1,
 242         type => 'string',
 243         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 244     },
 245     ostype => {
 246         optional => 1,
 247         type => 'string',
 248         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 249         description => <<EODESC,
 250 Used to enable special optimization/features for specific
 251 operating systems:
 252
 253 other  => unspecified OS
 254 wxp    => Microsoft Windows XP
 255 w2k    => Microsoft Windows 2000
 256 w2k3   => Microsoft Windows 2003
 257 w2k8   => Microsoft Windows 2008
 258 wvista => Microsoft Windows Vista
 259 win7   => Microsoft Windows 7
 260 win8   => Microsoft Windows 8/2012
 261 l24    => Linux 2.4 Kernel
 262 l26    => Linux 2.6/3.X Kernel
 263 solaris => solaris/opensolaris/openindiania kernel
 264
 265 other|l24|l26|solaris                       ... no special behaviour
 266 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 267 EODESC
 268     },
 269     boot => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 273         pattern => '[acdn]{1,4}',
 274         default => 'cdn',
 275     },
 276     bootdisk => {
 277         optional => 1,
 278         type => 'string', format => 'pve-qm-bootdisk',
 279         description => "Enable booting from specified disk.",
 280         pattern => '(ide|sata|scsi|virtio)\d+',
 281     },
 282     smp => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "The number of CPUs. Please use option -sockets instead.",
 286         minimum => 1,
 287         default => 1,
 288     },
 289     sockets => {
 290         optional => 1,
 291         type => 'integer',
 292         description => "The number of CPU sockets.",
 293         minimum => 1,
 294         default => 1,
 295     },
 296     cores => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of cores per socket.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     maxcpus => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "Maximum cpus for hotplug.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     acpi => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable ACPI.",
 314         default => 1,
 315     },
 316     agent => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable Qemu GuestAgent.",
 320         default => 0,
 321     },
 322     kvm => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable KVM hardware virtualization.",
 326         default => 1,
 327     },
 328     tdf => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable time drift fix.",
 332         default => 0,
 333     },
 334     localtime => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 338     },
 339     freeze => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 343     },
 344     vga => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 348         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 349     },
 350     watchdog => {
 351         optional => 1,
 352         type => 'string', format => 'pve-qm-watchdog',
 353         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 354         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 355     },
 356     startdate => {
 357         optional => 1,
 358         type => 'string',
 359         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 360         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 361         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 362         default => 'now',
 363     },
 364     startup => {
 365         optional => 1,
 366         type => 'string', format => 'pve-qm-startup',
 367         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 368         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 369     },
 370     template => {
 371         optional => 1,
 372         type => 'boolean',
 373         description => "Enable/disable Template.",
 374         default => 0,
 375     },
 376     args => {
 377         optional => 1,
 378         type => 'string',
 379         description => <<EODESCR,
 380 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 381
 382 args: -no-reboot -no-hpet
 383 EODESCR
 384     },
 385     tablet => {
 386         optional => 1,
 387         type => 'boolean',
 388         default => 1,
 389         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 390     },
 391     migrate_speed => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 395         minimum => 0,
 396         default => 0,
 397     },
 398     migrate_downtime => {
 399         optional => 1,
 400         type => 'number',
 401         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 402         minimum => 0,
 403         default => 0.1,
 404     },
 405     cdrom => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-drive',
 408         typetext => 'volume',
 409         description => "This is an alias for option -ide2",
 410     },
 411     cpu => {
 412         optional => 1,
 413         description => "Emulated CPU type.",
 414         type => 'string',
 415         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge Haswell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 416         default => 'kvm64',
 417     },
 418     parent => get_standard_option('pve-snapshot-name', {
 419         optional => 1,
 420         description => "Parent snapshot name. This is used internally, and should not be modified.",
 421     }),
 422     snaptime => {
 423         optional => 1,
 424         description => "Timestamp for snapshots.",
 425         type => 'integer',
 426         minimum => 0,
 427     },
 428     vmstate => {
 429         optional => 1,
 430         type => 'string', format => 'pve-volume-id',
 431         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 432     },
 433     machine => {
 434         description => "Specific the Qemu machine type.",
 435         type => 'string',
 436         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 437         maxLength => 40,
 438         optional => 1,
 439     },
 440 };
 441
 442 # what about other qemu settings ?
 443 #cpu => 'string',
 444 #machine => 'string',
 445 #fda => 'file',
 446 #fdb => 'file',
 447 #mtdblock => 'file',
 448 #sd => 'file',
 449 #pflash => 'file',
 450 #snapshot => 'bool',
 451 #bootp => 'file',
 452 ##tftp => 'dir',
 453 ##smb => 'dir',
 454 #kernel => 'file',
 455 #append => 'string',
 456 #initrd => 'file',
 457 ##soundhw => 'string',
 458
 459 while (my ($k, $v) = each %$confdesc) {
 460     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 461 }
 462
 463 my $MAX_IDE_DISKS = 4;
 464 my $MAX_SCSI_DISKS = 14;
 465 my $MAX_VIRTIO_DISKS = 16;
 466 my $MAX_SATA_DISKS = 6;
 467 my $MAX_USB_DEVICES = 5;
 468 my $MAX_NETS = 32;
 469 my $MAX_UNUSED_DISKS = 8;
 470 my $MAX_HOSTPCI_DEVICES = 2;
 471 my $MAX_SERIAL_PORTS = 4;
 472 my $MAX_PARALLEL_PORTS = 3;
 473
 474 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 475                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3'];
 476 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 477
 478 my $netdesc = {
 479     optional => 1,
 480     type => 'string', format => 'pve-qm-net',
 481     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>][,tag=<vlanid>][,firewall=0|1]",
 482     description => <<EODESCR,
 483 Specify network devices.
 484
 485 MODEL is one of: $nic_model_list_txt
 486
 487 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 488 automatically generated if not specified.
 489
 490 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 491
 492 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 493
 494 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 495
 496 10.0.2.2   Gateway
 497 10.0.2.3   DNS Server
 498 10.0.2.4   SMB Server
 499
 500 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 501
 502 EODESCR
 503 };
 504 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 505
 506 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 507     $confdesc->{"net$i"} = $netdesc;
 508 }
 509
 510 my $drivename_hash;
 511
 512 my $idedesc = {
 513     optional => 1,
 514     type => 'string', format => 'pve-qm-drive',
 515     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 516     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 517 };
 518 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 519
 520 my $scsidesc = {
 521     optional => 1,
 522     type => 'string', format => 'pve-qm-drive',
 523     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 524     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 527
 528 my $satadesc = {
 529     optional => 1,
 530     type => 'string', format => 'pve-qm-drive',
 531     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 532     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 533 };
 534 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 535
 536 my $virtiodesc = {
 537     optional => 1,
 538     type => 'string', format => 'pve-qm-drive',
 539     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 540     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 541 };
 542 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 543
 544 my $usbdesc = {
 545     optional => 1,
 546     type => 'string', format => 'pve-qm-usb-device',
 547     typetext => 'host=HOSTUSBDEVICE|spice',
 548     description => <<EODESCR,
 549 Configure an USB device (n is 0 to 4). This can be used to
 550 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 551
 552 'bus-port(.port)*' (decimal numbers) or
 553 'vendor_id:product_id' (hexadeciaml numbers)
 554
 555 You can use the 'lsusb -t' command to list existing usb devices.
 556
 557 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 558
 559 The value 'spice' can be used to add a usb redirection devices for spice.
 560
 561 EODESCR
 562 };
 563 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 564
 565 my $hostpcidesc = {
 566         optional => 1,
 567         type => 'string', format => 'pve-qm-hostpci',
 568         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off]",
 569         description => <<EODESCR,
 570 Map host pci devices. HOSTPCIDEVICE syntax is:
 571
 572 'bus:dev.func' (hexadecimal numbers)
 573
 574 You can us the 'lspci' command to list existing pci devices.
 575
 576 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 577
 578 The 'driver' option is currently ignored.
 579
 580 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 581
 582 Experimental: user reported problems with this option.
 583 EODESCR
 584 };
 585 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 586
 587 my $serialdesc = {
 588         optional => 1,
 589         type => 'string',
 590         pattern => '(/dev/ttyS\d+|socket)',
 591         description =>  <<EODESCR,
 592 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device, or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 593
 594 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 595
 596 Experimental: user reported problems with this option.
 597 EODESCR
 598 };
 599
 600 my $paralleldesc= {
 601         optional => 1,
 602         type => 'string',
 603         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 604         description =>  <<EODESCR,
 605 Map host parallel devices (n is 0 to 2).
 606
 607 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 608
 609 Experimental: user reported problems with this option.
 610 EODESCR
 611 };
 612
 613 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 614     $confdesc->{"parallel$i"} = $paralleldesc;
 615 }
 616
 617 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 618     $confdesc->{"serial$i"} = $serialdesc;
 619 }
 620
 621 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 622     $confdesc->{"hostpci$i"} = $hostpcidesc;
 623 }
 624
 625 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 626     $drivename_hash->{"ide$i"} = 1;
 627     $confdesc->{"ide$i"} = $idedesc;
 628 }
 629
 630 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 631     $drivename_hash->{"sata$i"} = 1;
 632     $confdesc->{"sata$i"} = $satadesc;
 633 }
 634
 635 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 636     $drivename_hash->{"scsi$i"} = 1;
 637     $confdesc->{"scsi$i"} = $scsidesc ;
 638 }
 639
 640 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 641     $drivename_hash->{"virtio$i"} = 1;
 642     $confdesc->{"virtio$i"} = $virtiodesc;
 643 }
 644
 645 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 646     $confdesc->{"usb$i"} = $usbdesc;
 647 }
 648
 649 my $unuseddesc = {
 650     optional => 1,
 651     type => 'string', format => 'pve-volume-id',
 652     description => "Reference to unused volumes.",
 653 };
 654
 655 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 656     $confdesc->{"unused$i"} = $unuseddesc;
 657 }
 658
 659 my $kvm_api_version = 0;
 660
 661 sub kvm_version {
 662
 663     return $kvm_api_version if $kvm_api_version;
 664
 665     my $fh = IO::File->new("</dev/kvm") ||
 666         return 0;
 667
 668     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 669         $kvm_api_version = $v;
 670     }
 671
 672     $fh->close();
 673
 674     return  $kvm_api_version;
 675 }
 676
 677 my $kvm_user_version;
 678
 679 sub kvm_user_version {
 680
 681     return $kvm_user_version if $kvm_user_version;
 682
 683     $kvm_user_version = 'unknown';
 684
 685     my $tmp = `kvm -help 2>/dev/null`;
 686
 687     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 688         $kvm_user_version = $2;
 689     }
 690
 691     return $kvm_user_version;
 692
 693 }
 694
 695 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 696
 697 sub disknames {
 698     # order is important - used to autoselect boot disk
 699     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 700             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 701             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 702             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 703 }
 704
 705 sub valid_drivename {
 706     my $dev = shift;
 707
 708     return defined($drivename_hash->{$dev});
 709 }
 710
 711 sub option_exists {
 712     my $key = shift;
 713     return defined($confdesc->{$key});
 714 }
 715
 716 sub nic_models {
 717     return $nic_model_list;
 718 }
 719
 720 sub os_list_description {
 721
 722     return {
 723         other => 'Other',
 724         wxp => 'Windows XP',
 725         w2k => 'Windows 2000',
 726         w2k3 =>, 'Windows 2003',
 727         w2k8 => 'Windows 2008',
 728         wvista => 'Windows Vista',
 729         win7 => 'Windows 7',
 730         win8 => 'Windows 8/2012',
 731         l24 => 'Linux 2.4',
 732         l26 => 'Linux 2.6',
 733     };
 734 }
 735
 736 my $cdrom_path;
 737
 738 sub get_cdrom_path {
 739
 740     return  $cdrom_path if $cdrom_path;
 741
 742     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 743     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 744     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 745 }
 746
 747 sub get_iso_path {
 748     my ($storecfg, $vmid, $cdrom) = @_;
 749
 750     if ($cdrom eq 'cdrom') {
 751         return get_cdrom_path();
 752     } elsif ($cdrom eq 'none') {
 753         return '';
 754     } elsif ($cdrom =~ m|^/|) {
 755         return $cdrom;
 756     } else {
 757         return PVE::Storage::path($storecfg, $cdrom);
 758     }
 759 }
 760
 761 # try to convert old style file names to volume IDs
 762 sub filename_to_volume_id {
 763     my ($vmid, $file, $media) = @_;
 764
 765     if (!($file eq 'none' || $file eq 'cdrom' ||
 766           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 767
 768         return undef if $file =~ m|/|;
 769
 770         if ($media && $media eq 'cdrom') {
 771             $file = "local:iso/$file";
 772         } else {
 773             $file = "local:$vmid/$file";
 774         }
 775     }
 776
 777     return $file;
 778 }
 779
 780 sub verify_media_type {
 781     my ($opt, $vtype, $media) = @_;
 782
 783     return if !$media;
 784
 785     my $etype;
 786     if ($media eq 'disk') {
 787         $etype = 'images';
 788     } elsif ($media eq 'cdrom') {
 789         $etype = 'iso';
 790     } else {
 791         die "internal error";
 792     }
 793
 794     return if ($vtype eq $etype);
 795
 796     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 797 }
 798
 799 sub cleanup_drive_path {
 800     my ($opt, $storecfg, $drive) = @_;
 801
 802     # try to convert filesystem paths to volume IDs
 803
 804     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 805         ($drive->{file} !~ m|^/dev/.+|) &&
 806         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 807         ($drive->{file} !~ m/^\d+$/)) {
 808         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 809         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 810         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 811         verify_media_type($opt, $vtype, $drive->{media});
 812         $drive->{file} = $volid;
 813     }
 814
 815     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 816 }
 817
 818 sub create_conf_nolock {
 819     my ($vmid, $settings) = @_;
 820
 821     my $filename = config_file($vmid);
 822
 823     die "configuration file '$filename' already exists\n" if -f $filename;
 824
 825     my $defaults = load_defaults();
 826
 827     $settings->{name} = "vm$vmid" if !$settings->{name};
 828     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 829
 830     my $data = '';
 831     foreach my $opt (keys %$settings) {
 832         next if !$confdesc->{$opt};
 833
 834         my $value = $settings->{$opt};
 835         next if !$value;
 836
 837         $data .= "$opt: $value\n";
 838     }
 839
 840     PVE::Tools::file_set_contents($filename, $data);
 841 }
 842
 843 my $parse_size = sub {
 844     my ($value) = @_;
 845
 846     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 847     my ($size, $unit) = ($1, $3);
 848     if ($unit) {
 849         if ($unit eq 'K') {
 850             $size = $size * 1024;
 851         } elsif ($unit eq 'M') {
 852             $size = $size * 1024 * 1024;
 853         } elsif ($unit eq 'G') {
 854             $size = $size * 1024 * 1024 * 1024;
 855         }
 856     }
 857     return int($size);
 858 };
 859
 860 my $format_size = sub {
 861     my ($size) = @_;
 862
 863     $size = int($size);
 864
 865     my $kb = int($size/1024);
 866     return $size if $kb*1024 != $size;
 867
 868     my $mb = int($kb/1024);
 869     return "${kb}K" if $mb*1024 != $kb;
 870
 871     my $gb = int($mb/1024);
 872     return "${mb}M" if $gb*1024 != $mb;
 873
 874     return "${gb}G";
 875 };
 876
 877 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 878 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 879 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 880 #        [,aio=native|threads][,discard=ignore|on]
 881
 882 sub parse_drive {
 883     my ($key, $data) = @_;
 884
 885     my $res = {};
 886
 887     # $key may be undefined - used to verify JSON parameters
 888     if (!defined($key)) {
 889         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 890         $res->{index} = 0;
 891     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 892         $res->{interface} = $1;
 893         $res->{index} = $2;
 894     } else {
 895         return undef;
 896     }
 897
 898     foreach my $p (split (/,/, $data)) {
 899         next if $p =~ m/^\s*$/;
 900
 901         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard)=(.+)$/) {
 902             my ($k, $v) = ($1, $2);
 903
 904             $k = 'file' if $k eq 'volume';
 905
 906             return undef if defined $res->{$k};
 907
 908             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 909                 return undef if !$v || $v !~ m/^\d+/;
 910                 $k = "m$k";
 911                 $v = sprintf("%.3f", $v / (1024*1024));
 912             }
 913             $res->{$k} = $v;
 914         } else {
 915             if (!$res->{file} && $p !~ m/=/) {
 916                 $res->{file} = $p;
 917             } else {
 918                 return undef;
 919             }
 920         }
 921     }
 922
 923     return undef if !$res->{file};
 924
 925     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 926         $res->{format} = $1;
 927     }
 928
 929     return undef if $res->{cache} &&
 930         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 931     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 932     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 933     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 934     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 935     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 936     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 937     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 938     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 939     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 940     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 941     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 942     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 943
 944     return undef if $res->{mbps_rd} && $res->{mbps};
 945     return undef if $res->{mbps_wr} && $res->{mbps};
 946
 947     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 949     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 950     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 951     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 952     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 953
 954     return undef if $res->{iops_rd} && $res->{iops};
 955     return undef if $res->{iops_wr} && $res->{iops};
 956
 957
 958     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 959     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 960     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 961     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 962     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 963     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 964
 965
 966     if ($res->{size}) {
 967         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 968     }
 969
 970     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 971         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 972         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 973         return undef if $res->{interface} eq 'virtio';
 974     }
 975
 976     # rerror does not work with scsi drives
 977     if ($res->{rerror}) {
 978         return undef if $res->{interface} eq 'scsi';
 979     }
 980
 981     return $res;
 982 }
 983
 984 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 985
 986 sub print_drive {
 987     my ($vmid, $drive) = @_;
 988
 989     my $opts = '';
 990     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup') {
 991         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 992     }
 993
 994     if ($drive->{size}) {
 995         $opts .= ",size=" . &$format_size($drive->{size});
 996     }
 997
 998     return "$drive->{file}$opts";
 999 }
1000
1001 sub scsi_inquiry {
1002     my($fh, $noerr) = @_;
1003
1004     my $SG_IO = 0x2285;
1005     my $SG_GET_VERSION_NUM = 0x2282;
1006
1007     my $versionbuf = "\x00" x 8;
1008     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1009     if (!$ret) {
1010         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1011         return undef;
1012     }
1013     my $version = unpack("I", $versionbuf);
1014     if ($version < 30000) {
1015         die "scsi generic interface too old\n"  if !$noerr;
1016         return undef;
1017     }
1018
1019     my $buf = "\x00" x 36;
1020     my $sensebuf = "\x00" x 8;
1021     my $cmd = pack("C x3 C x1", 0x12, 36);
1022
1023     # see /usr/include/scsi/sg.h
1024     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1025
1026     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1027                       length($sensebuf), 0, length($buf), $buf,
1028                       $cmd, $sensebuf, 6000);
1029
1030     $ret = ioctl($fh, $SG_IO, $packet);
1031     if (!$ret) {
1032         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1033         return undef;
1034     }
1035
1036     my @res = unpack($sg_io_hdr_t, $packet);
1037     if ($res[17] || $res[18]) {
1038         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1039         return undef;
1040     }
1041
1042     my $res = {};
1043     (my $byte0, my $byte1, $res->{vendor},
1044      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1045
1046     $res->{removable} = $byte1 & 128 ? 1 : 0;
1047     $res->{type} = $byte0 & 31;
1048
1049     return $res;
1050 }
1051
1052 sub path_is_scsi {
1053     my ($path) = @_;
1054
1055     my $fh = IO::File->new("+<$path") || return undef;
1056     my $res = scsi_inquiry($fh, 1);
1057     close($fh);
1058
1059     return $res;
1060 }
1061
1062 sub print_drivedevice_full {
1063     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1064
1065     my $device = '';
1066     my $maxdev = 0;
1067
1068     if ($drive->{interface} eq 'virtio') {
1069         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1070         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1071     } elsif ($drive->{interface} eq 'scsi') {
1072         $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
1073         my $controller = int($drive->{index} / $maxdev);
1074         my $unit = $drive->{index} % $maxdev;
1075         my $devicetype = 'hd';
1076         my $path = '';
1077         if (drive_is_cdrom($drive)) {
1078               $devicetype = 'cd';
1079         } else {
1080               if ($drive->{file} =~ m|^/|) {
1081                   $path = $drive->{file};
1082               } else {
1083                   $path = PVE::Storage::path($storecfg, $drive->{file});
1084               }
1085
1086               if($path =~ m/^iscsi\:\/\//){
1087                   $devicetype = 'generic';
1088               } else {
1089                   if (my $info = path_is_scsi($path)) {
1090                       if ($info->{type} == 0) {
1091                           $devicetype = 'block';
1092                       } elsif ($info->{type} == 1) { # tape
1093                           $devicetype = 'generic';
1094                       }
1095                   }
1096               }
1097          }
1098
1099         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1100             $device = "scsi-$devicetype,bus=scsihw$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1101         } else {
1102             $device = "scsi-$devicetype,bus=scsihw$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1103         }
1104
1105     } elsif ($drive->{interface} eq 'ide'){
1106         $maxdev = 2;
1107         my $controller = int($drive->{index} / $maxdev);
1108         my $unit = $drive->{index} % $maxdev;
1109         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1110
1111         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1112     } elsif ($drive->{interface} eq 'sata'){
1113         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1114         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1115         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1116     } elsif ($drive->{interface} eq 'usb') {
1117         die "implement me";
1118         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1119     } else {
1120         die "unsupported interface type";
1121     }
1122
1123     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1124
1125     return $device;
1126 }
1127
1128 sub get_initiator_name {
1129     my $initiator;
1130
1131     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1132     while (defined(my $line = <$fh>)) {
1133         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1134         $initiator = $1;
1135         last;
1136     }
1137     $fh->close();
1138
1139     return $initiator;
1140 }
1141
1142 sub print_drive_full {
1143     my ($storecfg, $vmid, $drive) = @_;
1144
1145     my $opts = '';
1146     foreach my $o (@qemu_drive_options) {
1147         next if $o eq 'bootindex';
1148         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1149     }
1150
1151     foreach my $o (qw(bps bps_rd bps_wr)) {
1152         my $v = $drive->{"m$o"};
1153         $opts .= ",$o=" . int($v*1024*1024) if $v;
1154     }
1155
1156     # use linux-aio by default (qemu default is threads)
1157     $opts .= ",aio=native" if !$drive->{aio};
1158
1159     my $path;
1160     my $volid = $drive->{file};
1161     if (drive_is_cdrom($drive)) {
1162         $path = get_iso_path($storecfg, $vmid, $volid);
1163     } else {
1164         if ($volid =~ m|^/|) {
1165             $path = $volid;
1166         } else {
1167             $path = PVE::Storage::path($storecfg, $volid);
1168         }
1169     }
1170
1171     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1172
1173     my $pathinfo = $path ? "file=$path," : '';
1174
1175     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1176 }
1177
1178 sub print_netdevice_full {
1179     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1180
1181     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1182
1183     my $device = $net->{model};
1184     if ($net->{model} eq 'virtio') {
1185          $device = 'virtio-net-pci';
1186      };
1187
1188     # qemu > 0.15 always try to boot from network - we disable that by
1189     # not loading the pxe rom file
1190     my $extra = ($bootorder !~ m/n/) ? "romfile=," : '';
1191     my $pciaddr = print_pci_addr("$netid", $bridges);
1192     my $tmpstr = "$device,${extra}mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1193     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1194         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1195         my $vectors = $net->{queues} * 2 + 2;
1196         $tmpstr .= ",vectors=$vectors,mq=on";
1197     }
1198     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1199     return $tmpstr;
1200 }
1201
1202 sub print_netdev_full {
1203     my ($vmid, $conf, $net, $netid) = @_;
1204
1205     my $i = '';
1206     if ($netid =~ m/^net(\d+)$/) {
1207         $i = int($1);
1208     }
1209
1210     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1211
1212     my $ifname = "tap${vmid}i$i";
1213
1214     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1215     die "interface name '$ifname' is too long (max 15 character)\n"
1216         if length($ifname) >= 16;
1217
1218     my $vhostparam = '';
1219     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1220
1221     my $vmname = $conf->{name} || "vm$vmid";
1222
1223     my $netdev = "";
1224
1225     if ($net->{bridge}) {
1226         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1227     } else {
1228         $netdev = "type=user,id=$netid,hostname=$vmname";
1229     }
1230
1231     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1232
1233     return $netdev;
1234 }
1235
1236 sub drive_is_cdrom {
1237     my ($drive) = @_;
1238
1239     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1240
1241 }
1242
1243 sub parse_hostpci {
1244     my ($value) = @_;
1245
1246     return undef if !$value;
1247
1248
1249     my @list = split(/,/, $value);
1250     my $found;
1251
1252     my $res = {};
1253     foreach my $kv (@list) {
1254
1255         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2}\.[a-f0-9])$/) {
1256             $found = 1;
1257             $res->{pciid} = $2;
1258         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1259             $res->{driver} = $1;
1260         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1261             $res->{rombar} = $1;
1262         } else {
1263             warn "unknown hostpci setting '$kv'\n";
1264         }
1265     }
1266
1267     return undef if !$found;
1268
1269     return $res;
1270 }
1271
1272 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1273 sub parse_net {
1274     my ($data) = @_;
1275
1276     my $res = {};
1277
1278     foreach my $kvp (split(/,/, $data)) {
1279
1280         if ($kvp =~ m/^(ne2k_pci|e1000|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1281             my $model = lc($1);
1282             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1283             $res->{model} = $model;
1284             $res->{macaddr} = $mac;
1285         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1286             $res->{bridge} = $1;
1287         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1288             $res->{queues} = $1;
1289         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1290             $res->{rate} = $1;
1291         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1292             $res->{tag} = $1;
1293         } elsif ($kvp =~ m/^firewall=(\d+)$/) {
1294             $res->{firewall} = $1;
1295         } else {
1296             return undef;
1297         }
1298
1299     }
1300
1301     return undef if !$res->{model};
1302
1303     return $res;
1304 }
1305
1306 sub print_net {
1307     my $net = shift;
1308
1309     my $res = "$net->{model}";
1310     $res .= "=$net->{macaddr}" if $net->{macaddr};
1311     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1312     $res .= ",rate=$net->{rate}" if $net->{rate};
1313     $res .= ",tag=$net->{tag}" if $net->{tag};
1314     $res .= ",firewall=$net->{firewall}" if $net->{firewall};
1315
1316     return $res;
1317 }
1318
1319 sub add_random_macs {
1320     my ($settings) = @_;
1321
1322     foreach my $opt (keys %$settings) {
1323         next if $opt !~ m/^net(\d+)$/;
1324         my $net = parse_net($settings->{$opt});
1325         next if !$net;
1326         $settings->{$opt} = print_net($net);
1327     }
1328 }
1329
1330 sub add_unused_volume {
1331     my ($config, $volid) = @_;
1332
1333     my $key;
1334     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1335         my $test = "unused$ind";
1336         if (my $vid = $config->{$test}) {
1337             return if $vid eq $volid; # do not add duplicates
1338         } else {
1339             $key = $test;
1340         }
1341     }
1342
1343     die "To many unused volume - please delete them first.\n" if !$key;
1344
1345     $config->{$key} = $volid;
1346
1347     return $key;
1348 }
1349
1350 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1351 sub verify_bootdisk {
1352     my ($value, $noerr) = @_;
1353
1354     return $value if valid_drivename($value);
1355
1356     return undef if $noerr;
1357
1358     die "invalid boot disk '$value'\n";
1359 }
1360
1361 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1362 sub verify_net {
1363     my ($value, $noerr) = @_;
1364
1365     return $value if parse_net($value);
1366
1367     return undef if $noerr;
1368
1369     die "unable to parse network options\n";
1370 }
1371
1372 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1373 sub verify_drive {
1374     my ($value, $noerr) = @_;
1375
1376     return $value if parse_drive(undef, $value);
1377
1378     return undef if $noerr;
1379
1380     die "unable to parse drive options\n";
1381 }
1382
1383 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1384 sub verify_hostpci {
1385     my ($value, $noerr) = @_;
1386
1387     return $value if parse_hostpci($value);
1388
1389     return undef if $noerr;
1390
1391     die "unable to parse pci id\n";
1392 }
1393
1394 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1395 sub verify_watchdog {
1396     my ($value, $noerr) = @_;
1397
1398     return $value if parse_watchdog($value);
1399
1400     return undef if $noerr;
1401
1402     die "unable to parse watchdog options\n";
1403 }
1404
1405 sub parse_watchdog {
1406     my ($value) = @_;
1407
1408     return undef if !$value;
1409
1410     my $res = {};
1411
1412     foreach my $p (split(/,/, $value)) {
1413         next if $p =~ m/^\s*$/;
1414
1415         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1416             $res->{model} = $2;
1417         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1418             $res->{action} = $2;
1419         } else {
1420             return undef;
1421         }
1422     }
1423
1424     return $res;
1425 }
1426
1427 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1428 sub verify_startup {
1429     my ($value, $noerr) = @_;
1430
1431     return $value if parse_startup($value);
1432
1433     return undef if $noerr;
1434
1435     die "unable to parse startup options\n";
1436 }
1437
1438 sub parse_startup {
1439     my ($value) = @_;
1440
1441     return undef if !$value;
1442
1443     my $res = {};
1444
1445     foreach my $p (split(/,/, $value)) {
1446         next if $p =~ m/^\s*$/;
1447
1448         if ($p =~ m/^(order=)?(\d+)$/) {
1449             $res->{order} = $2;
1450         } elsif ($p =~ m/^up=(\d+)$/) {
1451             $res->{up} = $1;
1452         } elsif ($p =~ m/^down=(\d+)$/) {
1453             $res->{down} = $1;
1454         } else {
1455             return undef;
1456         }
1457     }
1458
1459     return $res;
1460 }
1461
1462 sub parse_usb_device {
1463     my ($value) = @_;
1464
1465     return undef if !$value;
1466
1467     my @dl = split(/,/, $value);
1468     my $found;
1469
1470     my $res = {};
1471     foreach my $v (@dl) {
1472         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1473             $found = 1;
1474             $res->{vendorid} = $2;
1475             $res->{productid} = $4;
1476         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1477             $found = 1;
1478             $res->{hostbus} = $1;
1479             $res->{hostport} = $2;
1480         } elsif ($v =~ m/^spice$/) {
1481             $found = 1;
1482             $res->{spice} = 1;
1483         } else {
1484             return undef;
1485         }
1486     }
1487     return undef if !$found;
1488
1489     return $res;
1490 }
1491
1492 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1493 sub verify_usb_device {
1494     my ($value, $noerr) = @_;
1495
1496     return $value if parse_usb_device($value);
1497
1498     return undef if $noerr;
1499
1500     die "unable to parse usb device\n";
1501 }
1502
1503 # add JSON properties for create and set function
1504 sub json_config_properties {
1505     my $prop = shift;
1506
1507     foreach my $opt (keys %$confdesc) {
1508         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1509         $prop->{$opt} = $confdesc->{$opt};
1510     }
1511
1512     return $prop;
1513 }
1514
1515 sub check_type {
1516     my ($key, $value) = @_;
1517
1518     die "unknown setting '$key'\n" if !$confdesc->{$key};
1519
1520     my $type = $confdesc->{$key}->{type};
1521
1522     if (!defined($value)) {
1523         die "got undefined value\n";
1524     }
1525
1526     if ($value =~ m/[\n\r]/) {
1527         die "property contains a line feed\n";
1528     }
1529
1530     if ($type eq 'boolean') {
1531         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1532         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1533         die "type check ('boolean') failed - got '$value'\n";
1534     } elsif ($type eq 'integer') {
1535         return int($1) if $value =~ m/^(\d+)$/;
1536         die "type check ('integer') failed - got '$value'\n";
1537     } elsif ($type eq 'number') {
1538         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1539         die "type check ('number') failed - got '$value'\n";
1540     } elsif ($type eq 'string') {
1541         if (my $fmt = $confdesc->{$key}->{format}) {
1542             if ($fmt eq 'pve-qm-drive') {
1543                 # special case - we need to pass $key to parse_drive()
1544                 my $drive = parse_drive($key, $value);
1545                 return $value if $drive;
1546                 die "unable to parse drive options\n";
1547             }
1548             PVE::JSONSchema::check_format($fmt, $value);
1549             return $value;
1550         }
1551         $value =~ s/^\"(.*)\"$/$1/;
1552         return $value;
1553     } else {
1554         die "internal error"
1555     }
1556 }
1557
1558 sub lock_config_full {
1559     my ($vmid, $timeout, $code, @param) = @_;
1560
1561     my $filename = config_file_lock($vmid);
1562
1563     my $res = lock_file($filename, $timeout, $code, @param);
1564
1565     die $@ if $@;
1566
1567     return $res;
1568 }
1569
1570 sub lock_config_mode {
1571     my ($vmid, $timeout, $shared, $code, @param) = @_;
1572
1573     my $filename = config_file_lock($vmid);
1574
1575     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1576
1577     die $@ if $@;
1578
1579     return $res;
1580 }
1581
1582 sub lock_config {
1583     my ($vmid, $code, @param) = @_;
1584
1585     return lock_config_full($vmid, 10, $code, @param);
1586 }
1587
1588 sub cfs_config_path {
1589     my ($vmid, $node) = @_;
1590
1591     $node = $nodename if !$node;
1592     return "nodes/$node/qemu-server/$vmid.conf";
1593 }
1594
1595 sub check_iommu_support{
1596     #fixme : need to check IOMMU support
1597     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1598
1599     my $iommu=1;
1600     return $iommu;
1601
1602 }
1603
1604 sub config_file {
1605     my ($vmid, $node) = @_;
1606
1607     my $cfspath = cfs_config_path($vmid, $node);
1608     return "/etc/pve/$cfspath";
1609 }
1610
1611 sub config_file_lock {
1612     my ($vmid) = @_;
1613
1614     return "$lock_dir/lock-$vmid.conf";
1615 }
1616
1617 sub touch_config {
1618     my ($vmid) = @_;
1619
1620     my $conf = config_file($vmid);
1621     utime undef, undef, $conf;
1622 }
1623
1624 sub destroy_vm {
1625     my ($storecfg, $vmid, $keep_empty_config) = @_;
1626
1627     my $conffile = config_file($vmid);
1628
1629     my $conf = load_config($vmid);
1630
1631     check_lock($conf);
1632
1633     # only remove disks owned by this VM
1634     foreach_drive($conf, sub {
1635         my ($ds, $drive) = @_;
1636
1637         return if drive_is_cdrom($drive);
1638
1639         my $volid = $drive->{file};
1640
1641         return if !$volid || $volid =~ m|^/|;
1642
1643         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1644         return if !$path || !$owner || ($owner != $vmid);
1645
1646         PVE::Storage::vdisk_free($storecfg, $volid);
1647     });
1648
1649     if ($keep_empty_config) {
1650         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1651     } else {
1652         unlink $conffile;
1653     }
1654
1655     # also remove unused disk
1656     eval {
1657         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1658
1659         eval {
1660             PVE::Storage::foreach_volid($dl, sub {
1661                 my ($volid, $sid, $volname, $d) = @_;
1662                 PVE::Storage::vdisk_free($storecfg, $volid);
1663             });
1664         };
1665         warn $@ if $@;
1666
1667     };
1668     warn $@ if $@;
1669 }
1670
1671 sub load_config {
1672     my ($vmid, $node) = @_;
1673
1674     my $cfspath = cfs_config_path($vmid, $node);
1675
1676     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1677
1678     die "no such VM ('$vmid')\n" if !defined($conf);
1679
1680     return $conf;
1681 }
1682
1683 sub parse_vm_config {
1684     my ($filename, $raw) = @_;
1685
1686     return undef if !defined($raw);
1687
1688     my $res = {
1689         digest => Digest::SHA::sha1_hex($raw),
1690         snapshots => {},
1691     };
1692
1693     $filename =~ m|/qemu-server/(\d+)\.conf$|
1694         || die "got strange filename '$filename'";
1695
1696     my $vmid = $1;
1697
1698     my $conf = $res;
1699     my $descr = '';
1700
1701     my @lines = split(/\n/, $raw);
1702     foreach my $line (@lines) {
1703         next if $line =~ m/^\s*$/;
1704
1705         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1706             my $snapname = $1;
1707             $conf->{description} = $descr if $descr;
1708             $descr = '';
1709             $conf = $res->{snapshots}->{$snapname} = {};
1710             next;
1711         }
1712
1713         if ($line =~ m/^\#(.*)\s*$/) {
1714             $descr .= PVE::Tools::decode_text($1) . "\n";
1715             next;
1716         }
1717
1718         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1719             $descr .= PVE::Tools::decode_text($2);
1720         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1721             $conf->{snapstate} = $1;
1722         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1723             my $key = $1;
1724             my $value = $2;
1725             $conf->{$key} = $value;
1726         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1727             my $key = $1;
1728             my $value = $2;
1729             eval { $value = check_type($key, $value); };
1730             if ($@) {
1731                 warn "vm $vmid - unable to parse value of '$key' - $@";
1732             } else {
1733                 my $fmt = $confdesc->{$key}->{format};
1734                 if ($fmt && $fmt eq 'pve-qm-drive') {
1735                     my $v = parse_drive($key, $value);
1736                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1737                         $v->{file} = $volid;
1738                         $value = print_drive($vmid, $v);
1739                     } else {
1740                         warn "vm $vmid - unable to parse value of '$key'\n";
1741                         next;
1742                     }
1743                 }
1744
1745                 if ($key eq 'cdrom') {
1746                     $conf->{ide2} = $value;
1747                 } else {
1748                     $conf->{$key} = $value;
1749                 }
1750             }
1751         }
1752     }
1753
1754     $conf->{description} = $descr if $descr;
1755
1756     delete $res->{snapstate}; # just to be sure
1757
1758     return $res;
1759 }
1760
1761 sub write_vm_config {
1762     my ($filename, $conf) = @_;
1763
1764     delete $conf->{snapstate}; # just to be sure
1765
1766     if ($conf->{cdrom}) {
1767         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1768         $conf->{ide2} = $conf->{cdrom};
1769         delete $conf->{cdrom};
1770     }
1771
1772     # we do not use 'smp' any longer
1773     if ($conf->{sockets}) {
1774         delete $conf->{smp};
1775     } elsif ($conf->{smp}) {
1776         $conf->{sockets} = $conf->{smp};
1777         delete $conf->{cores};
1778         delete $conf->{smp};
1779     }
1780
1781     if ($conf->{maxcpus} && $conf->{sockets}) {
1782         delete $conf->{sockets};
1783     }
1784
1785     my $used_volids = {};
1786
1787     my $cleanup_config = sub {
1788         my ($cref, $snapname) = @_;
1789
1790         foreach my $key (keys %$cref) {
1791             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1792                 $key eq 'snapstate';
1793             my $value = $cref->{$key};
1794             eval { $value = check_type($key, $value); };
1795             die "unable to parse value of '$key' - $@" if $@;
1796
1797             $cref->{$key} = $value;
1798
1799             if (!$snapname && valid_drivename($key)) {
1800                 my $drive = parse_drive($key, $value);
1801                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1802             }
1803         }
1804     };
1805
1806     &$cleanup_config($conf);
1807     foreach my $snapname (keys %{$conf->{snapshots}}) {
1808         &$cleanup_config($conf->{snapshots}->{$snapname}, $snapname);
1809     }
1810
1811     # remove 'unusedX' settings if we re-add a volume
1812     foreach my $key (keys %$conf) {
1813         my $value = $conf->{$key};
1814         if ($key =~ m/^unused/ && $used_volids->{$value}) {
1815             delete $conf->{$key};
1816         }
1817     }
1818
1819     my $generate_raw_config = sub {
1820         my ($conf) = @_;
1821
1822         my $raw = '';
1823
1824         # add description as comment to top of file
1825         my $descr = $conf->{description} || '';
1826         foreach my $cl (split(/\n/, $descr)) {
1827             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
1828         }
1829
1830         foreach my $key (sort keys %$conf) {
1831             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots';
1832             $raw .= "$key: $conf->{$key}\n";
1833         }
1834         return $raw;
1835     };
1836
1837     my $raw = &$generate_raw_config($conf);
1838     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1839         $raw .= "\n[$snapname]\n";
1840         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1841     }
1842
1843     return $raw;
1844 }
1845
1846 sub update_config_nolock {
1847     my ($vmid, $conf, $skiplock) = @_;
1848
1849     check_lock($conf) if !$skiplock;
1850
1851     my $cfspath = cfs_config_path($vmid);
1852
1853     PVE::Cluster::cfs_write_file($cfspath, $conf);
1854 }
1855
1856 sub update_config {
1857     my ($vmid, $conf, $skiplock) = @_;
1858
1859     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
1860 }
1861
1862 sub load_defaults {
1863
1864     my $res = {};
1865
1866     # we use static defaults from our JSON schema configuration
1867     foreach my $key (keys %$confdesc) {
1868         if (defined(my $default = $confdesc->{$key}->{default})) {
1869             $res->{$key} = $default;
1870         }
1871     }
1872
1873     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
1874     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
1875
1876     return $res;
1877 }
1878
1879 sub config_list {
1880     my $vmlist = PVE::Cluster::get_vmlist();
1881     my $res = {};
1882     return $res if !$vmlist || !$vmlist->{ids};
1883     my $ids = $vmlist->{ids};
1884
1885     foreach my $vmid (keys %$ids) {
1886         my $d = $ids->{$vmid};
1887         next if !$d->{node} || $d->{node} ne $nodename;
1888         next if !$d->{type} || $d->{type} ne 'qemu';
1889         $res->{$vmid}->{exists} = 1;
1890     }
1891     return $res;
1892 }
1893
1894 # test if VM uses local resources (to prevent migration)
1895 sub check_local_resources {
1896     my ($conf, $noerr) = @_;
1897
1898     my $loc_res = 0;
1899
1900     $loc_res = 1 if $conf->{hostusb}; # old syntax
1901     $loc_res = 1 if $conf->{hostpci}; # old syntax
1902
1903     foreach my $k (keys %$conf) {
1904         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
1905         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
1906     }
1907
1908     die "VM uses local resources\n" if $loc_res && !$noerr;
1909
1910     return $loc_res;
1911 }
1912
1913 # check if used storages are available on all nodes (use by migrate)
1914 sub check_storage_availability {
1915     my ($storecfg, $conf, $node) = @_;
1916
1917     foreach_drive($conf, sub {
1918         my ($ds, $drive) = @_;
1919
1920         my $volid = $drive->{file};
1921         return if !$volid;
1922
1923         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1924         return if !$sid;
1925
1926         # check if storage is available on both nodes
1927         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
1928         PVE::Storage::storage_check_node($storecfg, $sid, $node);
1929    });
1930 }
1931
1932 # list nodes where all VM images are available (used by has_feature API)
1933 sub shared_nodes {
1934     my ($conf, $storecfg) = @_;
1935
1936     my $nodelist = PVE::Cluster::get_nodelist();
1937     my $nodehash = { map { $_ => 1 } @$nodelist };
1938     my $nodename = PVE::INotify::nodename();
1939
1940     foreach_drive($conf, sub {
1941         my ($ds, $drive) = @_;
1942
1943         my $volid = $drive->{file};
1944         return if !$volid;
1945
1946         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1947         if ($storeid) {
1948             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1949             if ($scfg->{disable}) {
1950                 $nodehash = {};
1951             } elsif (my $avail = $scfg->{nodes}) {
1952                 foreach my $node (keys %$nodehash) {
1953                     delete $nodehash->{$node} if !$avail->{$node};
1954                 }
1955             } elsif (!$scfg->{shared}) {
1956                 foreach my $node (keys %$nodehash) {
1957                     delete $nodehash->{$node} if $node ne $nodename
1958                 }
1959             }
1960         }
1961     });
1962
1963     return $nodehash
1964 }
1965
1966 sub check_lock {
1967     my ($conf) = @_;
1968
1969     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
1970 }
1971
1972 sub check_cmdline {
1973     my ($pidfile, $pid) = @_;
1974
1975     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
1976     if (defined($fh)) {
1977         my $line = <$fh>;
1978         $fh->close;
1979         return undef if !$line;
1980         my @param = split(/\0/, $line);
1981
1982         my $cmd = $param[0];
1983         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
1984
1985         for (my $i = 0; $i < scalar (@param); $i++) {
1986             my $p = $param[$i];
1987             next if !$p;
1988             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
1989                 my $p = $param[$i+1];
1990                 return 1 if $p && ($p eq $pidfile);
1991                 return undef;
1992             }
1993         }
1994     }
1995     return undef;
1996 }
1997
1998 sub check_running {
1999     my ($vmid, $nocheck, $node) = @_;
2000
2001     my $filename = config_file($vmid, $node);
2002
2003     die "unable to find configuration file for VM $vmid - no such machine\n"
2004         if !$nocheck && ! -f $filename;
2005
2006     my $pidfile = pidfile_name($vmid);
2007
2008     if (my $fd = IO::File->new("<$pidfile")) {
2009         my $st = stat($fd);
2010         my $line = <$fd>;
2011         close($fd);
2012
2013         my $mtime = $st->mtime;
2014         if ($mtime > time()) {
2015             warn "file '$filename' modified in future\n";
2016         }
2017
2018         if ($line =~ m/^(\d+)$/) {
2019             my $pid = $1;
2020             if (check_cmdline($pidfile, $pid)) {
2021                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2022                     return $pid;
2023                 }
2024             }
2025         }
2026     }
2027
2028     return undef;
2029 }
2030
2031 sub vzlist {
2032
2033     my $vzlist = config_list();
2034
2035     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2036
2037     while (defined(my $de = $fd->read)) {
2038         next if $de !~ m/^(\d+)\.pid$/;
2039         my $vmid = $1;
2040         next if !defined($vzlist->{$vmid});
2041         if (my $pid = check_running($vmid)) {
2042             $vzlist->{$vmid}->{pid} = $pid;
2043         }
2044     }
2045
2046     return $vzlist;
2047 }
2048
2049 sub disksize {
2050     my ($storecfg, $conf) = @_;
2051
2052     my $bootdisk = $conf->{bootdisk};
2053     return undef if !$bootdisk;
2054     return undef if !valid_drivename($bootdisk);
2055
2056     return undef if !$conf->{$bootdisk};
2057
2058     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2059     return undef if !defined($drive);
2060
2061     return undef if drive_is_cdrom($drive);
2062
2063     my $volid = $drive->{file};
2064     return undef if !$volid;
2065
2066     return $drive->{size};
2067 }
2068
2069 my $last_proc_pid_stat;
2070
2071 # get VM status information
2072 # This must be fast and should not block ($full == false)
2073 # We only query KVM using QMP if $full == true (this can be slow)
2074 sub vmstatus {
2075     my ($opt_vmid, $full) = @_;
2076
2077     my $res = {};
2078
2079     my $storecfg = PVE::Storage::config();
2080
2081     my $list = vzlist();
2082     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2083
2084     my $cpucount = $cpuinfo->{cpus} || 1;
2085
2086     foreach my $vmid (keys %$list) {
2087         next if $opt_vmid && ($vmid ne $opt_vmid);
2088
2089         my $cfspath = cfs_config_path($vmid);
2090         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2091
2092         my $d = {};
2093         $d->{pid} = $list->{$vmid}->{pid};
2094
2095         # fixme: better status?
2096         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2097
2098         my $size = disksize($storecfg, $conf);
2099         if (defined($size)) {
2100             $d->{disk} = 0; # no info available
2101             $d->{maxdisk} = $size;
2102         } else {
2103             $d->{disk} = 0;
2104             $d->{maxdisk} = 0;
2105         }
2106
2107         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2108         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2109
2110         $d->{name} = $conf->{name} || "VM $vmid";
2111         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2112
2113         if ($conf->{balloon}) {
2114             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2115             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2116         }
2117
2118         $d->{uptime} = 0;
2119         $d->{cpu} = 0;
2120         $d->{mem} = 0;
2121
2122         $d->{netout} = 0;
2123         $d->{netin} = 0;
2124
2125         $d->{diskread} = 0;
2126         $d->{diskwrite} = 0;
2127
2128         $d->{template} = is_template($conf);
2129
2130         $res->{$vmid} = $d;
2131     }
2132
2133     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2134     foreach my $dev (keys %$netdev) {
2135         next if $dev !~ m/^tap([1-9]\d*)i/;
2136         my $vmid = $1;
2137         my $d = $res->{$vmid};
2138         next if !$d;
2139
2140         $d->{netout} += $netdev->{$dev}->{receive};
2141         $d->{netin} += $netdev->{$dev}->{transmit};
2142     }
2143
2144     my $ctime = gettimeofday;
2145
2146     foreach my $vmid (keys %$list) {
2147
2148         my $d = $res->{$vmid};
2149         my $pid = $d->{pid};
2150         next if !$pid;
2151
2152         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2153         next if !$pstat; # not running
2154
2155         my $used = $pstat->{utime} + $pstat->{stime};
2156
2157         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2158
2159         if ($pstat->{vsize}) {
2160             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2161         }
2162
2163         my $old = $last_proc_pid_stat->{$pid};
2164         if (!$old) {
2165             $last_proc_pid_stat->{$pid} = {
2166                 time => $ctime,
2167                 used => $used,
2168                 cpu => 0,
2169             };
2170             next;
2171         }
2172
2173         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2174
2175         if ($dtime > 1000) {
2176             my $dutime = $used -  $old->{used};
2177
2178             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2179             $last_proc_pid_stat->{$pid} = {
2180                 time => $ctime,
2181                 used => $used,
2182                 cpu => $d->{cpu},
2183             };
2184         } else {
2185             $d->{cpu} = $old->{cpu};
2186         }
2187     }
2188
2189     return $res if !$full;
2190
2191     my $qmpclient = PVE::QMPClient->new();
2192
2193     my $ballooncb = sub {
2194         my ($vmid, $resp) = @_;
2195
2196         my $info = $resp->{'return'};
2197         return if !$info->{max_mem};
2198
2199         my $d = $res->{$vmid};
2200
2201         # use memory assigned to VM
2202         $d->{maxmem} = $info->{max_mem};
2203         $d->{balloon} = $info->{actual};
2204
2205         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2206             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2207             $d->{freemem} = $info->{free_mem};
2208         }
2209
2210     };
2211
2212     my $blockstatscb = sub {
2213         my ($vmid, $resp) = @_;
2214         my $data = $resp->{'return'} || [];
2215         my $totalrdbytes = 0;
2216         my $totalwrbytes = 0;
2217         for my $blockstat (@$data) {
2218             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2219             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2220         }
2221         $res->{$vmid}->{diskread} = $totalrdbytes;
2222         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2223     };
2224
2225     my $statuscb = sub {
2226         my ($vmid, $resp) = @_;
2227
2228         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2229         # this fails if ballon driver is not loaded, so this must be
2230         # the last commnand (following command are aborted if this fails).
2231         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2232
2233         my $status = 'unknown';
2234         if (!defined($status = $resp->{'return'}->{status})) {
2235             warn "unable to get VM status\n";
2236             return;
2237         }
2238
2239         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2240     };
2241
2242     foreach my $vmid (keys %$list) {
2243         next if $opt_vmid && ($vmid ne $opt_vmid);
2244         next if !$res->{$vmid}->{pid}; # not running
2245         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2246     }
2247
2248     $qmpclient->queue_execute();
2249
2250     foreach my $vmid (keys %$list) {
2251         next if $opt_vmid && ($vmid ne $opt_vmid);
2252         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2253     }
2254
2255     return $res;
2256 }
2257
2258 sub foreach_drive {
2259     my ($conf, $func) = @_;
2260
2261     foreach my $ds (keys %$conf) {
2262         next if !valid_drivename($ds);
2263
2264         my $drive = parse_drive($ds, $conf->{$ds});
2265         next if !$drive;
2266
2267         &$func($ds, $drive);
2268     }
2269 }
2270
2271 sub foreach_volid {
2272     my ($conf, $func) = @_;
2273
2274     my $volhash = {};
2275
2276     my $test_volid = sub {
2277         my ($volid, $is_cdrom) = @_;
2278
2279         return if !$volid;
2280
2281         $volhash->{$volid} = $is_cdrom || 0;
2282     };
2283
2284     foreach_drive($conf, sub {
2285         my ($ds, $drive) = @_;
2286         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2287     });
2288
2289     foreach my $snapname (keys %{$conf->{snapshots}}) {
2290         my $snap = $conf->{snapshots}->{$snapname};
2291         &$test_volid($snap->{vmstate}, 0);
2292         foreach_drive($snap, sub {
2293             my ($ds, $drive) = @_;
2294             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2295         });
2296     }
2297
2298     foreach my $volid (keys %$volhash) {
2299         &$func($volid, $volhash->{$volid});
2300     }
2301 }
2302
2303 sub vga_conf_has_spice {
2304     my ($vga) = @_;
2305
2306     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2307
2308     return $1 || 1;
2309 }
2310
2311 sub config_to_command {
2312     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2313
2314     my $cmd = [];
2315     my $globalFlags = [];
2316     my $machineFlags = [];
2317     my $rtcFlags = [];
2318     my $cpuFlags = [];
2319     my $devices = [];
2320     my $pciaddr = '';
2321     my $bridges = {};
2322     my $kvmver = kvm_user_version();
2323     my $vernum = 0; # unknown
2324     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2325         $vernum = $1*1000000+$2*1000;
2326     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2327         $vernum = $1*1000000+$2*1000+$3;
2328     }
2329
2330     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2331
2332     my $have_ovz = -f '/proc/vz/vestat';
2333
2334     my $q35 = 1 if($conf->{machine} && $conf->{machine} =~ m/q35/);
2335     push @$cmd, '/usr/bin/kvm';
2336
2337     push @$cmd, '-id', $vmid;
2338
2339     my $use_virtio = 0;
2340
2341     my $qmpsocket = qmp_socket($vmid);
2342     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2343     push @$cmd, '-mon', "chardev=qmp,mode=control";
2344
2345     my $socket = vnc_socket($vmid);
2346     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2347
2348     push @$cmd, '-pidfile' , pidfile_name($vmid);
2349
2350     push @$cmd, '-daemonize';
2351
2352     if($q35){
2353         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2354     }else{
2355         $pciaddr = print_pci_addr("piix3", $bridges);
2356         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2357
2358         my $use_usb2 = 0;
2359             for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2360                 next if !$conf->{"usb$i"};
2361                 $use_usb2 = 1;
2362             }
2363             # include usb device config
2364             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2365     }
2366
2367     my $vga = $conf->{vga};
2368
2369     my $qxlnum = vga_conf_has_spice($vga);
2370     $vga = 'qxl' if $qxlnum;
2371
2372     if (!$vga) {
2373         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2374                                 $conf->{ostype} eq 'win7' ||
2375                                 $conf->{ostype} eq 'w2k8')) {
2376             $vga = 'std';
2377         } else {
2378             $vga = 'cirrus';
2379         }
2380     }
2381
2382     # enable absolute mouse coordinates (needed by vnc)
2383     my $tablet;
2384     if (defined($conf->{tablet})) {
2385         $tablet = $conf->{tablet};
2386     } else {
2387         $tablet = $defaults->{tablet};
2388         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2389         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2390     }
2391
2392     my $usbbus = $q35 ? "ehci" : "uhci";
2393     push @$devices, '-device', "usb-tablet,id=tablet,bus=$usbbus.0,port=1" if $tablet;
2394
2395     # host pci devices
2396     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2397           my $d = parse_hostpci($conf->{"hostpci$i"});
2398           next if !$d;
2399           $pciaddr = print_pci_addr("hostpci$i", $bridges);
2400           my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2401           push @$devices, '-device', "pci-assign,host=$d->{pciid},id=hostpci$i$pciaddr$rombar";
2402     }
2403
2404     # usb devices
2405     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2406         my $d = parse_usb_device($conf->{"usb$i"});
2407         next if !$d;
2408         if ($d->{vendorid} && $d->{productid}) {
2409             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2410         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2411             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2412         } elsif ($d->{spice}) {
2413             # usb redir support for spice
2414             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2415             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2416         }
2417     }
2418
2419     # serial devices
2420     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2421         if (my $path = $conf->{"serial$i"}) {
2422             if ($path eq 'socket') {
2423                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2424                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2425                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2426             } else {
2427                 die "no such serial device\n" if ! -c $path;
2428                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2429                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2430             }
2431         }
2432     }
2433
2434     # parallel devices
2435     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2436         if (my $path = $conf->{"parallel$i"}) {
2437             die "no such parallel device\n" if ! -c $path;
2438             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2439             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2440             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2441         }
2442     }
2443
2444     my $vmname = $conf->{name} || "vm$vmid";
2445
2446     push @$cmd, '-name', $vmname;
2447
2448     my $sockets = 1;
2449     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2450     $sockets = $conf->{sockets} if  $conf->{sockets};
2451
2452     my $cores = $conf->{cores} || 1;
2453     my $maxcpus = $conf->{maxcpus} if $conf->{maxcpus};
2454
2455     if ($maxcpus) {
2456         push @$cmd, '-smp', "cpus=$cores,maxcpus=$maxcpus";
2457     } else {
2458         push @$cmd, '-smp', "sockets=$sockets,cores=$cores";
2459     }
2460
2461     push @$cmd, '-nodefaults';
2462
2463     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2464
2465     my $bootindex_hash = {};
2466     my $i = 1;
2467     foreach my $o (split(//, $bootorder)) {
2468         $bootindex_hash->{$o} = $i*100;
2469         $i++;
2470     }
2471
2472     push @$cmd, '-boot', "menu=on";
2473
2474     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2475
2476     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2477
2478     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2479
2480     # time drift fix
2481     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2482
2483     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2484     my $useLocaltime = $conf->{localtime};
2485
2486     if (my $ost = $conf->{ostype}) {
2487         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2488
2489         if ($ost =~ m/^w/) { # windows
2490             $useLocaltime = 1 if !defined($conf->{localtime});
2491
2492             # use time drift fix when acpi is enabled
2493             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2494                 $tdf = 1 if !defined($conf->{tdf});
2495             }
2496         }
2497
2498         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2499             $ost eq 'wvista') {
2500             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2501             push @$cmd, '-no-hpet';
2502             #push @$cpuFlags , 'hv_vapic" if !$nokvm;  #fixme, my win2008R2 hang at boot with this
2503             push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2504         }
2505
2506         if ($ost eq 'win7' || $ost eq 'win8') {
2507             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2508         }
2509     }
2510
2511     push @$rtcFlags, 'driftfix=slew' if $tdf;
2512
2513     if ($nokvm) {
2514         push @$machineFlags, 'accel=tcg';
2515     } else {
2516         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2517     }
2518
2519     my $machine_type = $forcemachine || $conf->{machine};
2520     if ($machine_type) {
2521         push @$machineFlags, "type=${machine_type}";
2522     }
2523
2524     if ($conf->{startdate}) {
2525         push @$rtcFlags, "base=$conf->{startdate}";
2526     } elsif ($useLocaltime) {
2527         push @$rtcFlags, 'base=localtime';
2528     }
2529
2530     my $cpu = $nokvm ? "qemu64" : "kvm64";
2531     $cpu = $conf->{cpu} if $conf->{cpu};
2532
2533     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2534
2535     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2536
2537     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2538
2539     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2540
2541     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2542
2543     # Note: enforce needs kernel 3.10, so we do not use it for now
2544     # push @$cmd, '-cpu', "$cpu,enforce";
2545     push @$cmd, '-cpu', $cpu;
2546
2547     push @$cmd, '-S' if $conf->{freeze};
2548
2549     # set keyboard layout
2550     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2551     push @$cmd, '-k', $kb if $kb;
2552
2553     # enable sound
2554     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2555     #push @$cmd, '-soundhw', 'es1370';
2556     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2557
2558     if($conf->{agent}) {
2559         my $qgasocket = qga_socket($vmid);
2560         my $pciaddr = print_pci_addr("qga0", $bridges);
2561         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2562         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2563         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2564     }
2565
2566     my $spice_port;
2567
2568     if ($qxlnum) {
2569         if ($qxlnum > 1) {
2570             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2571                 for(my $i = 1; $i < $qxlnum; $i++){
2572                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2573                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2574                 }
2575             } else {
2576                 # assume other OS works like Linux
2577                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2578                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2579             }
2580         }
2581
2582         my $pciaddr = print_pci_addr("spice", $bridges);
2583
2584         $spice_port = PVE::Tools::next_spice_port();
2585
2586         push @$cmd, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2587
2588         push @$cmd, '-device', "virtio-serial,id=spice$pciaddr";
2589         push @$cmd, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2590         push @$cmd, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2591     }
2592
2593     # enable balloon by default, unless explicitly disabled
2594     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2595         $pciaddr = print_pci_addr("balloon0", $bridges);
2596         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2597     }
2598
2599     if ($conf->{watchdog}) {
2600         my $wdopts = parse_watchdog($conf->{watchdog});
2601         $pciaddr = print_pci_addr("watchdog", $bridges);
2602         my $watchdog = $wdopts->{model} || 'i6300esb';
2603         push @$devices, '-device', "$watchdog$pciaddr";
2604         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
2605     }
2606
2607     my $vollist = [];
2608     my $scsicontroller = {};
2609     my $ahcicontroller = {};
2610     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
2611
2612     # Add iscsi initiator name if available
2613     if (my $initiator = get_initiator_name()) {
2614         push @$devices, '-iscsi', "initiator-name=$initiator";
2615     }
2616
2617     foreach_drive($conf, sub {
2618         my ($ds, $drive) = @_;
2619
2620         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
2621             push @$vollist, $drive->{file};
2622         }
2623
2624         $use_virtio = 1 if $ds =~ m/^virtio/;
2625
2626         if (drive_is_cdrom ($drive)) {
2627             if ($bootindex_hash->{d}) {
2628                 $drive->{bootindex} = $bootindex_hash->{d};
2629                 $bootindex_hash->{d} += 1;
2630             }
2631         } else {
2632             if ($bootindex_hash->{c}) {
2633                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
2634                 $bootindex_hash->{c} += 1;
2635             }
2636         }
2637
2638         if ($drive->{interface} eq 'scsi') {
2639
2640             my $maxdev = ($scsihw !~ m/^lsi/) ? 256 : 7;
2641             my $controller = int($drive->{index} / $maxdev);
2642             $pciaddr = print_pci_addr("scsihw$controller", $bridges);
2643             push @$devices, '-device', "$scsihw,id=scsihw$controller$pciaddr" if !$scsicontroller->{$controller};
2644             $scsicontroller->{$controller}=1;
2645         }
2646
2647         if ($drive->{interface} eq 'sata') {
2648            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
2649            $pciaddr = print_pci_addr("ahci$controller", $bridges);
2650            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
2651            $ahcicontroller->{$controller}=1;
2652         }
2653
2654         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
2655         push @$devices, '-drive',$drive_cmd;
2656         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
2657     });
2658
2659     push @$cmd, '-m', $conf->{memory} || $defaults->{memory};
2660
2661     for (my $i = 0; $i < $MAX_NETS; $i++) {
2662          next if !$conf->{"net$i"};
2663          my $d = parse_net($conf->{"net$i"});
2664          next if !$d;
2665
2666          $use_virtio = 1 if $d->{model} eq 'virtio';
2667
2668          if ($bootindex_hash->{n}) {
2669             $d->{bootindex} = $bootindex_hash->{n};
2670             $bootindex_hash->{n} += 1;
2671          }
2672
2673          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
2674          push @$devices, '-netdev', $netdevfull;
2675
2676          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
2677          push @$devices, '-device', $netdevicefull;
2678     }
2679
2680     if(!$q35){
2681         #bridges
2682         while (my ($k, $v) = each %$bridges) {
2683             $pciaddr = print_pci_addr("pci.$k");
2684             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
2685         }
2686     }
2687
2688     # hack: virtio with fairsched is unreliable, so we do not use fairsched
2689     # when the VM uses virtio devices.
2690     if (!$use_virtio && $have_ovz) {
2691
2692         my $cpuunits = defined($conf->{cpuunits}) ?
2693             $conf->{cpuunits} : $defaults->{cpuunits};
2694
2695         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
2696
2697         # fixme: cpulimit is currently ignored
2698         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
2699     }
2700
2701     # add custom args
2702     if ($conf->{args}) {
2703         my $aa = PVE::Tools::split_args($conf->{args});
2704         push @$cmd, @$aa;
2705     }
2706
2707     push @$cmd, @$devices;
2708     push @$cmd, '-rtc', join(',', @$rtcFlags)
2709         if scalar(@$rtcFlags);
2710     push @$cmd, '-machine', join(',', @$machineFlags)
2711         if scalar(@$machineFlags);
2712     push @$cmd, '-global', join(',', @$globalFlags)
2713         if scalar(@$globalFlags);
2714
2715     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
2716 }
2717
2718 sub vnc_socket {
2719     my ($vmid) = @_;
2720     return "${var_run_tmpdir}/$vmid.vnc";
2721 }
2722
2723 sub spice_port {
2724     my ($vmid) = @_;
2725
2726     my $res = vm_mon_cmd($vmid, 'query-spice');
2727
2728     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
2729 }
2730
2731 sub qmp_socket {
2732     my ($vmid) = @_;
2733     return "${var_run_tmpdir}/$vmid.qmp";
2734 }
2735
2736 sub qga_socket {
2737     my ($vmid) = @_;
2738     return "${var_run_tmpdir}/$vmid.qga";
2739 }
2740
2741 sub pidfile_name {
2742     my ($vmid) = @_;
2743     return "${var_run_tmpdir}/$vmid.pid";
2744 }
2745
2746 sub vm_devices_list {
2747     my ($vmid) = @_;
2748
2749     my $res = vm_mon_cmd($vmid, 'query-pci');
2750
2751     my $devices = {};
2752     foreach my $pcibus (@$res) {
2753         foreach my $device (@{$pcibus->{devices}}) {
2754             next if !$device->{'qdev_id'};
2755             $devices->{$device->{'qdev_id'}} = $device;
2756         }
2757     }
2758
2759     return $devices;
2760 }
2761
2762 sub vm_deviceplug {
2763     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
2764
2765     return 1 if !check_running($vmid);
2766     my $q35 = 1 if ($conf->{machine} && $conf->{machine} =~ m/q35/);
2767     if ($deviceid eq 'tablet') {
2768         my $usbbus = $q35 ? "ehci" : "uhci";
2769         my $devicefull = "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
2770         qemu_deviceadd($vmid, $devicefull);
2771         return 1;
2772     }
2773
2774     return 1 if !$conf->{hotplug};
2775
2776     my $devices_list = vm_devices_list($vmid);
2777     return 1 if defined($devices_list->{$deviceid});
2778
2779     qemu_bridgeadd($storecfg, $conf, $vmid, $deviceid); #add bridge if we need it for the device
2780
2781     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2782         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2783         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2784         qemu_deviceadd($vmid, $devicefull);
2785         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2786            qemu_drivedel($vmid, $deviceid);
2787            return undef;
2788         }
2789     }
2790
2791     if ($deviceid =~ m/^(scsihw)(\d+)$/) {
2792         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
2793         my $pciaddr = print_pci_addr($deviceid);
2794         my $devicefull = "$scsihw,id=$deviceid$pciaddr";
2795         qemu_deviceadd($vmid, $devicefull);
2796         return undef if(!qemu_deviceaddverify($vmid, $deviceid));
2797     }
2798
2799     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2800         return 1 if ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)); #virtio-scsi not yet support hotplug
2801         return undef if !qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
2802         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2803         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2804         if(!qemu_deviceadd($vmid, $devicefull)) {
2805            qemu_drivedel($vmid, $deviceid);
2806            return undef;
2807         }
2808     }
2809
2810     if ($deviceid =~ m/^(net)(\d+)$/) {
2811         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
2812         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
2813         qemu_deviceadd($vmid, $netdevicefull);
2814         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2815            qemu_netdevdel($vmid, $deviceid);
2816            return undef;
2817         }
2818     }
2819
2820
2821     if (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
2822         my $bridgeid = $2;
2823         my $pciaddr = print_pci_addr($deviceid);
2824         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
2825         qemu_deviceadd($vmid, $devicefull);
2826         return undef if !qemu_deviceaddverify($vmid, $deviceid);
2827     }
2828
2829     return 1;
2830 }
2831
2832 sub vm_deviceunplug {
2833     my ($vmid, $conf, $deviceid) = @_;
2834
2835     return 1 if !check_running ($vmid);
2836
2837     if ($deviceid eq 'tablet') {
2838         qemu_devicedel($vmid, $deviceid);
2839         return 1;
2840     }
2841
2842     return 1 if !$conf->{hotplug};
2843
2844     my $devices_list = vm_devices_list($vmid);
2845     return 1 if !defined($devices_list->{$deviceid});
2846
2847     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
2848
2849     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2850         qemu_devicedel($vmid, $deviceid);
2851         return undef if !qemu_devicedelverify($vmid, $deviceid);
2852         return undef if !qemu_drivedel($vmid, $deviceid);
2853     }
2854
2855     if ($deviceid =~ m/^(lsi)(\d+)$/) {
2856         return undef if !qemu_devicedel($vmid, $deviceid);
2857     }
2858
2859     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2860         return undef if !qemu_devicedel($vmid, $deviceid);
2861         return undef if !qemu_drivedel($vmid, $deviceid);
2862     }
2863
2864     if ($deviceid =~ m/^(net)(\d+)$/) {
2865         qemu_devicedel($vmid, $deviceid);
2866         return undef if !qemu_devicedelverify($vmid, $deviceid);
2867         return undef if !qemu_netdevdel($vmid, $deviceid);
2868     }
2869
2870     return 1;
2871 }
2872
2873 sub qemu_deviceadd {
2874     my ($vmid, $devicefull) = @_;
2875
2876     $devicefull = "driver=".$devicefull;
2877     my %options =  split(/[=,]/, $devicefull);
2878
2879     vm_mon_cmd($vmid, "device_add" , %options);
2880     return 1;
2881 }
2882
2883 sub qemu_devicedel {
2884     my($vmid, $deviceid) = @_;
2885     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
2886     return 1;
2887 }
2888
2889 sub qemu_driveadd {
2890     my($storecfg, $vmid, $device) = @_;
2891
2892     my $drive = print_drive_full($storecfg, $vmid, $device);
2893     my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
2894     # If the command succeeds qemu prints: "OK"
2895     if ($ret !~ m/OK/s) {
2896         syslog("err", "adding drive failed: $ret");
2897         return undef;
2898     }
2899     return 1;
2900 }
2901
2902 sub qemu_drivedel {
2903     my($vmid, $deviceid) = @_;
2904
2905     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
2906     $ret =~ s/^\s+//;
2907     if ($ret =~ m/Device \'.*?\' not found/s) {
2908         # NB: device not found errors mean the drive was auto-deleted and we ignore the error
2909     }
2910     elsif ($ret ne "") {
2911       syslog("err", "deleting drive $deviceid failed : $ret");
2912       return undef;
2913     }
2914     return 1;
2915 }
2916
2917 sub qemu_deviceaddverify {
2918     my ($vmid,$deviceid) = @_;
2919
2920     for (my $i = 0; $i <= 5; $i++) {
2921          my $devices_list = vm_devices_list($vmid);
2922          return 1 if defined($devices_list->{$deviceid});
2923          sleep 1;
2924     }
2925     syslog("err", "error on hotplug device $deviceid");
2926     return undef;
2927 }
2928
2929
2930 sub qemu_devicedelverify {
2931     my ($vmid,$deviceid) = @_;
2932
2933     #need to verify the device is correctly remove as device_del is async and empty return is not reliable
2934     for (my $i = 0; $i <= 5; $i++) {
2935          my $devices_list = vm_devices_list($vmid);
2936          return 1 if !defined($devices_list->{$deviceid});
2937          sleep 1;
2938     }
2939     syslog("err", "error on hot-unplugging device $deviceid");
2940     return undef;
2941 }
2942
2943 sub qemu_findorcreatescsihw {
2944     my ($storecfg, $conf, $vmid, $device) = @_;
2945
2946     my $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
2947     my $controller = int($device->{index} / $maxdev);
2948     my $scsihwid="scsihw$controller";
2949     my $devices_list = vm_devices_list($vmid);
2950
2951     if(!defined($devices_list->{$scsihwid})) {
2952        return undef if !vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
2953     }
2954     return 1;
2955 }
2956
2957 sub qemu_bridgeadd {
2958     my ($storecfg, $conf, $vmid, $device) = @_;
2959
2960     my $bridges = {};
2961     my $bridgeid = undef;
2962     print_pci_addr($device, $bridges);
2963
2964     while (my ($k, $v) = each %$bridges) {
2965         $bridgeid = $k;
2966     }
2967     return if !$bridgeid || $bridgeid < 1;
2968     my $bridge = "pci.$bridgeid";
2969     my $devices_list = vm_devices_list($vmid);
2970
2971     if(!defined($devices_list->{$bridge})) {
2972         return undef if !vm_deviceplug($storecfg, $conf, $vmid, $bridge);
2973     }
2974     return 1;
2975 }
2976
2977 sub qemu_netdevadd {
2978     my ($vmid, $conf, $device, $deviceid) = @_;
2979
2980     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
2981     my %options =  split(/[=,]/, $netdev);
2982
2983     vm_mon_cmd($vmid, "netdev_add",  %options);
2984     return 1;
2985 }
2986
2987 sub qemu_netdevdel {
2988     my ($vmid, $deviceid) = @_;
2989
2990     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
2991     return 1;
2992 }
2993
2994 sub qemu_cpu_hotplug {
2995     my ($vmid, $conf, $cores) = @_;
2996
2997     die "new cores config is not defined" if !$cores;
2998     die "you can't add more cores than maxcpus"
2999         if $conf->{maxcpus} && ($cores > $conf->{maxcpus});
3000     return if !check_running($vmid);
3001
3002     my $currentcores = $conf->{cores} if $conf->{cores};
3003     die "current cores is not defined" if !$currentcores;
3004     die "maxcpus is not defined" if !$conf->{maxcpus};
3005     raise_param_exc({ 'cores' => "online cpu unplug is not yet possible" })
3006         if($cores < $currentcores);
3007
3008     my $currentrunningcores = vm_mon_cmd($vmid, "query-cpus");
3009     raise_param_exc({ 'cores' => "cores number if running vm is different than configuration" })
3010         if scalar (@{$currentrunningcores}) != $currentcores;
3011
3012     for(my $i = $currentcores; $i < $cores; $i++) {
3013         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3014     }
3015 }
3016
3017 sub qemu_block_set_io_throttle {
3018     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3019
3020     return if !check_running($vmid) ;
3021
3022     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3023
3024 }
3025
3026 # old code, only used to shutdown old VM after update
3027 sub __read_avail {
3028     my ($fh, $timeout) = @_;
3029
3030     my $sel = new IO::Select;
3031     $sel->add($fh);
3032
3033     my $res = '';
3034     my $buf;
3035
3036     my @ready;
3037     while (scalar (@ready = $sel->can_read($timeout))) {
3038         my $count;
3039         if ($count = $fh->sysread($buf, 8192)) {
3040             if ($buf =~ /^(.*)\(qemu\) $/s) {
3041                 $res .= $1;
3042                 last;
3043             } else {
3044                 $res .= $buf;
3045             }
3046         } else {
3047             if (!defined($count)) {
3048                 die "$!\n";
3049             }
3050             last;
3051         }
3052     }
3053
3054     die "monitor read timeout\n" if !scalar(@ready);
3055
3056     return $res;
3057 }
3058
3059 # old code, only used to shutdown old VM after update
3060 sub vm_monitor_command {
3061     my ($vmid, $cmdstr, $nocheck) = @_;
3062
3063     my $res;
3064
3065     eval {
3066         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3067
3068         my $sname = "${var_run_tmpdir}/$vmid.mon";
3069
3070         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3071             die "unable to connect to VM $vmid socket - $!\n";
3072
3073         my $timeout = 3;
3074
3075         # hack: migrate sometime blocks the monitor (when migrate_downtime
3076         # is set)
3077         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3078             $timeout = 60*60; # 1 hour
3079         }
3080
3081         # read banner;
3082         my $data = __read_avail($sock, $timeout);
3083
3084         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3085             die "got unexpected qemu monitor banner\n";
3086         }
3087
3088         my $sel = new IO::Select;
3089         $sel->add($sock);
3090
3091         if (!scalar(my @ready = $sel->can_write($timeout))) {
3092             die "monitor write error - timeout";
3093         }
3094
3095         my $fullcmd = "$cmdstr\r";
3096
3097         # syslog('info', "VM $vmid monitor command: $cmdstr");
3098
3099         my $b;
3100         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3101             die "monitor write error - $!";
3102         }
3103
3104         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3105
3106         $timeout = 20;
3107
3108         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3109             $timeout = 60*60; # 1 hour
3110         } elsif ($cmdstr =~ m/^(eject|change)/) {
3111             $timeout = 60; # note: cdrom mount command is slow
3112         }
3113         if ($res = __read_avail($sock, $timeout)) {
3114
3115             my @lines = split("\r?\n", $res);
3116
3117             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3118
3119             $res = join("\n", @lines);
3120             $res .= "\n";
3121         }
3122     };
3123
3124     my $err = $@;
3125
3126     if ($err) {
3127         syslog("err", "VM $vmid monitor command failed - $err");
3128         die $err;
3129     }
3130
3131     return $res;
3132 }
3133
3134 sub qemu_block_resize {
3135     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3136
3137     my $running = check_running($vmid);
3138
3139     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3140
3141     return if !$running;
3142
3143     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3144
3145 }
3146
3147 sub qemu_volume_snapshot {
3148     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3149
3150     my $running = check_running($vmid);
3151
3152     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3153
3154     return if !$running;
3155
3156     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3157
3158 }
3159
3160 sub qemu_volume_snapshot_delete {
3161     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3162
3163     my $running = check_running($vmid);
3164
3165     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3166
3167     return if !$running;
3168
3169     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3170 }
3171
3172 sub qga_freezefs {
3173     my ($vmid) = @_;
3174
3175     #need to impplement call to qemu-ga
3176 }
3177
3178 sub qga_unfreezefs {
3179     my ($vmid) = @_;
3180
3181     #need to impplement call to qemu-ga
3182 }
3183
3184 sub set_migration_caps {
3185     my ($vmid) = @_;
3186
3187     my $cap_ref = [];
3188
3189     my $enabled_cap = {
3190         "auto-converge" => 1,
3191         "xbzrle" => 0,
3192         "x-rdma-pin-all" => 0,
3193         "zero-blocks" => 0,
3194     };
3195
3196     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3197
3198     for my $supported_capability (@$supported_capabilities) {
3199         push @$cap_ref, {
3200             capability => $supported_capability->{capability},
3201             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3202         };
3203     }
3204
3205     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3206 }
3207
3208 sub vm_start {
3209     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
3210
3211     lock_config($vmid, sub {
3212         my $conf = load_config($vmid, $migratedfrom);
3213
3214         die "you can't start a vm if it's a template\n" if is_template($conf);
3215
3216         check_lock($conf) if !$skiplock;
3217
3218         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
3219
3220         my $defaults = load_defaults();
3221
3222         # set environment variable useful inside network script
3223         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
3224
3225         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
3226
3227         my $migrate_port = 0;
3228         my $migrate_uri;
3229         if ($statefile) {
3230             if ($statefile eq 'tcp') {
3231                 my $localip = "localhost";
3232                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
3233                 if ($datacenterconf->{migration_unsecure}) {
3234                         my $nodename = PVE::INotify::nodename();
3235                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
3236                 }
3237                 $migrate_port = PVE::Tools::next_migrate_port();
3238                 $migrate_uri = "tcp:${localip}:${migrate_port}";
3239                 push @$cmd, '-incoming', $migrate_uri;
3240                 push @$cmd, '-S';
3241             } else {
3242                 push @$cmd, '-loadstate', $statefile;
3243             }
3244         } elsif ($paused) {
3245             push @$cmd, '-S';
3246         }
3247
3248         # host pci devices
3249         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3250           my $d = parse_hostpci($conf->{"hostpci$i"});
3251           next if !$d;
3252           my $info = pci_device_info("0000:$d->{pciid}");
3253           die "IOMMU not present\n" if !check_iommu_support();
3254           die "no pci device info for device '$d->{pciid}'\n" if !$info;
3255           die "can't unbind pci device '$d->{pciid}'\n" if !pci_dev_bind_to_stub($info);
3256           die "can't reset pci device '$d->{pciid}'\n" if !pci_dev_reset($info);
3257         }
3258
3259         PVE::Storage::activate_volumes($storecfg, $vollist);
3260
3261         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
3262                     umask => 0077); };
3263         my $err = $@;
3264         die "start failed: $err" if $err;
3265
3266         print "migration listens on $migrate_uri\n" if $migrate_uri;
3267
3268         if ($statefile && $statefile ne 'tcp')  {
3269             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
3270             warn $@ if $@;
3271         }
3272
3273         if ($migratedfrom) {
3274
3275             eval {
3276                 PVE::QemuServer::set_migration_caps($vmid);
3277             };
3278             warn $@ if $@;
3279
3280             if ($spice_port) {
3281                 print "spice listens on port $spice_port\n";
3282                 if ($spice_ticket) {
3283                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
3284                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
3285                 }
3286             }
3287
3288         } else {
3289
3290             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
3291                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
3292                     if $conf->{balloon};
3293                 vm_mon_cmd_nocheck($vmid, 'qom-set',
3294                             path => "machine/peripheral/balloon0",
3295                             property => "guest-stats-polling-interval",
3296                             value => 2);
3297             }
3298         }
3299     });
3300 }
3301
3302 sub vm_mon_cmd {
3303     my ($vmid, $execute, %params) = @_;
3304
3305     my $cmd = { execute => $execute, arguments => \%params };
3306     vm_qmp_command($vmid, $cmd);
3307 }
3308
3309 sub vm_mon_cmd_nocheck {
3310     my ($vmid, $execute, %params) = @_;
3311
3312     my $cmd = { execute => $execute, arguments => \%params };
3313     vm_qmp_command($vmid, $cmd, 1);
3314 }
3315
3316 sub vm_qmp_command {
3317     my ($vmid, $cmd, $nocheck) = @_;
3318
3319     my $res;
3320
3321     my $timeout;
3322     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
3323         $timeout = $cmd->{arguments}->{timeout};
3324         delete $cmd->{arguments}->{timeout};
3325     }
3326
3327     eval {
3328         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3329         my $sname = qmp_socket($vmid);
3330         if (-e $sname) {
3331             my $qmpclient = PVE::QMPClient->new();
3332
3333             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
3334         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
3335             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
3336                 if scalar(%{$cmd->{arguments}});
3337             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
3338         } else {
3339             die "unable to open monitor socket\n";
3340         }
3341     };
3342     if (my $err = $@) {
3343         syslog("err", "VM $vmid qmp command failed - $err");
3344         die $err;
3345     }
3346
3347     return $res;
3348 }
3349
3350 sub vm_human_monitor_command {
3351     my ($vmid, $cmdline) = @_;
3352
3353     my $res;
3354
3355     my $cmd = {
3356         execute => 'human-monitor-command',
3357         arguments => { 'command-line' => $cmdline},
3358     };
3359
3360     return vm_qmp_command($vmid, $cmd);
3361 }
3362
3363 sub vm_commandline {
3364     my ($storecfg, $vmid) = @_;
3365
3366     my $conf = load_config($vmid);
3367
3368     my $defaults = load_defaults();
3369
3370     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
3371
3372     return join(' ', @$cmd);
3373 }
3374
3375 sub vm_reset {
3376     my ($vmid, $skiplock) = @_;
3377
3378     lock_config($vmid, sub {
3379
3380         my $conf = load_config($vmid);
3381
3382         check_lock($conf) if !$skiplock;
3383
3384         vm_mon_cmd($vmid, "system_reset");
3385     });
3386 }
3387
3388 sub get_vm_volumes {
3389     my ($conf) = @_;
3390
3391     my $vollist = [];
3392     foreach_volid($conf, sub {
3393         my ($volid, $is_cdrom) = @_;
3394
3395         return if $volid =~ m|^/|;
3396
3397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3398         return if !$sid;
3399
3400         push @$vollist, $volid;
3401     });
3402
3403     return $vollist;
3404 }
3405
3406 sub vm_stop_cleanup {
3407     my ($storecfg, $vmid, $conf, $keepActive) = @_;
3408
3409     eval {
3410         fairsched_rmnod($vmid); # try to destroy group
3411
3412         if (!$keepActive) {
3413             my $vollist = get_vm_volumes($conf);
3414             PVE::Storage::deactivate_volumes($storecfg, $vollist);
3415         }
3416
3417         foreach my $ext (qw(mon qmp pid vnc qga)) {
3418             unlink "/var/run/qemu-server/${vmid}.$ext";
3419         }
3420     };
3421     warn $@ if $@; # avoid errors - just warn
3422 }
3423
3424 # Note: use $nockeck to skip tests if VM configuration file exists.
3425 # We need that when migration VMs to other nodes (files already moved)
3426 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
3427 sub vm_stop {
3428     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
3429
3430     $force = 1 if !defined($force) && !$shutdown;
3431
3432     if ($migratedfrom){
3433         my $pid = check_running($vmid, $nocheck, $migratedfrom);
3434         kill 15, $pid if $pid;
3435         my $conf = load_config($vmid, $migratedfrom);
3436         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive);
3437         return;
3438     }
3439
3440     lock_config($vmid, sub {
3441
3442         my $pid = check_running($vmid, $nocheck);
3443         return if !$pid;
3444
3445         my $conf;
3446         if (!$nocheck) {
3447             $conf = load_config($vmid);
3448             check_lock($conf) if !$skiplock;
3449             if (!defined($timeout) && $shutdown && $conf->{startup}) {
3450                 my $opts = parse_startup($conf->{startup});
3451                 $timeout = $opts->{down} if $opts->{down};
3452             }
3453         }
3454
3455         $timeout = 60 if !defined($timeout);
3456
3457         eval {
3458             if ($shutdown) {
3459                 $nocheck ? vm_mon_cmd_nocheck($vmid, "system_powerdown") : vm_mon_cmd($vmid, "system_powerdown");
3460
3461             } else {
3462                 $nocheck ? vm_mon_cmd_nocheck($vmid, "quit") : vm_mon_cmd($vmid, "quit");
3463             }
3464         };
3465         my $err = $@;
3466
3467         if (!$err) {
3468             my $count = 0;
3469             while (($count < $timeout) && check_running($vmid, $nocheck)) {
3470                 $count++;
3471                 sleep 1;
3472             }
3473
3474             if ($count >= $timeout) {
3475                 if ($force) {
3476                     warn "VM still running - terminating now with SIGTERM\n";
3477                     kill 15, $pid;
3478                 } else {
3479                     die "VM quit/powerdown failed - got timeout\n";
3480                 }
3481             } else {
3482                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3483                 return;
3484             }
3485         } else {
3486             if ($force) {
3487                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
3488                 kill 15, $pid;
3489             } else {
3490                 die "VM quit/powerdown failed\n";
3491             }
3492         }
3493
3494         # wait again
3495         $timeout = 10;
3496
3497         my $count = 0;
3498         while (($count < $timeout) && check_running($vmid, $nocheck)) {
3499             $count++;
3500             sleep 1;
3501         }
3502
3503         if ($count >= $timeout) {
3504             warn "VM still running - terminating now with SIGKILL\n";
3505             kill 9, $pid;
3506             sleep 1;
3507         }
3508
3509         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3510    });
3511 }
3512
3513 sub vm_suspend {
3514     my ($vmid, $skiplock) = @_;
3515
3516     lock_config($vmid, sub {
3517
3518         my $conf = load_config($vmid);
3519
3520         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3521
3522         vm_mon_cmd($vmid, "stop");
3523     });
3524 }
3525
3526 sub vm_resume {
3527     my ($vmid, $skiplock) = @_;
3528
3529     lock_config($vmid, sub {
3530
3531         my $conf = load_config($vmid);
3532
3533         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3534
3535         vm_mon_cmd($vmid, "cont");
3536     });
3537 }
3538
3539 sub vm_sendkey {
3540     my ($vmid, $skiplock, $key) = @_;
3541
3542     lock_config($vmid, sub {
3543
3544         my $conf = load_config($vmid);
3545
3546         # there is no qmp command, so we use the human monitor command
3547         vm_human_monitor_command($vmid, "sendkey $key");
3548     });
3549 }
3550
3551 sub vm_destroy {
3552     my ($storecfg, $vmid, $skiplock) = @_;
3553
3554     lock_config($vmid, sub {
3555
3556         my $conf = load_config($vmid);
3557
3558         check_lock($conf) if !$skiplock;
3559
3560         if (!check_running($vmid)) {
3561             fairsched_rmnod($vmid); # try to destroy group
3562             destroy_vm($storecfg, $vmid);
3563         } else {
3564             die "VM $vmid is running - destroy failed\n";
3565         }
3566     });
3567 }
3568
3569 # pci helpers
3570
3571 sub file_write {
3572     my ($filename, $buf) = @_;
3573
3574     my $fh = IO::File->new($filename, "w");
3575     return undef if !$fh;
3576
3577     my $res = print $fh $buf;
3578
3579     $fh->close();
3580
3581     return $res;
3582 }
3583
3584 sub pci_device_info {
3585     my ($name) = @_;
3586
3587     my $res;
3588
3589     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
3590     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
3591
3592     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
3593     return undef if !defined($irq) || $irq !~ m/^\d+$/;
3594
3595     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
3596     return undef if !defined($vendor) || $vendor !~ s/^0x//;
3597
3598     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
3599     return undef if !defined($product) || $product !~ s/^0x//;
3600
3601     $res = {
3602         name => $name,
3603         vendor => $vendor,
3604         product => $product,
3605         domain => $domain,
3606         bus => $bus,
3607         slot => $slot,
3608         func => $func,
3609         irq => $irq,
3610         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
3611     };
3612
3613     return $res;
3614 }
3615
3616 sub pci_dev_reset {
3617     my ($dev) = @_;
3618
3619     my $name = $dev->{name};
3620
3621     my $fn = "$pcisysfs/devices/$name/reset";
3622
3623     return file_write($fn, "1");
3624 }
3625
3626 sub pci_dev_bind_to_stub {
3627     my ($dev) = @_;
3628
3629     my $name = $dev->{name};
3630
3631     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
3632     return 1 if -d $testdir;
3633
3634     my $data = "$dev->{vendor} $dev->{product}";
3635     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
3636
3637     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3638     if (!file_write($fn, $name)) {
3639         return undef if -f $fn;
3640     }
3641
3642     $fn = "$pcisysfs/drivers/pci-stub/bind";
3643     if (! -d $testdir) {
3644         return undef if !file_write($fn, $name);
3645     }
3646
3647     return -d $testdir;
3648 }
3649
3650 sub print_pci_addr {
3651     my ($id, $bridges) = @_;
3652
3653     my $res = '';
3654     my $devices = {
3655         piix3 => { bus => 0, addr => 1 },
3656         #addr2 : first videocard
3657         balloon0 => { bus => 0, addr => 3 },
3658         watchdog => { bus => 0, addr => 4 },
3659         scsihw0 => { bus => 0, addr => 5 },
3660         scsihw1 => { bus => 0, addr => 6 },
3661         ahci0 => { bus => 0, addr => 7 },
3662         qga0 => { bus => 0, addr => 8 },
3663         spice => { bus => 0, addr => 9 },
3664         virtio0 => { bus => 0, addr => 10 },
3665         virtio1 => { bus => 0, addr => 11 },
3666         virtio2 => { bus => 0, addr => 12 },
3667         virtio3 => { bus => 0, addr => 13 },
3668         virtio4 => { bus => 0, addr => 14 },
3669         virtio5 => { bus => 0, addr => 15 },
3670         hostpci0 => { bus => 0, addr => 16 },
3671         hostpci1 => { bus => 0, addr => 17 },
3672         net0 => { bus => 0, addr => 18 },
3673         net1 => { bus => 0, addr => 19 },
3674         net2 => { bus => 0, addr => 20 },
3675         net3 => { bus => 0, addr => 21 },
3676         net4 => { bus => 0, addr => 22 },
3677         net5 => { bus => 0, addr => 23 },
3678         vga1 => { bus => 0, addr => 24 },
3679         vga2 => { bus => 0, addr => 25 },
3680         vga3 => { bus => 0, addr => 26 },
3681         #addr29 : usb-host (pve-usb.cfg)
3682         'pci.1' => { bus => 0, addr => 30 },
3683         'pci.2' => { bus => 0, addr => 31 },
3684         'net6' => { bus => 1, addr => 1 },
3685         'net7' => { bus => 1, addr => 2 },
3686         'net8' => { bus => 1, addr => 3 },
3687         'net9' => { bus => 1, addr => 4 },
3688         'net10' => { bus => 1, addr => 5 },
3689         'net11' => { bus => 1, addr => 6 },
3690         'net12' => { bus => 1, addr => 7 },
3691         'net13' => { bus => 1, addr => 8 },
3692         'net14' => { bus => 1, addr => 9 },
3693         'net15' => { bus => 1, addr => 10 },
3694         'net16' => { bus => 1, addr => 11 },
3695         'net17' => { bus => 1, addr => 12 },
3696         'net18' => { bus => 1, addr => 13 },
3697         'net19' => { bus => 1, addr => 14 },
3698         'net20' => { bus => 1, addr => 15 },
3699         'net21' => { bus => 1, addr => 16 },
3700         'net22' => { bus => 1, addr => 17 },
3701         'net23' => { bus => 1, addr => 18 },
3702         'net24' => { bus => 1, addr => 19 },
3703         'net25' => { bus => 1, addr => 20 },
3704         'net26' => { bus => 1, addr => 21 },
3705         'net27' => { bus => 1, addr => 22 },
3706         'net28' => { bus => 1, addr => 23 },
3707         'net29' => { bus => 1, addr => 24 },
3708         'net30' => { bus => 1, addr => 25 },
3709         'net31' => { bus => 1, addr => 26 },
3710         'virtio6' => { bus => 2, addr => 1 },
3711         'virtio7' => { bus => 2, addr => 2 },
3712         'virtio8' => { bus => 2, addr => 3 },
3713         'virtio9' => { bus => 2, addr => 4 },
3714         'virtio10' => { bus => 2, addr => 5 },
3715         'virtio11' => { bus => 2, addr => 6 },
3716         'virtio12' => { bus => 2, addr => 7 },
3717         'virtio13' => { bus => 2, addr => 8 },
3718         'virtio14' => { bus => 2, addr => 9 },
3719         'virtio15' => { bus => 2, addr => 10 },
3720     };
3721
3722     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3723            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3724            my $bus = $devices->{$id}->{bus};
3725            $res = ",bus=pci.$bus,addr=$addr";
3726            $bridges->{$bus} = 1 if $bridges;
3727     }
3728     return $res;
3729
3730 }
3731
3732 # vzdump restore implementaion
3733
3734 sub tar_archive_read_firstfile {
3735     my $archive = shift;
3736
3737     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
3738
3739     # try to detect archive type first
3740     my $pid = open (TMP, "tar tf '$archive'|") ||
3741         die "unable to open file '$archive'\n";
3742     my $firstfile = <TMP>;
3743     kill 15, $pid;
3744     close TMP;
3745
3746     die "ERROR: archive contaions no data\n" if !$firstfile;
3747     chomp $firstfile;
3748
3749     return $firstfile;
3750 }
3751
3752 sub tar_restore_cleanup {
3753     my ($storecfg, $statfile) = @_;
3754
3755     print STDERR "starting cleanup\n";
3756
3757     if (my $fd = IO::File->new($statfile, "r")) {
3758         while (defined(my $line = <$fd>)) {
3759             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
3760                 my $volid = $2;
3761                 eval {
3762                     if ($volid =~ m|^/|) {
3763                         unlink $volid || die 'unlink failed\n';
3764                     } else {
3765                         PVE::Storage::vdisk_free($storecfg, $volid);
3766                     }
3767                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
3768                 };
3769                 print STDERR "unable to cleanup '$volid' - $@" if $@;
3770             } else {
3771                 print STDERR "unable to parse line in statfile - $line";
3772             }
3773         }
3774         $fd->close();
3775     }
3776 }
3777
3778 sub restore_archive {
3779     my ($archive, $vmid, $user, $opts) = @_;
3780
3781     my $format = $opts->{format};
3782     my $comp;
3783
3784     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
3785         $format = 'tar' if !$format;
3786         $comp = 'gzip';
3787     } elsif ($archive =~ m/\.tar$/) {
3788         $format = 'tar' if !$format;
3789     } elsif ($archive =~ m/.tar.lzo$/) {
3790         $format = 'tar' if !$format;
3791         $comp = 'lzop';
3792     } elsif ($archive =~ m/\.vma$/) {
3793         $format = 'vma' if !$format;
3794     } elsif ($archive =~ m/\.vma\.gz$/) {
3795         $format = 'vma' if !$format;
3796         $comp = 'gzip';
3797     } elsif ($archive =~ m/\.vma\.lzo$/) {
3798         $format = 'vma' if !$format;
3799         $comp = 'lzop';
3800     } else {
3801         $format = 'vma' if !$format; # default
3802     }
3803
3804     # try to detect archive format
3805     if ($format eq 'tar') {
3806         return restore_tar_archive($archive, $vmid, $user, $opts);
3807     } else {
3808         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
3809     }
3810 }
3811
3812 sub restore_update_config_line {
3813     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
3814
3815     return if $line =~ m/^\#qmdump\#/;
3816     return if $line =~ m/^\#vzdump\#/;
3817     return if $line =~ m/^lock:/;
3818     return if $line =~ m/^unused\d+:/;
3819     return if $line =~ m/^parent:/;
3820     return if $line =~ m/^template:/; # restored VM is never a template
3821
3822     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
3823         # try to convert old 1.X settings
3824         my ($id, $ind, $ethcfg) = ($1, $2, $3);
3825         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
3826             my ($model, $macaddr) = split(/\=/, $devconfig);
3827             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
3828             my $net = {
3829                 model => $model,
3830                 bridge => "vmbr$ind",
3831                 macaddr => $macaddr,
3832             };
3833             my $netstr = print_net($net);
3834
3835             print $outfd "net$cookie->{netcount}: $netstr\n";
3836             $cookie->{netcount}++;
3837         }
3838     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
3839         my ($id, $netstr) = ($1, $2);
3840         my $net = parse_net($netstr);
3841         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
3842         $netstr = print_net($net);
3843         print $outfd "$id: $netstr\n";
3844     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
3845         my $virtdev = $1;
3846         my $value = $3;
3847         if ($line =~ m/backup=no/) {
3848             print $outfd "#$line";
3849         } elsif ($virtdev && $map->{$virtdev}) {
3850             my $di = parse_drive($virtdev, $value);
3851             delete $di->{format}; # format can change on restore
3852             $di->{file} = $map->{$virtdev};
3853             $value = print_drive($vmid, $di);
3854             print $outfd "$virtdev: $value\n";
3855         } else {
3856             print $outfd $line;
3857         }
3858     } else {
3859         print $outfd $line;
3860     }
3861 }
3862
3863 sub scan_volids {
3864     my ($cfg, $vmid) = @_;
3865
3866     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
3867
3868     my $volid_hash = {};
3869     foreach my $storeid (keys %$info) {
3870         foreach my $item (@{$info->{$storeid}}) {
3871             next if !($item->{volid} && $item->{size});
3872             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
3873             $volid_hash->{$item->{volid}} = $item;
3874         }
3875     }
3876
3877     return $volid_hash;
3878 }
3879
3880 sub get_used_paths {
3881     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
3882
3883     my $used_path = {};
3884
3885     my $scan_config = sub {
3886         my ($cref, $snapname) = @_;
3887
3888         foreach my $key (keys %$cref) {
3889             my $value = $cref->{$key};
3890             if (valid_drivename($key)) {
3891                 next if $skip_drive && $key eq $skip_drive;
3892                 my $drive = parse_drive($key, $value);
3893                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
3894                 if ($drive->{file} =~ m!^/!) {
3895                     $used_path->{$drive->{file}}++; # = 1;
3896                 } else {
3897                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
3898                     next if !$storeid;
3899                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
3900                     next if !$scfg;
3901                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
3902                     $used_path->{$path}++; # = 1;
3903                 }
3904             }
3905         }
3906     };
3907
3908     &$scan_config($conf);
3909
3910     undef $skip_drive;
3911
3912     if ($scan_snapshots) {
3913         foreach my $snapname (keys %{$conf->{snapshots}}) {
3914             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
3915         }
3916     }
3917
3918     return $used_path;
3919 }
3920
3921 sub update_disksize {
3922     my ($vmid, $conf, $volid_hash) = @_;
3923
3924     my $changes;
3925
3926     my $used = {};
3927
3928     # Note: it is allowed to define multiple storages with same path (alias), so
3929     # we need to check both 'volid' and real 'path' (two different volid can point
3930     # to the same path).
3931
3932     my $usedpath = {};
3933
3934     # update size info
3935     foreach my $opt (keys %$conf) {
3936         if (valid_drivename($opt)) {
3937             my $drive = parse_drive($opt, $conf->{$opt});
3938             my $volid = $drive->{file};
3939             next if !$volid;
3940
3941             $used->{$volid} = 1;
3942             if ($volid_hash->{$volid} &&
3943                 (my $path = $volid_hash->{$volid}->{path})) {
3944                 $usedpath->{$path} = 1;
3945             }
3946
3947             next if drive_is_cdrom($drive);
3948             next if !$volid_hash->{$volid};
3949
3950             $drive->{size} = $volid_hash->{$volid}->{size};
3951             my $new = print_drive($vmid, $drive);
3952             if ($new ne $conf->{$opt}) {
3953                 $changes = 1;
3954                 $conf->{$opt} = $new;
3955             }
3956         }
3957     }
3958
3959     # remove 'unusedX' entry if volume is used
3960     foreach my $opt (keys %$conf) {
3961         next if $opt !~ m/^unused\d+$/;
3962         my $volid = $conf->{$opt};
3963         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
3964         if ($used->{$volid} || ($path && $usedpath->{$path})) {
3965             $changes = 1;
3966             delete $conf->{$opt};
3967         }
3968     }
3969
3970     foreach my $volid (sort keys %$volid_hash) {
3971         next if $volid =~ m/vm-$vmid-state-/;
3972         next if $used->{$volid};
3973         my $path = $volid_hash->{$volid}->{path};
3974         next if !$path; # just to be sure
3975         next if $usedpath->{$path};
3976         $changes = 1;
3977         add_unused_volume($conf, $volid);
3978         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
3979     }
3980
3981     return $changes;
3982 }
3983
3984 sub rescan {
3985     my ($vmid, $nolock) = @_;
3986
3987     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
3988
3989     my $volid_hash = scan_volids($cfg, $vmid);
3990
3991     my $updatefn =  sub {
3992         my ($vmid) = @_;
3993
3994         my $conf = load_config($vmid);
3995
3996         check_lock($conf);
3997
3998         my $vm_volids = {};
3999         foreach my $volid (keys %$volid_hash) {
4000             my $info = $volid_hash->{$volid};
4001             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
4002         }
4003
4004         my $changes = update_disksize($vmid, $conf, $vm_volids);
4005
4006         update_config_nolock($vmid, $conf, 1) if $changes;
4007     };
4008
4009     if (defined($vmid)) {
4010         if ($nolock) {
4011             &$updatefn($vmid);
4012         } else {
4013             lock_config($vmid, $updatefn, $vmid);
4014         }
4015     } else {
4016         my $vmlist = config_list();
4017         foreach my $vmid (keys %$vmlist) {
4018             if ($nolock) {
4019                 &$updatefn($vmid);
4020             } else {
4021                 lock_config($vmid, $updatefn, $vmid);
4022             }
4023         }
4024     }
4025 }
4026
4027 sub restore_vma_archive {
4028     my ($archive, $vmid, $user, $opts, $comp) = @_;
4029
4030     my $input = $archive eq '-' ? "<&STDIN" : undef;
4031     my $readfrom = $archive;
4032
4033     my $uncomp = '';
4034     if ($comp) {
4035         $readfrom = '-';
4036         my $qarchive = PVE::Tools::shellquote($archive);
4037         if ($comp eq 'gzip') {
4038             $uncomp = "zcat $qarchive|";
4039         } elsif ($comp eq 'lzop') {
4040             $uncomp = "lzop -d -c $qarchive|";
4041         } else {
4042             die "unknown compression method '$comp'\n";
4043         }
4044
4045     }
4046
4047     my $tmpdir = "/var/tmp/vzdumptmp$$";
4048     rmtree $tmpdir;
4049
4050     # disable interrupts (always do cleanups)
4051     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4052         warn "got interrupt - ignored\n";
4053     };
4054
4055     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
4056     POSIX::mkfifo($mapfifo, 0600);
4057     my $fifofh;
4058
4059     my $openfifo = sub {
4060         open($fifofh, '>', $mapfifo) || die $!;
4061     };
4062
4063     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
4064
4065     my $oldtimeout;
4066     my $timeout = 5;
4067
4068     my $devinfo = {};
4069
4070     my $rpcenv = PVE::RPCEnvironment::get();
4071
4072     my $conffile = config_file($vmid);
4073     my $tmpfn = "$conffile.$$.tmp";
4074
4075     # Note: $oldconf is undef if VM does not exists
4076     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
4077
4078     my $print_devmap = sub {
4079         my $virtdev_hash = {};
4080
4081         my $cfgfn = "$tmpdir/qemu-server.conf";
4082
4083         # we can read the config - that is already extracted
4084         my $fh = IO::File->new($cfgfn, "r") ||
4085             "unable to read qemu-server.conf - $!\n";
4086
4087         while (defined(my $line = <$fh>)) {
4088             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
4089                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
4090                 die "archive does not contain data for drive '$virtdev'\n"
4091                     if !$devinfo->{$devname};
4092                 if (defined($opts->{storage})) {
4093                     $storeid = $opts->{storage} || 'local';
4094                 } elsif (!$storeid) {
4095                     $storeid = 'local';
4096                 }
4097                 $format = 'raw' if !$format;
4098                 $devinfo->{$devname}->{devname} = $devname;
4099                 $devinfo->{$devname}->{virtdev} = $virtdev;
4100                 $devinfo->{$devname}->{format} = $format;
4101                 $devinfo->{$devname}->{storeid} = $storeid;
4102
4103                 # check permission on storage
4104                 my $pool = $opts->{pool}; # todo: do we need that?
4105                 if ($user ne 'root@pam') {
4106                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
4107                 }
4108
4109                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
4110             }
4111         }
4112
4113         foreach my $devname (keys %$devinfo) {
4114             die "found no device mapping information for device '$devname'\n"
4115                 if !$devinfo->{$devname}->{virtdev};
4116         }
4117
4118         my $cfg = cfs_read_file('storage.cfg');
4119
4120         # create empty/temp config
4121         if ($oldconf) {
4122             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
4123             foreach_drive($oldconf, sub {
4124                 my ($ds, $drive) = @_;
4125
4126                 return if drive_is_cdrom($drive);
4127
4128                 my $volid = $drive->{file};
4129
4130                 return if !$volid || $volid =~ m|^/|;
4131
4132                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
4133                 return if !$path || !$owner || ($owner != $vmid);
4134
4135                 # Note: only delete disk we want to restore
4136                 # other volumes will become unused
4137                 if ($virtdev_hash->{$ds}) {
4138                     PVE::Storage::vdisk_free($cfg, $volid);
4139                 }
4140             });
4141         }
4142
4143         my $map = {};
4144         foreach my $virtdev (sort keys %$virtdev_hash) {
4145             my $d = $virtdev_hash->{$virtdev};
4146             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
4147             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
4148
4149             # test if requested format is supported
4150             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
4151             my $supported = grep { $_ eq $d->{format} } @$validFormats;
4152             $d->{format} = $defFormat if !$supported;
4153
4154             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
4155                                                   $d->{format}, undef, $alloc_size);
4156             print STDERR "new volume ID is '$volid'\n";
4157             $d->{volid} = $volid;
4158             my $path = PVE::Storage::path($cfg, $volid);
4159
4160             my $write_zeros = 1;
4161             # fixme: what other storages types initialize volumes with zero?
4162             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
4163                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
4164                 $write_zeros = 0;
4165             }
4166
4167             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
4168
4169             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
4170             $map->{$virtdev} = $volid;
4171         }
4172
4173         $fh->seek(0, 0) || die "seek failed - $!\n";
4174
4175         my $outfd = new IO::File ($tmpfn, "w") ||
4176             die "unable to write config for VM $vmid\n";
4177
4178         my $cookie = { netcount => 0 };
4179         while (defined(my $line = <$fh>)) {
4180             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4181         }
4182
4183         $fh->close();
4184         $outfd->close();
4185     };
4186
4187     eval {
4188         # enable interrupts
4189         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4190             die "interrupted by signal\n";
4191         };
4192         local $SIG{ALRM} = sub { die "got timeout\n"; };
4193
4194         $oldtimeout = alarm($timeout);
4195
4196         my $parser = sub {
4197             my $line = shift;
4198
4199             print "$line\n";
4200
4201             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
4202                 my ($dev_id, $size, $devname) = ($1, $2, $3);
4203                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
4204             } elsif ($line =~ m/^CTIME: /) {
4205                 # we correctly received the vma config, so we can disable
4206                 # the timeout now for disk allocation (set to 10 minutes, so
4207                 # that we always timeout if something goes wrong)
4208                 alarm(600);
4209                 &$print_devmap();
4210                 print $fifofh "done\n";
4211                 my $tmp = $oldtimeout || 0;
4212                 $oldtimeout = undef;
4213                 alarm($tmp);
4214                 close($fifofh);
4215             }
4216         };
4217
4218         print "restore vma archive: $cmd\n";
4219         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
4220     };
4221     my $err = $@;
4222
4223     alarm($oldtimeout) if $oldtimeout;
4224
4225     unlink $mapfifo;
4226
4227     if ($err) {
4228         rmtree $tmpdir;
4229         unlink $tmpfn;
4230
4231         my $cfg = cfs_read_file('storage.cfg');
4232         foreach my $devname (keys %$devinfo) {
4233             my $volid = $devinfo->{$devname}->{volid};
4234             next if !$volid;
4235             eval {
4236                 if ($volid =~ m|^/|) {
4237                     unlink $volid || die 'unlink failed\n';
4238                 } else {
4239                     PVE::Storage::vdisk_free($cfg, $volid);
4240                 }
4241                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
4242             };
4243             print STDERR "unable to cleanup '$volid' - $@" if $@;
4244         }
4245         die $err;
4246     }
4247
4248     rmtree $tmpdir;
4249
4250     rename($tmpfn, $conffile) ||
4251         die "unable to commit configuration file '$conffile'\n";
4252
4253     PVE::Cluster::cfs_update(); # make sure we read new file
4254
4255     eval { rescan($vmid, 1); };
4256     warn $@ if $@;
4257 }
4258
4259 sub restore_tar_archive {
4260     my ($archive, $vmid, $user, $opts) = @_;
4261
4262     if ($archive ne '-') {
4263         my $firstfile = tar_archive_read_firstfile($archive);
4264         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
4265             if $firstfile ne 'qemu-server.conf';
4266     }
4267
4268     my $storecfg = cfs_read_file('storage.cfg');
4269
4270     # destroy existing data - keep empty config
4271     my $vmcfgfn = PVE::QemuServer::config_file($vmid);
4272     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
4273
4274     my $tocmd = "/usr/lib/qemu-server/qmextract";
4275
4276     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
4277     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
4278     $tocmd .= ' --prealloc' if $opts->{prealloc};
4279     $tocmd .= ' --info' if $opts->{info};
4280
4281     # tar option "xf" does not autodetect compression when read from STDIN,
4282     # so we pipe to zcat
4283     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
4284         PVE::Tools::shellquote("--to-command=$tocmd");
4285
4286     my $tmpdir = "/var/tmp/vzdumptmp$$";
4287     mkpath $tmpdir;
4288
4289     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
4290     local $ENV{VZDUMP_VMID} = $vmid;
4291     local $ENV{VZDUMP_USER} = $user;
4292
4293     my $conffile = config_file($vmid);
4294     my $tmpfn = "$conffile.$$.tmp";
4295
4296     # disable interrupts (always do cleanups)
4297     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4298         print STDERR "got interrupt - ignored\n";
4299     };
4300
4301     eval {
4302         # enable interrupts
4303         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4304             die "interrupted by signal\n";
4305         };
4306
4307         if ($archive eq '-') {
4308             print "extracting archive from STDIN\n";
4309             run_command($cmd, input => "<&STDIN");
4310         } else {
4311             print "extracting archive '$archive'\n";
4312             run_command($cmd);
4313         }
4314
4315         return if $opts->{info};
4316
4317         # read new mapping
4318         my $map = {};
4319         my $statfile = "$tmpdir/qmrestore.stat";
4320         if (my $fd = IO::File->new($statfile, "r")) {
4321             while (defined (my $line = <$fd>)) {
4322                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4323                     $map->{$1} = $2 if $1;
4324                 } else {
4325                     print STDERR "unable to parse line in statfile - $line\n";
4326                 }
4327             }
4328             $fd->close();
4329         }
4330
4331         my $confsrc = "$tmpdir/qemu-server.conf";
4332
4333         my $srcfd = new IO::File($confsrc, "r") ||
4334             die "unable to open file '$confsrc'\n";
4335
4336         my $outfd = new IO::File ($tmpfn, "w") ||
4337             die "unable to write config for VM $vmid\n";
4338
4339         my $cookie = { netcount => 0 };
4340         while (defined (my $line = <$srcfd>)) {
4341             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4342         }
4343
4344         $srcfd->close();
4345         $outfd->close();
4346     };
4347     my $err = $@;
4348
4349     if ($err) {
4350
4351         unlink $tmpfn;
4352
4353         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
4354
4355         die $err;
4356     }
4357
4358     rmtree $tmpdir;
4359
4360     rename $tmpfn, $conffile ||
4361         die "unable to commit configuration file '$conffile'\n";
4362
4363     PVE::Cluster::cfs_update(); # make sure we read new file
4364
4365     eval { rescan($vmid, 1); };
4366     warn $@ if $@;
4367 };
4368
4369
4370 # Internal snapshots
4371
4372 # NOTE: Snapshot create/delete involves several non-atomic
4373 # action, and can take a long time.
4374 # So we try to avoid locking the file and use 'lock' variable
4375 # inside the config file instead.
4376
4377 my $snapshot_copy_config = sub {
4378     my ($source, $dest) = @_;
4379
4380     foreach my $k (keys %$source) {
4381         next if $k eq 'snapshots';
4382         next if $k eq 'snapstate';
4383         next if $k eq 'snaptime';
4384         next if $k eq 'vmstate';
4385         next if $k eq 'lock';
4386         next if $k eq 'digest';
4387         next if $k eq 'description';
4388         next if $k =~ m/^unused\d+$/;
4389
4390         $dest->{$k} = $source->{$k};
4391     }
4392 };
4393
4394 my $snapshot_apply_config = sub {
4395     my ($conf, $snap) = @_;
4396
4397     # copy snapshot list
4398     my $newconf = {
4399         snapshots => $conf->{snapshots},
4400     };
4401
4402     # keep description and list of unused disks
4403     foreach my $k (keys %$conf) {
4404         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
4405         $newconf->{$k} = $conf->{$k};
4406     }
4407
4408     &$snapshot_copy_config($snap, $newconf);
4409
4410     return $newconf;
4411 };
4412
4413 sub foreach_writable_storage {
4414     my ($conf, $func) = @_;
4415
4416     my $sidhash = {};
4417
4418     foreach my $ds (keys %$conf) {
4419         next if !valid_drivename($ds);
4420
4421         my $drive = parse_drive($ds, $conf->{$ds});
4422         next if !$drive;
4423         next if drive_is_cdrom($drive);
4424
4425         my $volid = $drive->{file};
4426
4427         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4428         $sidhash->{$sid} = $sid if $sid;
4429     }
4430
4431     foreach my $sid (sort keys %$sidhash) {
4432         &$func($sid);
4433     }
4434 }
4435
4436 my $alloc_vmstate_volid = sub {
4437     my ($storecfg, $vmid, $conf, $snapname) = @_;
4438
4439     # Note: we try to be smart when selecting a $target storage
4440
4441     my $target;
4442
4443     # search shared storage first
4444     foreach_writable_storage($conf, sub {
4445         my ($sid) = @_;
4446         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4447         return if !$scfg->{shared};
4448
4449         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
4450     });
4451
4452     if (!$target) {
4453         # now search local storage
4454         foreach_writable_storage($conf, sub {
4455             my ($sid) = @_;
4456             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4457             return if $scfg->{shared};
4458
4459             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
4460         });
4461     }
4462
4463     $target = 'local' if !$target;
4464
4465     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
4466     # we abort live save after $conf->{memory}, so we need at max twice that space
4467     my $size = $conf->{memory}*2 + $driver_state_size;
4468
4469     my $name = "vm-$vmid-state-$snapname";
4470     my $scfg = PVE::Storage::storage_config($storecfg, $target);
4471     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
4472     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
4473
4474     return $volid;
4475 };
4476
4477 my $snapshot_prepare = sub {
4478     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
4479
4480     my $snap;
4481
4482     my $updatefn =  sub {
4483
4484         my $conf = load_config($vmid);
4485
4486         die "you can't take a snapshot if it's a template\n"
4487             if is_template($conf);
4488
4489         check_lock($conf);
4490
4491         $conf->{lock} = 'snapshot';
4492
4493         die "snapshot name '$snapname' already used\n"
4494             if defined($conf->{snapshots}->{$snapname});
4495
4496         my $storecfg = PVE::Storage::config();
4497         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
4498
4499         $snap = $conf->{snapshots}->{$snapname} = {};
4500
4501         if ($save_vmstate && check_running($vmid)) {
4502             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
4503         }
4504
4505         &$snapshot_copy_config($conf, $snap);
4506
4507         $snap->{snapstate} = "prepare";
4508         $snap->{snaptime} = time();
4509         $snap->{description} = $comment if $comment;
4510
4511         # always overwrite machine if we save vmstate. This makes sure we
4512         # can restore it later using correct machine type
4513         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
4514
4515         update_config_nolock($vmid, $conf, 1);
4516     };
4517
4518     lock_config($vmid, $updatefn);
4519
4520     return $snap;
4521 };
4522
4523 my $snapshot_commit = sub {
4524     my ($vmid, $snapname) = @_;
4525
4526     my $updatefn = sub {
4527
4528         my $conf = load_config($vmid);
4529
4530         die "missing snapshot lock\n"
4531             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
4532
4533         my $snap = $conf->{snapshots}->{$snapname};
4534
4535         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4536
4537         die "wrong snapshot state\n"
4538             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
4539
4540         delete $snap->{snapstate};
4541         delete $conf->{lock};
4542
4543         my $newconf = &$snapshot_apply_config($conf, $snap);
4544
4545         $newconf->{parent} = $snapname;
4546
4547         update_config_nolock($vmid, $newconf, 1);
4548     };
4549
4550     lock_config($vmid, $updatefn);
4551 };
4552
4553 sub snapshot_rollback {
4554     my ($vmid, $snapname) = @_;
4555
4556     my $snap;
4557
4558     my $prepare = 1;
4559
4560     my $storecfg = PVE::Storage::config();
4561
4562     my $updatefn = sub {
4563
4564         my $conf = load_config($vmid);
4565
4566         die "you can't rollback if vm is a template\n" if is_template($conf);
4567
4568         $snap = $conf->{snapshots}->{$snapname};
4569
4570         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4571
4572         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
4573             if $snap->{snapstate};
4574
4575         if ($prepare) {
4576             check_lock($conf);
4577             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
4578         }
4579
4580         die "unable to rollback vm $vmid: vm is running\n"
4581             if check_running($vmid);
4582
4583         if ($prepare) {
4584             $conf->{lock} = 'rollback';
4585         } else {
4586             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
4587             delete $conf->{lock};
4588         }
4589
4590         my $forcemachine;
4591
4592         if (!$prepare) {
4593             my $has_machine_config = defined($conf->{machine});
4594
4595             # copy snapshot config to current config
4596             $conf = &$snapshot_apply_config($conf, $snap);
4597             $conf->{parent} = $snapname;
4598
4599             # Note: old code did not store 'machine', so we try to be smart
4600             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
4601             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
4602             # we remove the 'machine' configuration if not explicitly specified
4603             # in the original config.
4604             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
4605         }
4606
4607         update_config_nolock($vmid, $conf, 1);
4608
4609         if (!$prepare && $snap->{vmstate}) {
4610             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
4611             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
4612         }
4613     };
4614
4615     lock_config($vmid, $updatefn);
4616
4617     foreach_drive($snap, sub {
4618         my ($ds, $drive) = @_;
4619
4620         return if drive_is_cdrom($drive);
4621
4622         my $volid = $drive->{file};
4623         my $device = "drive-$ds";
4624
4625         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
4626     });
4627
4628     $prepare = 0;
4629     lock_config($vmid, $updatefn);
4630 }
4631
4632 my $savevm_wait = sub {
4633     my ($vmid) = @_;
4634
4635     for(;;) {
4636         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
4637         if (!$stat->{status}) {
4638             die "savevm not active\n";
4639         } elsif ($stat->{status} eq 'active') {
4640             sleep(1);
4641             next;
4642         } elsif ($stat->{status} eq 'completed') {
4643             last;
4644         } else {
4645             die "query-savevm returned status '$stat->{status}'\n";
4646         }
4647     }
4648 };
4649
4650 sub snapshot_create {
4651     my ($vmid, $snapname, $save_vmstate, $freezefs, $comment) = @_;
4652
4653     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
4654
4655     $freezefs = $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
4656
4657     my $drivehash = {};
4658
4659     my $running = check_running($vmid);
4660
4661     eval {
4662         # create internal snapshots of all drives
4663
4664         my $storecfg = PVE::Storage::config();
4665
4666         if ($running) {
4667             if ($snap->{vmstate}) {
4668                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
4669                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
4670                 &$savevm_wait($vmid);
4671             } else {
4672                 vm_mon_cmd($vmid, "savevm-start");
4673             }
4674         };
4675
4676         qga_freezefs($vmid) if $running && $freezefs;
4677
4678         foreach_drive($snap, sub {
4679             my ($ds, $drive) = @_;
4680
4681             return if drive_is_cdrom($drive);
4682
4683             my $volid = $drive->{file};
4684             my $device = "drive-$ds";
4685
4686             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
4687             $drivehash->{$ds} = 1;
4688        });
4689     };
4690     my $err = $@;
4691
4692     eval { qga_unfreezefs($vmid) if $running && $freezefs; };
4693     warn $@ if $@;
4694
4695     eval { vm_mon_cmd($vmid, "savevm-end") if $running; };
4696     warn $@ if $@;
4697
4698     if ($err) {
4699         warn "snapshot create failed: starting cleanup\n";
4700         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
4701         warn $@ if $@;
4702         die $err;
4703     }
4704
4705     &$snapshot_commit($vmid, $snapname);
4706 }
4707
4708 # Note: $drivehash is only set when called from snapshot_create.
4709 sub snapshot_delete {
4710     my ($vmid, $snapname, $force, $drivehash) = @_;
4711
4712     my $prepare = 1;
4713
4714     my $snap;
4715     my $unused = [];
4716
4717     my $unlink_parent = sub {
4718         my ($confref, $new_parent) = @_;
4719
4720         if ($confref->{parent} && $confref->{parent} eq $snapname) {
4721             if ($new_parent) {
4722                 $confref->{parent} = $new_parent;
4723             } else {
4724                 delete $confref->{parent};
4725             }
4726         }
4727     };
4728
4729     my $updatefn =  sub {
4730         my ($remove_drive) = @_;
4731
4732         my $conf = load_config($vmid);
4733
4734         if (!$drivehash) {
4735             check_lock($conf);
4736             die "you can't delete a snapshot if vm is a template\n"
4737                 if is_template($conf);
4738         }
4739
4740         $snap = $conf->{snapshots}->{$snapname};
4741
4742         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4743
4744         # remove parent refs
4745         &$unlink_parent($conf, $snap->{parent});
4746         foreach my $sn (keys %{$conf->{snapshots}}) {
4747             next if $sn eq $snapname;
4748             &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
4749         }
4750
4751         if ($remove_drive) {
4752             if ($remove_drive eq 'vmstate') {
4753                 delete $snap->{$remove_drive};
4754             } else {
4755                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
4756                 my $volid = $drive->{file};
4757                 delete $snap->{$remove_drive};
4758                 add_unused_volume($conf, $volid);
4759             }
4760         }
4761
4762         if ($prepare) {
4763             $snap->{snapstate} = 'delete';
4764         } else {
4765             delete $conf->{snapshots}->{$snapname};
4766             delete $conf->{lock} if $drivehash;
4767             foreach my $volid (@$unused) {
4768                 add_unused_volume($conf, $volid);
4769             }
4770         }
4771
4772         update_config_nolock($vmid, $conf, 1);
4773     };
4774
4775     lock_config($vmid, $updatefn);
4776
4777     # now remove vmstate file
4778
4779     my $storecfg = PVE::Storage::config();
4780
4781     if ($snap->{vmstate}) {
4782         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
4783         if (my $err = $@) {
4784             die $err if !$force;
4785             warn $err;
4786         }
4787         # save changes (remove vmstate from snapshot)
4788         lock_config($vmid, $updatefn, 'vmstate') if !$force;
4789     };
4790
4791     # now remove all internal snapshots
4792     foreach_drive($snap, sub {
4793         my ($ds, $drive) = @_;
4794
4795         return if drive_is_cdrom($drive);
4796
4797         my $volid = $drive->{file};
4798         my $device = "drive-$ds";
4799
4800         if (!$drivehash || $drivehash->{$ds}) {
4801             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
4802             if (my $err = $@) {
4803                 die $err if !$force;
4804                 warn $err;
4805             }
4806         }
4807
4808         # save changes (remove drive fron snapshot)
4809         lock_config($vmid, $updatefn, $ds) if !$force;
4810         push @$unused, $volid;
4811     });
4812
4813     # now cleanup config
4814     $prepare = 0;
4815     lock_config($vmid, $updatefn);
4816 }
4817
4818 sub has_feature {
4819     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
4820
4821     my $err;
4822     foreach_drive($conf, sub {
4823         my ($ds, $drive) = @_;
4824
4825         return if drive_is_cdrom($drive);
4826         my $volid = $drive->{file};
4827         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
4828     });
4829
4830     return $err ? 0 : 1;
4831 }
4832
4833 sub template_create {
4834     my ($vmid, $conf, $disk) = @_;
4835
4836     my $storecfg = PVE::Storage::config();
4837
4838     foreach_drive($conf, sub {
4839         my ($ds, $drive) = @_;
4840
4841         return if drive_is_cdrom($drive);
4842         return if $disk && $ds ne $disk;
4843
4844         my $volid = $drive->{file};
4845         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
4846
4847         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
4848         $drive->{file} = $voliddst;
4849         $conf->{$ds} = print_drive($vmid, $drive);
4850         update_config_nolock($vmid, $conf, 1);
4851     });
4852 }
4853
4854 sub is_template {
4855     my ($conf) = @_;
4856
4857     return 1 if defined $conf->{template} && $conf->{template} == 1;
4858 }
4859
4860 sub qemu_img_convert {
4861     my ($src_volid, $dst_volid, $size, $snapname) = @_;
4862
4863     my $storecfg = PVE::Storage::config();
4864     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
4865     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
4866
4867     if ($src_storeid && $dst_storeid) {
4868         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
4869         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
4870
4871         my $src_format = qemu_img_format($src_scfg, $src_volname);
4872         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
4873
4874         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
4875         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
4876
4877         my $cmd = [];
4878         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
4879         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
4880         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
4881
4882         my $parser = sub {
4883             my $line = shift;
4884             if($line =~ m/\((\S+)\/100\%\)/){
4885                 my $percent = $1;
4886                 my $transferred = int($size * $percent / 100);
4887                 my $remaining = $size - $transferred;
4888
4889                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
4890             }
4891
4892         };
4893
4894         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
4895         my $err = $@;
4896         die "copy failed: $err" if $err;
4897     }
4898 }
4899
4900 sub qemu_img_format {
4901     my ($scfg, $volname) = @_;
4902
4903     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
4904         return $1;
4905     } elsif ($scfg->{type} eq 'iscsi') {
4906         return "host_device";
4907     } else {
4908         return "raw";
4909     }
4910 }
4911
4912 sub qemu_drive_mirror {
4913     my ($vmid, $drive, $dst_volid, $vmiddst, $maxwait) = @_;
4914
4915     my $count = 1;
4916     my $old_len = 0;
4917     my $frozen = undef;
4918
4919     my $storecfg = PVE::Storage::config();
4920     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
4921
4922     if ($dst_storeid) {
4923         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
4924
4925         my $format;
4926         if ($dst_volname =~ m/\.(raw|qcow2)$/){
4927             $format = $1;
4928         }
4929
4930         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
4931
4932         if ($format) {
4933             #fixme : sometime drive-mirror timeout, but works fine after.
4934             # (I have see the problem with big volume > 200GB), so we need to eval
4935             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
4936                               sync => "full", target => $dst_path, format => $format); };
4937         } else {
4938             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
4939                               sync => "full", target => $dst_path); };
4940         }
4941
4942         eval {
4943             while (1) {
4944                 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
4945                 my $stat = @$stats[0];
4946                 die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
4947                 die "error job is not mirroring" if $stat->{type} ne "mirror";
4948
4949                 my $transferred = $stat->{offset};
4950                 my $total = $stat->{len};
4951                 my $remaining = $total - $transferred;
4952                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
4953
4954                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent %\n";
4955
4956                 last if ($stat->{len} == $stat->{offset});
4957                 if ($old_len == $stat->{offset}) {
4958                     if ($maxwait && $count > $maxwait) {
4959                     # if writes to disk occurs the disk needs to be freezed
4960                     # to be able to complete the migration
4961                         vm_suspend($vmid,1);
4962                         $count = 0;
4963                         $frozen = 1;
4964                     } else {
4965                         $count++ unless $frozen;
4966                     }
4967                 } elsif ($frozen) {
4968                     vm_resume($vmid,1);
4969                     $count = 0;
4970                 }
4971                 $old_len = $stat->{offset};
4972                 sleep 1;
4973             }
4974
4975             if ($vmiddst == $vmid) {
4976                 # switch the disk if source and destination are on the same guest
4977                 vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive");
4978             }
4979         };
4980         if (my $err = $@) {
4981             eval { vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive"); };
4982             die "mirroring error: $err";
4983         }
4984
4985         if ($vmiddst != $vmid) {
4986             # if we clone a disk for a new target vm, we don't switch the disk
4987             vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
4988         }
4989     }
4990 }
4991
4992 sub clone_disk {
4993     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
4994         $newvmid, $storage, $format, $full, $newvollist) = @_;
4995
4996     my $newvolid;
4997
4998     if (!$full) {
4999         print "create linked clone of drive $drivename ($drive->{file})\n";
5000         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid);
5001         push @$newvollist, $newvolid;
5002     } else {
5003         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5004         $storeid = $storage if $storage;
5005
5006         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5007         if (!$format) {
5008             $format = $drive->{format} || $defFormat;
5009         }
5010
5011         # test if requested format is supported - else use default
5012         my $supported = grep { $_ eq $format } @$validFormats;
5013         $format = $defFormat if !$supported;
5014
5015         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5016
5017         print "create full clone of drive $drivename ($drive->{file})\n";
5018         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5019         push @$newvollist, $newvolid;
5020
5021         if (!$running || $snapname) {
5022             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5023         } else {
5024             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5025         }
5026     }
5027
5028     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5029
5030     my $disk = $drive;
5031     $disk->{format} = undef;
5032     $disk->{file} = $newvolid;
5033     $disk->{size} = $size;
5034
5035     return $disk;
5036 }
5037
5038 # this only works if VM is running
5039 sub get_current_qemu_machine {
5040     my ($vmid) = @_;
5041
5042     my $cmd = { execute => 'query-machines', arguments => {} };
5043     my $res = PVE::QemuServer::vm_qmp_command($vmid, $cmd);
5044
5045     my ($current, $default);
5046     foreach my $e (@$res) {
5047         $default = $e->{name} if $e->{'is-default'};
5048         $current = $e->{name} if $e->{'is-current'};
5049     }
5050
5051     # fallback to the default machine if current is not supported by qemu
5052     return $current || $default || 'pc';
5053 }
5054
5055 1;