PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs'
 182 );
 183 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 184
 185 my $cpu_fmt = {
 186     cputype => {
 187         description => "Emulated CPU type.",
 188         type => 'string',
 189         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 190         default => 'kvm64',
 191         default_key => 1,
 192     },
 193     hidden => {
 194         description => "Do not identify as a KVM virtual machine.",
 195         type => 'boolean',
 196         optional => 1,
 197         default => 0
 198     },
 199     'hv-vendor-id' => {
 200         type => 'string',
 201         pattern => qr/[a-zA-Z0-9]{1,12}/,
 202         format_description => 'vendor-id',
 203         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 204         optional => 1,
 205     },
 206     flags => {
 207         description => "List of additional CPU flags separated by ';'."
 208                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 209                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 210         format_description => '+FLAG[;-FLAG...]',
 211         type => 'string',
 212         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 213         optional => 1,
 214     },
 215 };
 216
 217 my $watchdog_fmt = {
 218     model => {
 219         default_key => 1,
 220         type => 'string',
 221         enum => [qw(i6300esb ib700)],
 222         description => "Watchdog type to emulate.",
 223         default => 'i6300esb',
 224         optional => 1,
 225     },
 226     action => {
 227         type => 'string',
 228         enum => [qw(reset shutdown poweroff pause debug none)],
 229         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 230         optional => 1,
 231     },
 232 };
 233 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 234
 235 my $agent_fmt = {
 236     enabled => {
 237         description => "Enable/disable Qemu GuestAgent.",
 238         type => 'boolean',
 239         default => 0,
 240         default_key => 1,
 241     },
 242     fstrim_cloned_disks => {
 243         description => "Run fstrim after cloning/moving a disk.",
 244         type => 'boolean',
 245         optional => 1,
 246         default => 0
 247     },
 248 };
 249
 250 my $vga_fmt = {
 251     type => {
 252         description => "Select the VGA type.",
 253         type => 'string',
 254         default => 'std',
 255         optional => 1,
 256         default_key => 1,
 257         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 258     },
 259     memory => {
 260         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 261         type => 'integer',
 262         optional => 1,
 263         minimum => 4,
 264         maximum => 512,
 265     },
 266 };
 267
 268 my $ivshmem_fmt = {
 269     size => {
 270         type => 'integer',
 271         minimum => 1,
 272         description => "The size of the file in MB.",
 273     },
 274     name => {
 275         type => 'string',
 276         pattern => '[a-zA-Z0-9\-]+',
 277         optional => 1,
 278         format_description => 'string',
 279         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 280     },
 281 };
 282
 283 my $audio_fmt = {
 284     device => {
 285         type => 'string',
 286         enum => [qw(ich9-intel-hda intel-hda AC97)],
 287         description =>  "Configure an audio device."
 288     },
 289     driver =>  {
 290         type => 'string',
 291         enum => ['spice'],
 292         default => 'spice',
 293         optional => 1,
 294         description => "Driver backend for the audio device."
 295     },
 296 };
 297
 298 my $confdesc = {
 299     onboot => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Specifies whether a VM will be started during system bootup.",
 303         default => 0,
 304     },
 305     autostart => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Automatic restart after crash (currently ignored).",
 309         default => 0,
 310     },
 311     hotplug => {
 312         optional => 1,
 313         type => 'string', format => 'pve-hotplug-features',
 314         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 315         default => 'network,disk,usb',
 316     },
 317     reboot => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 321         default => 1,
 322     },
 323     lock => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Lock/unlock the VM.",
 327         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 328     },
 329     cpulimit => {
 330         optional => 1,
 331         type => 'number',
 332         description => "Limit of CPU usage.",
 333         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 334         minimum => 0,
 335         maximum => 128,
 336         default => 0,
 337     },
 338     cpuunits => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "CPU weight for a VM.",
 342         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 343         minimum => 2,
 344         maximum => 262144,
 345         default => 1024,
 346     },
 347     memory => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 351         minimum => 16,
 352         default => 512,
 353     },
 354     balloon => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 358         minimum => 0,
 359     },
 360     shares => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 364         minimum => 0,
 365         maximum => 50000,
 366         default => 1000,
 367     },
 368     keyboard => {
 369         optional => 1,
 370         type => 'string',
 371         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 372                        "It should not be necessary to set it.",
 373         enum => PVE::Tools::kvmkeymaplist(),
 374         default => undef,
 375     },
 376     name => {
 377         optional => 1,
 378         type => 'string', format => 'dns-name',
 379         description => "Set a name for the VM. Only used on the configuration web interface.",
 380     },
 381     scsihw => {
 382         optional => 1,
 383         type => 'string',
 384         description => "SCSI controller model",
 385         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 386         default => 'lsi',
 387     },
 388     description => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6/3.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string',
 420         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 421         pattern => '[acdn]{1,4}',
 422         default => 'cdn',
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     vcpus => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "Number of hotplugged vcpus.",
 467         minimum => 1,
 468         default => 0,
 469     },
 470     acpi => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable ACPI.",
 474         default => 1,
 475     },
 476     agent => {
 477         optional => 1,
 478         description => "Enable/disable Qemu GuestAgent and its properties.",
 479         type => 'string',
 480         format => $agent_fmt,
 481     },
 482     kvm => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Enable/disable KVM hardware virtualization.",
 486         default => 1,
 487     },
 488     tdf => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable time drift fix.",
 492         default => 0,
 493     },
 494     localtime => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 498     },
 499     freeze => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 503     },
 504     vga => {
 505         optional => 1,
 506         type => 'string', format => $vga_fmt,
 507         description => "Configure the VGA hardware.",
 508         verbose_description => "Configure the VGA Hardware. If you want to use ".
 509             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 510             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 511             "is 'std' for all OS types besides some Windows versions (XP and " .
 512             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 513             "display server. For win* OS you can select how many independent " .
 514             "displays you want, Linux guests can add displays them self.\n".
 515             "You can also run without any graphic card, using a serial device as terminal.",
 516     },
 517     watchdog => {
 518         optional => 1,
 519         type => 'string', format => 'pve-qm-watchdog',
 520         description => "Create a virtual hardware watchdog device.",
 521         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 522             " (by a guest action), the watchdog must be periodically polled " .
 523             "by an agent inside the guest or else the watchdog will reset " .
 524             "the guest (or execute the respective action specified)",
 525     },
 526     startdate => {
 527         optional => 1,
 528         type => 'string',
 529         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 530         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 531         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 532         default => 'now',
 533     },
 534     startup =>  get_standard_option('pve-startup-order'),
 535     template => {
 536         optional => 1,
 537         type => 'boolean',
 538         description => "Enable/disable Template.",
 539         default => 0,
 540     },
 541     args => {
 542         optional => 1,
 543         type => 'string',
 544         description => "Arbitrary arguments passed to kvm.",
 545         verbose_description => <<EODESCR,
 546 Arbitrary arguments passed to kvm, for example:
 547
 548 args: -no-reboot -no-hpet
 549
 550 NOTE: this option is for experts only.
 551 EODESCR
 552     },
 553     tablet => {
 554         optional => 1,
 555         type => 'boolean',
 556         default => 1,
 557         description => "Enable/disable the USB tablet device.",
 558         verbose_description => "Enable/disable the USB tablet device. This device is " .
 559             "usually needed to allow absolute mouse positioning with VNC. " .
 560             "Else the mouse runs out of sync with normal VNC clients. " .
 561             "If you're running lots of console-only guests on one host, " .
 562             "you may consider disabling this to save some context switches. " .
 563             "This is turned off by default if you use spice (-vga=qxl).",
 564     },
 565     migrate_speed => {
 566         optional => 1,
 567         type => 'integer',
 568         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 569         minimum => 0,
 570         default => 0,
 571     },
 572     migrate_downtime => {
 573         optional => 1,
 574         type => 'number',
 575         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 576         minimum => 0,
 577         default => 0.1,
 578     },
 579     cdrom => {
 580         optional => 1,
 581         type => 'string', format => 'pve-qm-ide',
 582         typetext => '<volume>',
 583         description => "This is an alias for option -ide2",
 584     },
 585     cpu => {
 586         optional => 1,
 587         description => "Emulated CPU type.",
 588         type => 'string',
 589         format => $cpu_fmt,
 590     },
 591     parent => get_standard_option('pve-snapshot-name', {
 592         optional => 1,
 593         description => "Parent snapshot name. This is used internally, and should not be modified.",
 594     }),
 595     snaptime => {
 596         optional => 1,
 597         description => "Timestamp for snapshots.",
 598         type => 'integer',
 599         minimum => 0,
 600     },
 601     vmstate => {
 602         optional => 1,
 603         type => 'string', format => 'pve-volume-id',
 604         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 605     },
 606     vmstatestorage => get_standard_option('pve-storage-id', {
 607         description => "Default storage for VM state volumes/files.",
 608         optional => 1,
 609     }),
 610     runningmachine => get_standard_option('pve-qemu-machine', {
 611         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 612     }),
 613     machine => get_standard_option('pve-qemu-machine'),
 614     arch => {
 615         description => "Virtual processor architecture. Defaults to the host.",
 616         optional => 1,
 617         type => 'string',
 618         enum => [qw(x86_64 aarch64)],
 619     },
 620     smbios1 => {
 621         description => "Specify SMBIOS type 1 fields.",
 622         type => 'string', format => 'pve-qm-smbios1',
 623         maxLength => 512,
 624         optional => 1,
 625     },
 626     protection => {
 627         optional => 1,
 628         type => 'boolean',
 629         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 630         default => 0,
 631     },
 632     bios => {
 633         optional => 1,
 634         type => 'string',
 635         enum => [ qw(seabios ovmf) ],
 636         description => "Select BIOS implementation.",
 637         default => 'seabios',
 638     },
 639     vmgenid => {
 640         type => 'string',
 641         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 642         format_description => 'UUID',
 643         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 644         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 645             " 128-bit integer value identifier to the guest OS. This allows to".
 646             " notify the guest operating system when the virtual machine is".
 647             " executed with a different configuration (e.g. snapshot execution".
 648             " or creation from a template). The guest operating system notices".
 649             " the change, and is then able to react as appropriate by marking".
 650             " its copies of distributed databases as dirty, re-initializing its".
 651             " random number generator, etc.\n".
 652             "Note that auto-creation only works when done throug API/CLI create".
 653             " or update methods, but not when manually editing the config file.",
 654         default => "1 (autogenerated)",
 655         optional => 1,
 656     },
 657     hookscript => {
 658         type => 'string',
 659         format => 'pve-volume-id',
 660         optional => 1,
 661         description => "Script that will be executed during various steps in the vms lifetime.",
 662     },
 663     ivshmem => {
 664         type => 'string',
 665         format => $ivshmem_fmt,
 666         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 667         optional => 1,
 668     },
 669     audio0 => {
 670         type => 'string',
 671         format => $audio_fmt,
 672         description => "Configure a audio device, useful in combination with QXL/Spice.",
 673         optional => 1
 674     },
 675 };
 676
 677 my $cicustom_fmt = {
 678     meta => {
 679         type => 'string',
 680         optional => 1,
 681         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 682         format => 'pve-volume-id',
 683         format_description => 'volume',
 684     },
 685     network => {
 686         type => 'string',
 687         optional => 1,
 688         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 689         format => 'pve-volume-id',
 690         format_description => 'volume',
 691     },
 692     user => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 696         format => 'pve-volume-id',
 697         format_description => 'volume',
 698     },
 699 };
 700 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 701
 702 my $confdesc_cloudinit = {
 703     citype => {
 704         optional => 1,
 705         type => 'string',
 706         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 707         enum => ['configdrive2', 'nocloud'],
 708     },
 709     ciuser => {
 710         optional => 1,
 711         type => 'string',
 712         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 713     },
 714     cipassword => {
 715         optional => 1,
 716         type => 'string',
 717         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 718     },
 719     cicustom => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 723         format => 'pve-qm-cicustom',
 724     },
 725     searchdomain => {
 726         optional => 1,
 727         type => 'string',
 728         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 729     },
 730     nameserver => {
 731         optional => 1,
 732         type => 'string', format => 'address-list',
 733         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     sshkeys => {
 736         optional => 1,
 737         type => 'string',
 738         format => 'urlencoded',
 739         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 740     },
 741 };
 742
 743 # what about other qemu settings ?
 744 #cpu => 'string',
 745 #machine => 'string',
 746 #fda => 'file',
 747 #fdb => 'file',
 748 #mtdblock => 'file',
 749 #sd => 'file',
 750 #pflash => 'file',
 751 #snapshot => 'bool',
 752 #bootp => 'file',
 753 ##tftp => 'dir',
 754 ##smb => 'dir',
 755 #kernel => 'file',
 756 #append => 'string',
 757 #initrd => 'file',
 758 ##soundhw => 'string',
 759
 760 while (my ($k, $v) = each %$confdesc) {
 761     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 762 }
 763
 764 my $MAX_IDE_DISKS = 4;
 765 my $MAX_SCSI_DISKS = 14;
 766 my $MAX_VIRTIO_DISKS = 16;
 767 my $MAX_SATA_DISKS = 6;
 768 my $MAX_USB_DEVICES = 5;
 769 my $MAX_NETS = 32;
 770 my $MAX_UNUSED_DISKS = 256;
 771 my $MAX_HOSTPCI_DEVICES = 4;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         optional => 1,
 850     },
 851     queues => {
 852         type => 'integer',
 853         minimum => 0, maximum => 16,
 854         description => 'Number of packet queues to be used on the device.',
 855         optional => 1,
 856     },
 857     rate => {
 858         type => 'number',
 859         minimum => 0,
 860         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 861         optional => 1,
 862     },
 863     tag => {
 864         type => 'integer',
 865         minimum => 1, maximum => 4094,
 866         description => 'VLAN tag to apply to packets on this interface.',
 867         optional => 1,
 868     },
 869     trunks => {
 870         type => 'string',
 871         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 872         description => 'VLAN trunks to pass through this interface.',
 873         format_description => 'vlanid[;vlanid...]',
 874         optional => 1,
 875     },
 876     firewall => {
 877         type => 'boolean',
 878         description => 'Whether this interface should be protected by the firewall.',
 879         optional => 1,
 880     },
 881     link_down => {
 882         type => 'boolean',
 883         description => 'Whether this interface should be disconnected (like pulling the plug).',
 884         optional => 1,
 885     },
 886 };
 887
 888 my $netdesc = {
 889     optional => 1,
 890     type => 'string', format => $net_fmt,
 891     description => "Specify network devices.",
 892 };
 893
 894 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 895
 896 my $ipconfig_fmt = {
 897     ip => {
 898         type => 'string',
 899         format => 'pve-ipv4-config',
 900         format_description => 'IPv4Format/CIDR',
 901         description => 'IPv4 address in CIDR format.',
 902         optional => 1,
 903         default => 'dhcp',
 904     },
 905     gw => {
 906         type => 'string',
 907         format => 'ipv4',
 908         format_description => 'GatewayIPv4',
 909         description => 'Default gateway for IPv4 traffic.',
 910         optional => 1,
 911         requires => 'ip',
 912     },
 913     ip6 => {
 914         type => 'string',
 915         format => 'pve-ipv6-config',
 916         format_description => 'IPv6Format/CIDR',
 917         description => 'IPv6 address in CIDR format.',
 918         optional => 1,
 919         default => 'dhcp',
 920     },
 921     gw6 => {
 922         type => 'string',
 923         format => 'ipv6',
 924         format_description => 'GatewayIPv6',
 925         description => 'Default gateway for IPv6 traffic.',
 926         optional => 1,
 927         requires => 'ip6',
 928     },
 929 };
 930 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 931 my $ipconfigdesc = {
 932     optional => 1,
 933     type => 'string', format => 'pve-qm-ipconfig',
 934     description => <<'EODESCR',
 935 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 936
 937 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 938
 939 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 940 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 941
 942 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 943 EODESCR
 944 };
 945 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 946
 947 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 948     $confdesc->{"net$i"} = $netdesc;
 949     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 950 }
 951
 952 foreach my $key (keys %$confdesc_cloudinit) {
 953     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 954 }
 955
 956 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 957 sub verify_volume_id_or_qm_path {
 958     my ($volid, $noerr) = @_;
 959
 960     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 961         return $volid;
 962     }
 963
 964     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 965     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 966     if ($@) {
 967         return undef if $noerr;
 968         die $@;
 969     }
 970     return $volid;
 971 }
 972
 973 my $drivename_hash;
 974
 975 my %drivedesc_base = (
 976     volume => { alias => 'file' },
 977     file => {
 978         type => 'string',
 979         format => 'pve-volume-id-or-qm-path',
 980         default_key => 1,
 981         format_description => 'volume',
 982         description => "The drive's backing volume.",
 983     },
 984     media => {
 985         type => 'string',
 986         enum => [qw(cdrom disk)],
 987         description => "The drive's media type.",
 988         default => 'disk',
 989         optional => 1
 990     },
 991     cyls => {
 992         type => 'integer',
 993         description => "Force the drive's physical geometry to have a specific cylinder count.",
 994         optional => 1
 995     },
 996     heads => {
 997         type => 'integer',
 998         description => "Force the drive's physical geometry to have a specific head count.",
 999         optional => 1
1000     },
1001     secs => {
1002         type => 'integer',
1003         description => "Force the drive's physical geometry to have a specific sector count.",
1004         optional => 1
1005     },
1006     trans => {
1007         type => 'string',
1008         enum => [qw(none lba auto)],
1009         description => "Force disk geometry bios translation mode.",
1010         optional => 1,
1011     },
1012     snapshot => {
1013         type => 'boolean',
1014         description => "Controls qemu's snapshot mode feature."
1015             . " If activated, changes made to the disk are temporary and will"
1016             . " be discarded when the VM is shutdown.",
1017         optional => 1,
1018     },
1019     cache => {
1020         type => 'string',
1021         enum => [qw(none writethrough writeback unsafe directsync)],
1022         description => "The drive's cache mode",
1023         optional => 1,
1024     },
1025     format => get_standard_option('pve-qm-image-format'),
1026     size => {
1027         type => 'string',
1028         format => 'disk-size',
1029         format_description => 'DiskSize',
1030         description => "Disk size. This is purely informational and has no effect.",
1031         optional => 1,
1032     },
1033     backup => {
1034         type => 'boolean',
1035         description => "Whether the drive should be included when making backups.",
1036         optional => 1,
1037     },
1038     replicate => {
1039         type => 'boolean',
1040         description => 'Whether the drive should considered for replication jobs.',
1041         optional => 1,
1042         default => 1,
1043     },
1044     rerror => {
1045         type => 'string',
1046         enum => [qw(ignore report stop)],
1047         description => 'Read error action.',
1048         optional => 1,
1049     },
1050     werror => {
1051         type => 'string',
1052         enum => [qw(enospc ignore report stop)],
1053         description => 'Write error action.',
1054         optional => 1,
1055     },
1056     aio => {
1057         type => 'string',
1058         enum => [qw(native threads)],
1059         description => 'AIO type to use.',
1060         optional => 1,
1061     },
1062     discard => {
1063         type => 'string',
1064         enum => [qw(ignore on)],
1065         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1066         optional => 1,
1067     },
1068     detect_zeroes => {
1069         type => 'boolean',
1070         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1071         optional => 1,
1072     },
1073     serial => {
1074         type => 'string',
1075         format => 'urlencoded',
1076         format_description => 'serial',
1077         maxLength => 20*3, # *3 since it's %xx url enoded
1078         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1079         optional => 1,
1080     },
1081     shared => {
1082         type => 'boolean',
1083         description => 'Mark this locally-managed volume as available on all nodes',
1084         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1085         optional => 1,
1086         default => 0,
1087     }
1088 );
1089
1090 my %iothread_fmt = ( iothread => {
1091         type => 'boolean',
1092         description => "Whether to use iothreads for this drive",
1093         optional => 1,
1094 });
1095
1096 my %model_fmt = (
1097     model => {
1098         type => 'string',
1099         format => 'urlencoded',
1100         format_description => 'model',
1101         maxLength => 40*3, # *3 since it's %xx url enoded
1102         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1103         optional => 1,
1104     },
1105 );
1106
1107 my %queues_fmt = (
1108     queues => {
1109         type => 'integer',
1110         description => "Number of queues.",
1111         minimum => 2,
1112         optional => 1
1113     }
1114 );
1115
1116 my %scsiblock_fmt = (
1117     scsiblock => {
1118         type => 'boolean',
1119         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1120         optional => 1,
1121         default => 0,
1122     },
1123 );
1124
1125 my %ssd_fmt = (
1126     ssd => {
1127         type => 'boolean',
1128         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %wwn_fmt = (
1134     wwn => {
1135         type => 'string',
1136         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1137         format_description => 'wwn',
1138         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1139         optional => 1,
1140     },
1141 );
1142
1143 my $add_throttle_desc = sub {
1144     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1145     my $d = {
1146         type => $type,
1147         format_description => $unit,
1148         description => "Maximum $what in $longunit.",
1149         optional => 1,
1150     };
1151     $d->{minimum} = $minimum if defined($minimum);
1152     $drivedesc_base{$key} = $d;
1153 };
1154 # throughput: (leaky bucket)
1155 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1156 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1157 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1158 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1159 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1160 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1161 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1162 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1163 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1164
1165 # pools: (pool of IO before throttling starts taking effect)
1166 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1167 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1168 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1169 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1170 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1171 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1172
1173 # burst lengths
1174 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1175 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1176 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1177 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1178 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1179 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1180
1181 # legacy support
1182 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1183 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1184 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1185 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1186
1187 my $ide_fmt = {
1188     %drivedesc_base,
1189     %model_fmt,
1190     %ssd_fmt,
1191     %wwn_fmt,
1192 };
1193 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1194
1195 my $idedesc = {
1196     optional => 1,
1197     type => 'string', format => $ide_fmt,
1198     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1199 };
1200 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1201
1202 my $scsi_fmt = {
1203     %drivedesc_base,
1204     %iothread_fmt,
1205     %queues_fmt,
1206     %scsiblock_fmt,
1207     %ssd_fmt,
1208     %wwn_fmt,
1209 };
1210 my $scsidesc = {
1211     optional => 1,
1212     type => 'string', format => $scsi_fmt,
1213     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1214 };
1215 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1216
1217 my $sata_fmt = {
1218     %drivedesc_base,
1219     %ssd_fmt,
1220     %wwn_fmt,
1221 };
1222 my $satadesc = {
1223     optional => 1,
1224     type => 'string', format => $sata_fmt,
1225     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1226 };
1227 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1228
1229 my $virtio_fmt = {
1230     %drivedesc_base,
1231     %iothread_fmt,
1232 };
1233 my $virtiodesc = {
1234     optional => 1,
1235     type => 'string', format => $virtio_fmt,
1236     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1237 };
1238 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1239
1240 my $alldrive_fmt = {
1241     %drivedesc_base,
1242     %iothread_fmt,
1243     %model_fmt,
1244     %queues_fmt,
1245     %scsiblock_fmt,
1246     %ssd_fmt,
1247     %wwn_fmt,
1248 };
1249
1250 my $efidisk_fmt = {
1251     volume => { alias => 'file' },
1252     file => {
1253         type => 'string',
1254         format => 'pve-volume-id-or-qm-path',
1255         default_key => 1,
1256         format_description => 'volume',
1257         description => "The drive's backing volume.",
1258     },
1259     format => get_standard_option('pve-qm-image-format'),
1260     size => {
1261         type => 'string',
1262         format => 'disk-size',
1263         format_description => 'DiskSize',
1264         description => "Disk size. This is purely informational and has no effect.",
1265         optional => 1,
1266     },
1267 };
1268
1269 my $efidisk_desc = {
1270     optional => 1,
1271     type => 'string', format => $efidisk_fmt,
1272     description => "Configure a Disk for storing EFI vars",
1273 };
1274
1275 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1276
1277 my $usb_fmt = {
1278     host => {
1279         default_key => 1,
1280         type => 'string', format => 'pve-qm-usb-device',
1281         format_description => 'HOSTUSBDEVICE|spice',
1282         description => <<EODESCR,
1283 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1284
1285  'bus-port(.port)*' (decimal numbers) or
1286  'vendor_id:product_id' (hexadeciaml numbers) or
1287  'spice'
1288
1289 You can use the 'lsusb -t' command to list existing usb devices.
1290
1291 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1292
1293 The value 'spice' can be used to add a usb redirection devices for spice.
1294 EODESCR
1295     },
1296     usb3 => {
1297         optional => 1,
1298         type => 'boolean',
1299         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1300         default => 0,
1301     },
1302 };
1303
1304 my $usbdesc = {
1305     optional => 1,
1306     type => 'string', format => $usb_fmt,
1307     description => "Configure an USB device (n is 0 to 4).",
1308 };
1309 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1310
1311 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1312 my $hostpci_fmt = {
1313     host => {
1314         default_key => 1,
1315         type => 'string',
1316         pattern => qr/$PCIRE(;$PCIRE)*/,
1317         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1318         description => <<EODESCR,
1319 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1320 of PCI virtual functions of the host. HOSTPCIID syntax is:
1321
1322 'bus:dev.func' (hexadecimal numbers)
1323
1324 You can us the 'lspci' command to list existing PCI devices.
1325 EODESCR
1326     },
1327     rombar => {
1328         type => 'boolean',
1329         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1330         optional => 1,
1331         default => 1,
1332     },
1333     romfile => {
1334         type => 'string',
1335         pattern => '[^,;]+',
1336         format_description => 'string',
1337         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1338         optional => 1,
1339     },
1340     pcie => {
1341         type => 'boolean',
1342         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1343         optional => 1,
1344         default => 0,
1345     },
1346     'x-vga' => {
1347         type => 'boolean',
1348         description =>  "Enable vfio-vga device support.",
1349         optional => 1,
1350         default => 0,
1351     },
1352     'mdev' => {
1353         type => 'string',
1354         format_description => 'string',
1355         pattern => '[^/\.:]+',
1356         optional => 1,
1357         description => <<EODESCR
1358 The type of mediated device to use.
1359 An instance of this type will be created on startup of the VM and
1360 will be cleaned up when the VM stops.
1361 EODESCR
1362     }
1363 };
1364 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1365
1366 my $hostpcidesc = {
1367         optional => 1,
1368         type => 'string', format => 'pve-qm-hostpci',
1369         description => "Map host PCI devices into guest.",
1370         verbose_description =>  <<EODESCR,
1371 Map host PCI devices into guest.
1372
1373 NOTE: This option allows direct access to host hardware. So it is no longer
1374 possible to migrate such machines - use with special care.
1375
1376 CAUTION: Experimental! User reported problems with this option.
1377 EODESCR
1378 };
1379 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1380
1381 my $serialdesc = {
1382         optional => 1,
1383         type => 'string',
1384         pattern => '(/dev/.+|socket)',
1385         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1386         verbose_description =>  <<EODESCR,
1387 Create a serial device inside the VM (n is 0 to 3), and pass through a
1388 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1389 host side (use 'qm terminal' to open a terminal connection).
1390
1391 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1392
1393 CAUTION: Experimental! User reported problems with this option.
1394 EODESCR
1395 };
1396
1397 my $paralleldesc= {
1398         optional => 1,
1399         type => 'string',
1400         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1401         description =>  "Map host parallel devices (n is 0 to 2).",
1402         verbose_description =>  <<EODESCR,
1403 Map host parallel devices (n is 0 to 2).
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1406
1407 CAUTION: Experimental! User reported problems with this option.
1408 EODESCR
1409 };
1410
1411 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1412     $confdesc->{"parallel$i"} = $paralleldesc;
1413 }
1414
1415 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1416     $confdesc->{"serial$i"} = $serialdesc;
1417 }
1418
1419 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1420     $confdesc->{"hostpci$i"} = $hostpcidesc;
1421 }
1422
1423 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1424     $drivename_hash->{"ide$i"} = 1;
1425     $confdesc->{"ide$i"} = $idedesc;
1426 }
1427
1428 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1429     $drivename_hash->{"sata$i"} = 1;
1430     $confdesc->{"sata$i"} = $satadesc;
1431 }
1432
1433 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1434     $drivename_hash->{"scsi$i"} = 1;
1435     $confdesc->{"scsi$i"} = $scsidesc ;
1436 }
1437
1438 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1439     $drivename_hash->{"virtio$i"} = 1;
1440     $confdesc->{"virtio$i"} = $virtiodesc;
1441 }
1442
1443 $drivename_hash->{efidisk0} = 1;
1444 $confdesc->{efidisk0} = $efidisk_desc;
1445
1446 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1447     $confdesc->{"usb$i"} = $usbdesc;
1448 }
1449
1450 my $unuseddesc = {
1451     optional => 1,
1452     type => 'string', format => 'pve-volume-id',
1453     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1454 };
1455
1456 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1457     $confdesc->{"unused$i"} = $unuseddesc;
1458 }
1459
1460 my $kvm_api_version = 0;
1461
1462 sub kvm_version {
1463     return $kvm_api_version if $kvm_api_version;
1464
1465     open my $fh, '<', '/dev/kvm'
1466         or return undef;
1467
1468     # 0xae00 => KVM_GET_API_VERSION
1469     $kvm_api_version = ioctl($fh, 0xae00, 0);
1470
1471     return $kvm_api_version;
1472 }
1473
1474 my $kvm_user_version;
1475
1476 sub kvm_user_version {
1477
1478     return $kvm_user_version if $kvm_user_version;
1479
1480     $kvm_user_version = 'unknown';
1481
1482     my $code = sub {
1483         my $line = shift;
1484         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1485             $kvm_user_version = $2;
1486         }
1487     };
1488
1489     eval { run_command("kvm -version", outfunc => $code); };
1490     warn $@ if $@;
1491
1492     return $kvm_user_version;
1493
1494 }
1495
1496 sub kernel_has_vhost_net {
1497     return -c '/dev/vhost-net';
1498 }
1499
1500 sub valid_drive_names {
1501     # order is important - used to autoselect boot disk
1502     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1503             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1504             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1505             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1506             'efidisk0');
1507 }
1508
1509 sub is_valid_drivename {
1510     my $dev = shift;
1511
1512     return defined($drivename_hash->{$dev});
1513 }
1514
1515 sub option_exists {
1516     my $key = shift;
1517     return defined($confdesc->{$key});
1518 }
1519
1520 sub nic_models {
1521     return $nic_model_list;
1522 }
1523
1524 sub os_list_description {
1525
1526     return {
1527         other => 'Other',
1528         wxp => 'Windows XP',
1529         w2k => 'Windows 2000',
1530         w2k3 =>, 'Windows 2003',
1531         w2k8 => 'Windows 2008',
1532         wvista => 'Windows Vista',
1533         win7 => 'Windows 7',
1534         win8 => 'Windows 8/2012',
1535         win10 => 'Windows 10/2016',
1536         l24 => 'Linux 2.4',
1537         l26 => 'Linux 2.6',
1538     };
1539 }
1540
1541 my $cdrom_path;
1542
1543 sub get_cdrom_path {
1544
1545     return  $cdrom_path if $cdrom_path;
1546
1547     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1548     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1549     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1550 }
1551
1552 sub get_iso_path {
1553     my ($storecfg, $vmid, $cdrom) = @_;
1554
1555     if ($cdrom eq 'cdrom') {
1556         return get_cdrom_path();
1557     } elsif ($cdrom eq 'none') {
1558         return '';
1559     } elsif ($cdrom =~ m|^/|) {
1560         return $cdrom;
1561     } else {
1562         return PVE::Storage::path($storecfg, $cdrom);
1563     }
1564 }
1565
1566 # try to convert old style file names to volume IDs
1567 sub filename_to_volume_id {
1568     my ($vmid, $file, $media) = @_;
1569
1570      if (!($file eq 'none' || $file eq 'cdrom' ||
1571           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1572
1573         return undef if $file =~ m|/|;
1574
1575         if ($media && $media eq 'cdrom') {
1576             $file = "local:iso/$file";
1577         } else {
1578             $file = "local:$vmid/$file";
1579         }
1580     }
1581
1582     return $file;
1583 }
1584
1585 sub verify_media_type {
1586     my ($opt, $vtype, $media) = @_;
1587
1588     return if !$media;
1589
1590     my $etype;
1591     if ($media eq 'disk') {
1592         $etype = 'images';
1593     } elsif ($media eq 'cdrom') {
1594         $etype = 'iso';
1595     } else {
1596         die "internal error";
1597     }
1598
1599     return if ($vtype eq $etype);
1600
1601     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1602 }
1603
1604 sub cleanup_drive_path {
1605     my ($opt, $storecfg, $drive) = @_;
1606
1607     # try to convert filesystem paths to volume IDs
1608
1609     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1610         ($drive->{file} !~ m|^/dev/.+|) &&
1611         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1612         ($drive->{file} !~ m/^\d+$/)) {
1613         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1614         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1615         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1616         verify_media_type($opt, $vtype, $drive->{media});
1617         $drive->{file} = $volid;
1618     }
1619
1620     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1621 }
1622
1623 sub parse_hotplug_features {
1624     my ($data) = @_;
1625
1626     my $res = {};
1627
1628     return $res if $data eq '0';
1629
1630     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1631
1632     foreach my $feature (PVE::Tools::split_list($data)) {
1633         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1634             $res->{$1} = 1;
1635         } else {
1636             die "invalid hotplug feature '$feature'\n";
1637         }
1638     }
1639     return $res;
1640 }
1641
1642 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1643 sub pve_verify_hotplug_features {
1644     my ($value, $noerr) = @_;
1645
1646     return $value if parse_hotplug_features($value);
1647
1648     return undef if $noerr;
1649
1650     die "unable to parse hotplug option\n";
1651 }
1652
1653 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1654 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1655 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1656 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1657 #        [,iothread=on][,serial=serial][,model=model]
1658
1659 sub parse_drive {
1660     my ($key, $data) = @_;
1661
1662     my ($interface, $index);
1663
1664     if ($key =~ m/^([^\d]+)(\d+)$/) {
1665         $interface = $1;
1666         $index = $2;
1667     } else {
1668         return undef;
1669     }
1670
1671     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1672                                      : $confdesc->{$key}->{format};
1673     if (!$desc) {
1674         warn "invalid drive key: $key\n";
1675         return undef;
1676     }
1677     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1678     return undef if !$res;
1679     $res->{interface} = $interface;
1680     $res->{index} = $index;
1681
1682     my $error = 0;
1683     foreach my $opt (qw(bps bps_rd bps_wr)) {
1684         if (my $bps = defined(delete $res->{$opt})) {
1685             if (defined($res->{"m$opt"})) {
1686                 warn "both $opt and m$opt specified\n";
1687                 ++$error;
1688                 next;
1689             }
1690             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1691         }
1692     }
1693
1694     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1695     for my $requirement (
1696         [mbps_max => 'mbps'],
1697         [mbps_rd_max => 'mbps_rd'],
1698         [mbps_wr_max => 'mbps_wr'],
1699         [miops_max => 'miops'],
1700         [miops_rd_max => 'miops_rd'],
1701         [miops_wr_max => 'miops_wr'],
1702         [bps_max_length => 'mbps_max'],
1703         [bps_rd_max_length => 'mbps_rd_max'],
1704         [bps_wr_max_length => 'mbps_wr_max'],
1705         [iops_max_length => 'iops_max'],
1706         [iops_rd_max_length => 'iops_rd_max'],
1707         [iops_wr_max_length => 'iops_wr_max']) {
1708         my ($option, $requires) = @$requirement;
1709         if ($res->{$option} && !$res->{$requires}) {
1710             warn "$option requires $requires\n";
1711             ++$error;
1712         }
1713     }
1714
1715     return undef if $error;
1716
1717     return undef if $res->{mbps_rd} && $res->{mbps};
1718     return undef if $res->{mbps_wr} && $res->{mbps};
1719     return undef if $res->{iops_rd} && $res->{iops};
1720     return undef if $res->{iops_wr} && $res->{iops};
1721
1722     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1723         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1724         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1725         return undef if $res->{interface} eq 'virtio';
1726     }
1727
1728     if (my $size = $res->{size}) {
1729         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1730     }
1731
1732     return $res;
1733 }
1734
1735 sub print_drive {
1736     my ($vmid, $drive) = @_;
1737     my $data = { %$drive };
1738     delete $data->{$_} for qw(index interface);
1739     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1740 }
1741
1742 sub scsi_inquiry {
1743     my($fh, $noerr) = @_;
1744
1745     my $SG_IO = 0x2285;
1746     my $SG_GET_VERSION_NUM = 0x2282;
1747
1748     my $versionbuf = "\x00" x 8;
1749     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1750     if (!$ret) {
1751         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1752         return undef;
1753     }
1754     my $version = unpack("I", $versionbuf);
1755     if ($version < 30000) {
1756         die "scsi generic interface too old\n"  if !$noerr;
1757         return undef;
1758     }
1759
1760     my $buf = "\x00" x 36;
1761     my $sensebuf = "\x00" x 8;
1762     my $cmd = pack("C x3 C x1", 0x12, 36);
1763
1764     # see /usr/include/scsi/sg.h
1765     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1766
1767     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1768                       length($sensebuf), 0, length($buf), $buf,
1769                       $cmd, $sensebuf, 6000);
1770
1771     $ret = ioctl($fh, $SG_IO, $packet);
1772     if (!$ret) {
1773         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1774         return undef;
1775     }
1776
1777     my @res = unpack($sg_io_hdr_t, $packet);
1778     if ($res[17] || $res[18]) {
1779         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1780         return undef;
1781     }
1782
1783     my $res = {};
1784     (my $byte0, my $byte1, $res->{vendor},
1785      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1786
1787     $res->{removable} = $byte1 & 128 ? 1 : 0;
1788     $res->{type} = $byte0 & 31;
1789
1790     return $res;
1791 }
1792
1793 sub path_is_scsi {
1794     my ($path) = @_;
1795
1796     my $fh = IO::File->new("+<$path") || return undef;
1797     my $res = scsi_inquiry($fh, 1);
1798     close($fh);
1799
1800     return $res;
1801 }
1802
1803 sub machine_type_is_q35 {
1804     my ($conf) = @_;
1805
1806     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1807 }
1808
1809 sub print_tabletdevice_full {
1810     my ($conf, $arch) = @_;
1811
1812     my $q35 = machine_type_is_q35($conf);
1813
1814     # we use uhci for old VMs because tablet driver was buggy in older qemu
1815     my $usbbus;
1816     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1817         $usbbus = 'ehci';
1818     } else {
1819         $usbbus = 'uhci';
1820     }
1821
1822     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1823 }
1824
1825 sub print_keyboarddevice_full {
1826     my ($conf, $arch, $machine) = @_;
1827
1828     return undef if $arch ne 'aarch64';
1829
1830     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1831 }
1832
1833 sub print_drivedevice_full {
1834     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1835
1836     my $device = '';
1837     my $maxdev = 0;
1838
1839     if ($drive->{interface} eq 'virtio') {
1840         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1841         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1842         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1843     } elsif ($drive->{interface} eq 'scsi') {
1844
1845         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1846         my $unit = $drive->{index} % $maxdev;
1847         my $devicetype = 'hd';
1848         my $path = '';
1849         if (drive_is_cdrom($drive)) {
1850             $devicetype = 'cd';
1851         } else {
1852             if ($drive->{file} =~ m|^/|) {
1853                 $path = $drive->{file};
1854                 if (my $info = path_is_scsi($path)) {
1855                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1856                         $devicetype = 'block';
1857                     } elsif ($info->{type} == 1) { # tape
1858                         $devicetype = 'generic';
1859                     }
1860                 }
1861             } else {
1862                  $path = PVE::Storage::path($storecfg, $drive->{file});
1863             }
1864
1865             if($path =~ m/^iscsi\:\/\//){
1866                 $devicetype = 'generic';
1867             }
1868         }
1869
1870         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1871             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1872         } else {
1873             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1874         }
1875
1876         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1877             $device .= ",rotation_rate=1";
1878         }
1879         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1880
1881     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1882         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1883         my $controller = int($drive->{index} / $maxdev);
1884         my $unit = $drive->{index} % $maxdev;
1885         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1886
1887         $device = "ide-$devicetype";
1888         if ($drive->{interface} eq 'ide') {
1889             $device .= ",bus=ide.$controller,unit=$unit";
1890         } else {
1891             $device .= ",bus=ahci$controller.$unit";
1892         }
1893         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1894
1895         if ($devicetype eq 'hd') {
1896             if (my $model = $drive->{model}) {
1897                 $model = URI::Escape::uri_unescape($model);
1898                 $device .= ",model=$model";
1899             }
1900             if ($drive->{ssd}) {
1901                 $device .= ",rotation_rate=1";
1902             }
1903         }
1904         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1905     } elsif ($drive->{interface} eq 'usb') {
1906         die "implement me";
1907         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1908     } else {
1909         die "unsupported interface type";
1910     }
1911
1912     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1913
1914     if (my $serial = $drive->{serial}) {
1915         $serial = URI::Escape::uri_unescape($serial);
1916         $device .= ",serial=$serial";
1917     }
1918
1919
1920     return $device;
1921 }
1922
1923 sub get_initiator_name {
1924     my $initiator;
1925
1926     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1927     while (defined(my $line = <$fh>)) {
1928         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1929         $initiator = $1;
1930         last;
1931     }
1932     $fh->close();
1933
1934     return $initiator;
1935 }
1936
1937 sub print_drive_full {
1938     my ($storecfg, $vmid, $drive) = @_;
1939
1940     my $path;
1941     my $volid = $drive->{file};
1942     my $format;
1943
1944     if (drive_is_cdrom($drive)) {
1945         $path = get_iso_path($storecfg, $vmid, $volid);
1946     } else {
1947         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1948         if ($storeid) {
1949             $path = PVE::Storage::path($storecfg, $volid);
1950             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1951             $format = qemu_img_format($scfg, $volname);
1952         } else {
1953             $path = $volid;
1954             $format = "raw";
1955         }
1956    }
1957
1958     my $opts = '';
1959     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1960     foreach my $o (@qemu_drive_options) {
1961         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1962     }
1963
1964     # snapshot only accepts on|off
1965     if (defined($drive->{snapshot})) {
1966         my $v = $drive->{snapshot} ? 'on' : 'off';
1967         $opts .= ",snapshot=$v";
1968     }
1969
1970     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1971         my ($dir, $qmpname) = @$type;
1972         if (my $v = $drive->{"mbps$dir"}) {
1973             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1974         }
1975         if (my $v = $drive->{"mbps${dir}_max"}) {
1976             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1977         }
1978         if (my $v = $drive->{"bps${dir}_max_length"}) {
1979             $opts .= ",throttling.bps$qmpname-max-length=$v";
1980         }
1981         if (my $v = $drive->{"iops${dir}"}) {
1982             $opts .= ",throttling.iops$qmpname=$v";
1983         }
1984         if (my $v = $drive->{"iops${dir}_max"}) {
1985             $opts .= ",throttling.iops$qmpname-max=$v";
1986         }
1987         if (my $v = $drive->{"iops${dir}_max_length"}) {
1988             $opts .= ",throttling.iops$qmpname-max-length=$v";
1989         }
1990     }
1991
1992     $opts .= ",format=$format" if $format && !$drive->{format};
1993
1994     my $cache_direct = 0;
1995
1996     if (my $cache = $drive->{cache}) {
1997         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1998     } elsif (!drive_is_cdrom($drive)) {
1999         $opts .= ",cache=none";
2000         $cache_direct = 1;
2001     }
2002
2003     # aio native works only with O_DIRECT
2004     if (!$drive->{aio}) {
2005         if($cache_direct) {
2006             $opts .= ",aio=native";
2007         } else {
2008             $opts .= ",aio=threads";
2009         }
2010     }
2011
2012     if (!drive_is_cdrom($drive)) {
2013         my $detectzeroes;
2014         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2015             $detectzeroes = 'off';
2016         } elsif ($drive->{discard}) {
2017             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2018         } else {
2019             # This used to be our default with discard not being specified:
2020             $detectzeroes = 'on';
2021         }
2022         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2023     }
2024
2025     my $pathinfo = $path ? "file=$path," : '';
2026
2027     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2028 }
2029
2030 sub print_netdevice_full {
2031     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2032
2033     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2034
2035     my $device = $net->{model};
2036     if ($net->{model} eq 'virtio') {
2037          $device = 'virtio-net-pci';
2038      };
2039
2040     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2041     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2042     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2043         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2044         my $vectors = $net->{queues} * 2 + 2;
2045         $tmpstr .= ",vectors=$vectors,mq=on";
2046     }
2047     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2048
2049     if ($use_old_bios_files) {
2050         my $romfile;
2051         if ($device eq 'virtio-net-pci') {
2052             $romfile = 'pxe-virtio.rom';
2053         } elsif ($device eq 'e1000') {
2054             $romfile = 'pxe-e1000.rom';
2055         } elsif ($device eq 'ne2k') {
2056             $romfile = 'pxe-ne2k_pci.rom';
2057         } elsif ($device eq 'pcnet') {
2058             $romfile = 'pxe-pcnet.rom';
2059         } elsif ($device eq 'rtl8139') {
2060             $romfile = 'pxe-rtl8139.rom';
2061         }
2062         $tmpstr .= ",romfile=$romfile" if $romfile;
2063     }
2064
2065     return $tmpstr;
2066 }
2067
2068 sub print_netdev_full {
2069     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2070
2071     my $i = '';
2072     if ($netid =~ m/^net(\d+)$/) {
2073         $i = int($1);
2074     }
2075
2076     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2077
2078     my $ifname = "tap${vmid}i$i";
2079
2080     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2081     die "interface name '$ifname' is too long (max 15 character)\n"
2082         if length($ifname) >= 16;
2083
2084     my $vhostparam = '';
2085     if (is_native($arch)) {
2086         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2087     }
2088
2089     my $vmname = $conf->{name} || "vm$vmid";
2090
2091     my $netdev = "";
2092     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2093
2094     if ($net->{bridge}) {
2095         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2096     } else {
2097         $netdev = "type=user,id=$netid,hostname=$vmname";
2098     }
2099
2100     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2101
2102     return $netdev;
2103 }
2104
2105
2106 sub print_cpu_device {
2107     my ($conf, $id) = @_;
2108
2109     my $kvm = $conf->{kvm} // 1;
2110     my $cpu = $kvm ? "kvm64" : "qemu64";
2111     if (my $cputype = $conf->{cpu}) {
2112         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2113             or die "Cannot parse cpu description: $cputype\n";
2114         $cpu = $cpuconf->{cputype};
2115     }
2116
2117     my $cores = $conf->{cores} || 1;
2118
2119     my $current_core = ($id - 1) % $cores;
2120     my $current_socket = int(($id - 1 - $current_core)/$cores);
2121
2122     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2123 }
2124
2125 my $vga_map = {
2126     'cirrus' => 'cirrus-vga',
2127     'std' => 'VGA',
2128     'vmware' => 'vmware-svga',
2129     'virtio' => 'virtio-vga',
2130 };
2131
2132 sub print_vga_device {
2133     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2134
2135     my $type = $vga_map->{$vga->{type}};
2136     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2137         $type = 'virtio-gpu';
2138     }
2139     my $vgamem_mb = $vga->{memory};
2140     if ($qxlnum) {
2141         $type = $id ? 'qxl' : 'qxl-vga';
2142     }
2143     die "no devicetype for $vga->{type}\n" if !$type;
2144
2145     my $memory = "";
2146     if ($vgamem_mb) {
2147         if ($vga->{type} eq 'virtio') {
2148             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2149             $memory = ",max_hostmem=$bytes";
2150         } elsif ($qxlnum) {
2151             # from https://www.spice-space.org/multiple-monitors.html
2152             $memory = ",vgamem_mb=$vga->{memory}";
2153             my $ram = $vgamem_mb * 4;
2154             my $vram = $vgamem_mb * 2;
2155             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2156         } else {
2157             $memory = ",vgamem_mb=$vga->{memory}";
2158         }
2159     } elsif ($qxlnum && $id) {
2160         $memory = ",ram_size=67108864,vram_size=33554432";
2161     }
2162
2163     my $q35 = machine_type_is_q35($conf);
2164     my $vgaid = "vga" . ($id // '');
2165     my $pciaddr;
2166
2167     if ($q35 && $vgaid eq 'vga') {
2168         # the first display uses pcie.0 bus on q35 machines
2169         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2170     } else {
2171         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2172     }
2173
2174     return "$type,id=${vgaid}${memory}${pciaddr}";
2175 }
2176
2177 sub drive_is_cloudinit {
2178     my ($drive) = @_;
2179     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2180 }
2181
2182 sub drive_is_cdrom {
2183     my ($drive, $exclude_cloudinit) = @_;
2184
2185     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2186
2187     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2188
2189 }
2190
2191 sub parse_number_sets {
2192     my ($set) = @_;
2193     my $res = [];
2194     foreach my $part (split(/;/, $set)) {
2195         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2196             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2197             push @$res, [ $1, $2 ];
2198         } else {
2199             die "invalid range: $part\n";
2200         }
2201     }
2202     return $res;
2203 }
2204
2205 sub parse_numa {
2206     my ($data) = @_;
2207
2208     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2209     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2210     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2211     return $res;
2212 }
2213
2214 sub parse_hostpci {
2215     my ($value) = @_;
2216
2217     return undef if !$value;
2218
2219     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2220
2221     my @idlist = split(/;/, $res->{host});
2222     delete $res->{host};
2223     foreach my $id (@idlist) {
2224         if ($id =~ m/\./) { # full id 00:00.1
2225             push @{$res->{pciid}}, {
2226                 id => $id,
2227             };
2228         } else { # partial id 00:00
2229             $res->{pciid} = PVE::SysFSTools::lspci($id);
2230         }
2231     }
2232     return $res;
2233 }
2234
2235 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2236 sub parse_net {
2237     my ($data) = @_;
2238
2239     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2240     if ($@) {
2241         warn $@;
2242         return undef;
2243     }
2244     if (!defined($res->{macaddr})) {
2245         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2246         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2247     }
2248     return $res;
2249 }
2250
2251 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2252 sub parse_ipconfig {
2253     my ($data) = @_;
2254
2255     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2256     if ($@) {
2257         warn $@;
2258         return undef;
2259     }
2260
2261     if ($res->{gw} && !$res->{ip}) {
2262         warn 'gateway specified without specifying an IP address';
2263         return undef;
2264     }
2265     if ($res->{gw6} && !$res->{ip6}) {
2266         warn 'IPv6 gateway specified without specifying an IPv6 address';
2267         return undef;
2268     }
2269     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2270         warn 'gateway specified together with DHCP';
2271         return undef;
2272     }
2273     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2274         # gw6 + auto/dhcp
2275         warn "IPv6 gateway specified together with $res->{ip6} address";
2276         return undef;
2277     }
2278
2279     if (!$res->{ip} && !$res->{ip6}) {
2280         return { ip => 'dhcp', ip6 => 'dhcp' };
2281     }
2282
2283     return $res;
2284 }
2285
2286 sub print_net {
2287     my $net = shift;
2288
2289     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2290 }
2291
2292 sub add_random_macs {
2293     my ($settings) = @_;
2294
2295     foreach my $opt (keys %$settings) {
2296         next if $opt !~ m/^net(\d+)$/;
2297         my $net = parse_net($settings->{$opt});
2298         next if !$net;
2299         $settings->{$opt} = print_net($net);
2300     }
2301 }
2302
2303 sub vm_is_volid_owner {
2304     my ($storecfg, $vmid, $volid) = @_;
2305
2306     if ($volid !~  m|^/|) {
2307         my ($path, $owner);
2308         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2309         if ($owner && ($owner == $vmid)) {
2310             return 1;
2311         }
2312     }
2313
2314     return undef;
2315 }
2316
2317 sub split_flagged_list {
2318     my $text = shift || '';
2319     $text =~ s/[,;]/ /g;
2320     $text =~ s/^\s+//;
2321     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2322 }
2323
2324 sub join_flagged_list {
2325     my ($how, $lst) = @_;
2326     join $how, map { $lst->{$_} . $_ } keys %$lst;
2327 }
2328
2329 sub vmconfig_delete_pending_option {
2330     my ($conf, $key, $force) = @_;
2331
2332     delete $conf->{pending}->{$key};
2333     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2334     $pending_delete_hash->{$key} = $force ? '!' : '';
2335     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2336 }
2337
2338 sub vmconfig_undelete_pending_option {
2339     my ($conf, $key) = @_;
2340
2341     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2342     delete $pending_delete_hash->{$key};
2343
2344     if (%$pending_delete_hash) {
2345         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2346     } else {
2347         delete $conf->{pending}->{delete};
2348     }
2349 }
2350
2351 sub vmconfig_register_unused_drive {
2352     my ($storecfg, $vmid, $conf, $drive) = @_;
2353
2354     if (drive_is_cloudinit($drive)) {
2355         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2356         warn $@ if $@;
2357     } elsif (!drive_is_cdrom($drive)) {
2358         my $volid = $drive->{file};
2359         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2360             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2361         }
2362     }
2363 }
2364
2365 sub vmconfig_cleanup_pending {
2366     my ($conf) = @_;
2367
2368     # remove pending changes when nothing changed
2369     my $changes;
2370     foreach my $opt (keys %{$conf->{pending}}) {
2371         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2372             $changes = 1;
2373             delete $conf->{pending}->{$opt};
2374         }
2375     }
2376
2377     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2378     my $pending_delete_hash = {};
2379     while (my ($opt, $force) = each %$current_delete_hash) {
2380         if (defined($conf->{$opt})) {
2381             $pending_delete_hash->{$opt} = $force;
2382         } else {
2383             $changes = 1;
2384         }
2385     }
2386
2387     if (%$pending_delete_hash) {
2388         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2389     } else {
2390         delete $conf->{pending}->{delete};
2391     }
2392
2393     return $changes;
2394 }
2395
2396 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2397 my $smbios1_fmt = {
2398     uuid => {
2399         type => 'string',
2400         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2401         format_description => 'UUID',
2402         description => "Set SMBIOS1 UUID.",
2403         optional => 1,
2404     },
2405     version => {
2406         type => 'string',
2407         pattern => '[A-Za-z0-9+\/]+={0,2}',
2408         format_description => 'Base64 encoded string',
2409         description => "Set SMBIOS1 version.",
2410         optional => 1,
2411     },
2412     serial => {
2413         type => 'string',
2414         pattern => '[A-Za-z0-9+\/]+={0,2}',
2415         format_description => 'Base64 encoded string',
2416         description => "Set SMBIOS1 serial number.",
2417         optional => 1,
2418     },
2419     manufacturer => {
2420         type => 'string',
2421         pattern => '[A-Za-z0-9+\/]+={0,2}',
2422         format_description => 'Base64 encoded string',
2423         description => "Set SMBIOS1 manufacturer.",
2424         optional => 1,
2425     },
2426     product => {
2427         type => 'string',
2428         pattern => '[A-Za-z0-9+\/]+={0,2}',
2429         format_description => 'Base64 encoded string',
2430         description => "Set SMBIOS1 product ID.",
2431         optional => 1,
2432     },
2433     sku => {
2434         type => 'string',
2435         pattern => '[A-Za-z0-9+\/]+={0,2}',
2436         format_description => 'Base64 encoded string',
2437         description => "Set SMBIOS1 SKU string.",
2438         optional => 1,
2439     },
2440     family => {
2441         type => 'string',
2442         pattern => '[A-Za-z0-9+\/]+={0,2}',
2443         format_description => 'Base64 encoded string',
2444         description => "Set SMBIOS1 family string.",
2445         optional => 1,
2446     },
2447     base64 => {
2448         type => 'boolean',
2449         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2450         optional => 1,
2451     },
2452 };
2453
2454 sub parse_smbios1 {
2455     my ($data) = @_;
2456
2457     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2458     warn $@ if $@;
2459     return $res;
2460 }
2461
2462 sub print_smbios1 {
2463     my ($smbios1) = @_;
2464     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2465 }
2466
2467 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2468
2469 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2470 sub verify_bootdisk {
2471     my ($value, $noerr) = @_;
2472
2473     return $value if is_valid_drivename($value);
2474
2475     return undef if $noerr;
2476
2477     die "invalid boot disk '$value'\n";
2478 }
2479
2480 sub parse_watchdog {
2481     my ($value) = @_;
2482
2483     return undef if !$value;
2484
2485     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2486     warn $@ if $@;
2487     return $res;
2488 }
2489
2490 sub parse_guest_agent {
2491     my ($value) = @_;
2492
2493     return {} if !defined($value->{agent});
2494
2495     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2496     warn $@ if $@;
2497
2498     # if the agent is disabled ignore the other potentially set properties
2499     return {} if !$res->{enabled};
2500     return $res;
2501 }
2502
2503 sub parse_vga {
2504     my ($value) = @_;
2505
2506     return {} if !$value;
2507     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2508     warn $@ if $@;
2509     return $res;
2510 }
2511
2512 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2513 sub verify_usb_device {
2514     my ($value, $noerr) = @_;
2515
2516     return $value if parse_usb_device($value);
2517
2518     return undef if $noerr;
2519
2520     die "unable to parse usb device\n";
2521 }
2522
2523 # add JSON properties for create and set function
2524 sub json_config_properties {
2525     my $prop = shift;
2526
2527     foreach my $opt (keys %$confdesc) {
2528         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2529         $prop->{$opt} = $confdesc->{$opt};
2530     }
2531
2532     return $prop;
2533 }
2534
2535 # return copy of $confdesc_cloudinit to generate documentation
2536 sub cloudinit_config_properties {
2537
2538     return dclone($confdesc_cloudinit);
2539 }
2540
2541 sub check_type {
2542     my ($key, $value) = @_;
2543
2544     die "unknown setting '$key'\n" if !$confdesc->{$key};
2545
2546     my $type = $confdesc->{$key}->{type};
2547
2548     if (!defined($value)) {
2549         die "got undefined value\n";
2550     }
2551
2552     if ($value =~ m/[\n\r]/) {
2553         die "property contains a line feed\n";
2554     }
2555
2556     if ($type eq 'boolean') {
2557         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2558         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2559         die "type check ('boolean') failed - got '$value'\n";
2560     } elsif ($type eq 'integer') {
2561         return int($1) if $value =~ m/^(\d+)$/;
2562         die "type check ('integer') failed - got '$value'\n";
2563     } elsif ($type eq 'number') {
2564         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2565         die "type check ('number') failed - got '$value'\n";
2566     } elsif ($type eq 'string') {
2567         if (my $fmt = $confdesc->{$key}->{format}) {
2568             PVE::JSONSchema::check_format($fmt, $value);
2569             return $value;
2570         }
2571         $value =~ s/^\"(.*)\"$/$1/;
2572         return $value;
2573     } else {
2574         die "internal error"
2575     }
2576 }
2577
2578 sub touch_config {
2579     my ($vmid) = @_;
2580
2581     my $conf = PVE::QemuConfig->config_file($vmid);
2582     utime undef, undef, $conf;
2583 }
2584
2585 sub destroy_vm {
2586     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2587
2588     my $conffile = PVE::QemuConfig->config_file($vmid);
2589
2590     my $conf = PVE::QemuConfig->load_config($vmid);
2591
2592     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2593
2594     if ($conf->{template}) {
2595         # check if any base image is still used by a linked clone
2596         foreach_drive($conf, sub {
2597                 my ($ds, $drive) = @_;
2598
2599                 return if drive_is_cdrom($drive);
2600
2601                 my $volid = $drive->{file};
2602
2603                 return if !$volid || $volid =~ m|^/|;
2604
2605                 die "base volume '$volid' is still in use by linked cloned\n"
2606                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2607
2608         });
2609     }
2610
2611     # only remove disks owned by this VM
2612     foreach_drive($conf, sub {
2613         my ($ds, $drive) = @_;
2614
2615         return if drive_is_cdrom($drive, 1);
2616
2617         my $volid = $drive->{file};
2618
2619         return if !$volid || $volid =~ m|^/|;
2620
2621         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2622         return if !$path || !$owner || ($owner != $vmid);
2623
2624         eval {
2625             PVE::Storage::vdisk_free($storecfg, $volid);
2626         };
2627         warn "Could not remove disk '$volid', check manually: $@" if $@;
2628
2629     });
2630
2631     if ($keep_empty_config) {
2632         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2633     } else {
2634         unlink $conffile;
2635     }
2636
2637     # also remove unused disk
2638     eval {
2639         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2640
2641         eval {
2642             PVE::Storage::foreach_volid($dl, sub {
2643                 my ($volid, $sid, $volname, $d) = @_;
2644                 PVE::Storage::vdisk_free($storecfg, $volid);
2645             });
2646         };
2647         warn $@ if $@;
2648
2649     };
2650     warn $@ if $@;
2651 }
2652
2653 sub parse_vm_config {
2654     my ($filename, $raw) = @_;
2655
2656     return undef if !defined($raw);
2657
2658     my $res = {
2659         digest => Digest::SHA::sha1_hex($raw),
2660         snapshots => {},
2661         pending => {},
2662     };
2663
2664     $filename =~ m|/qemu-server/(\d+)\.conf$|
2665         || die "got strange filename '$filename'";
2666
2667     my $vmid = $1;
2668
2669     my $conf = $res;
2670     my $descr;
2671     my $section = '';
2672
2673     my @lines = split(/\n/, $raw);
2674     foreach my $line (@lines) {
2675         next if $line =~ m/^\s*$/;
2676
2677         if ($line =~ m/^\[PENDING\]\s*$/i) {
2678             $section = 'pending';
2679             if (defined($descr)) {
2680                 $descr =~ s/\s+$//;
2681                 $conf->{description} = $descr;
2682             }
2683             $descr = undef;
2684             $conf = $res->{$section} = {};
2685             next;
2686
2687         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2688             $section = $1;
2689             if (defined($descr)) {
2690                 $descr =~ s/\s+$//;
2691                 $conf->{description} = $descr;
2692             }
2693             $descr = undef;
2694             $conf = $res->{snapshots}->{$section} = {};
2695             next;
2696         }
2697
2698         if ($line =~ m/^\#(.*)\s*$/) {
2699             $descr = '' if !defined($descr);
2700             $descr .= PVE::Tools::decode_text($1) . "\n";
2701             next;
2702         }
2703
2704         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2705             $descr = '' if !defined($descr);
2706             $descr .= PVE::Tools::decode_text($2);
2707         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2708             $conf->{snapstate} = $1;
2709         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2710             my $key = $1;
2711             my $value = $2;
2712             $conf->{$key} = $value;
2713         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2714             my $value = $1;
2715             if ($section eq 'pending') {
2716                 $conf->{delete} = $value; # we parse this later
2717             } else {
2718                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2719             }
2720         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2721             my $key = $1;
2722             my $value = $2;
2723             eval { $value = check_type($key, $value); };
2724             if ($@) {
2725                 warn "vm $vmid - unable to parse value of '$key' - $@";
2726             } else {
2727                 $key = 'ide2' if $key eq 'cdrom';
2728                 my $fmt = $confdesc->{$key}->{format};
2729                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2730                     my $v = parse_drive($key, $value);
2731                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2732                         $v->{file} = $volid;
2733                         $value = print_drive($vmid, $v);
2734                     } else {
2735                         warn "vm $vmid - unable to parse value of '$key'\n";
2736                         next;
2737                     }
2738                 }
2739
2740                 $conf->{$key} = $value;
2741             }
2742         }
2743     }
2744
2745     if (defined($descr)) {
2746         $descr =~ s/\s+$//;
2747         $conf->{description} = $descr;
2748     }
2749     delete $res->{snapstate}; # just to be sure
2750
2751     return $res;
2752 }
2753
2754 sub write_vm_config {
2755     my ($filename, $conf) = @_;
2756
2757     delete $conf->{snapstate}; # just to be sure
2758
2759     if ($conf->{cdrom}) {
2760         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2761         $conf->{ide2} = $conf->{cdrom};
2762         delete $conf->{cdrom};
2763     }
2764
2765     # we do not use 'smp' any longer
2766     if ($conf->{sockets}) {
2767         delete $conf->{smp};
2768     } elsif ($conf->{smp}) {
2769         $conf->{sockets} = $conf->{smp};
2770         delete $conf->{cores};
2771         delete $conf->{smp};
2772     }
2773
2774     my $used_volids = {};
2775
2776     my $cleanup_config = sub {
2777         my ($cref, $pending, $snapname) = @_;
2778
2779         foreach my $key (keys %$cref) {
2780             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2781                 $key eq 'snapstate' || $key eq 'pending';
2782             my $value = $cref->{$key};
2783             if ($key eq 'delete') {
2784                 die "propertry 'delete' is only allowed in [PENDING]\n"
2785                     if !$pending;
2786                 # fixme: check syntax?
2787                 next;
2788             }
2789             eval { $value = check_type($key, $value); };
2790             die "unable to parse value of '$key' - $@" if $@;
2791
2792             $cref->{$key} = $value;
2793
2794             if (!$snapname && is_valid_drivename($key)) {
2795                 my $drive = parse_drive($key, $value);
2796                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2797             }
2798         }
2799     };
2800
2801     &$cleanup_config($conf);
2802
2803     &$cleanup_config($conf->{pending}, 1);
2804
2805     foreach my $snapname (keys %{$conf->{snapshots}}) {
2806         die "internal error" if $snapname eq 'pending';
2807         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2808     }
2809
2810     # remove 'unusedX' settings if we re-add a volume
2811     foreach my $key (keys %$conf) {
2812         my $value = $conf->{$key};
2813         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2814             delete $conf->{$key};
2815         }
2816     }
2817
2818     my $generate_raw_config = sub {
2819         my ($conf, $pending) = @_;
2820
2821         my $raw = '';
2822
2823         # add description as comment to top of file
2824         if (defined(my $descr = $conf->{description})) {
2825             if ($descr) {
2826                 foreach my $cl (split(/\n/, $descr)) {
2827                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2828                 }
2829             } else {
2830                 $raw .= "#\n" if $pending;
2831             }
2832         }
2833
2834         foreach my $key (sort keys %$conf) {
2835             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2836             $raw .= "$key: $conf->{$key}\n";
2837         }
2838         return $raw;
2839     };
2840
2841     my $raw = &$generate_raw_config($conf);
2842
2843     if (scalar(keys %{$conf->{pending}})){
2844         $raw .= "\n[PENDING]\n";
2845         $raw .= &$generate_raw_config($conf->{pending}, 1);
2846     }
2847
2848     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2849         $raw .= "\n[$snapname]\n";
2850         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2851     }
2852
2853     return $raw;
2854 }
2855
2856 sub load_defaults {
2857
2858     my $res = {};
2859
2860     # we use static defaults from our JSON schema configuration
2861     foreach my $key (keys %$confdesc) {
2862         if (defined(my $default = $confdesc->{$key}->{default})) {
2863             $res->{$key} = $default;
2864         }
2865     }
2866
2867     return $res;
2868 }
2869
2870 sub config_list {
2871     my $vmlist = PVE::Cluster::get_vmlist();
2872     my $res = {};
2873     return $res if !$vmlist || !$vmlist->{ids};
2874     my $ids = $vmlist->{ids};
2875
2876     foreach my $vmid (keys %$ids) {
2877         my $d = $ids->{$vmid};
2878         next if !$d->{node} || $d->{node} ne $nodename;
2879         next if !$d->{type} || $d->{type} ne 'qemu';
2880         $res->{$vmid}->{exists} = 1;
2881     }
2882     return $res;
2883 }
2884
2885 # test if VM uses local resources (to prevent migration)
2886 sub check_local_resources {
2887     my ($conf, $noerr) = @_;
2888
2889     my @loc_res = ();
2890
2891     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2892     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2893
2894     push @loc_res, "ivshmem" if $conf->{ivshmem};
2895
2896     foreach my $k (keys %$conf) {
2897         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2898         # sockets are safe: they will recreated be on the target side post-migrate
2899         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2900         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2901     }
2902
2903     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2904
2905     return \@loc_res;
2906 }
2907
2908 # check if used storages are available on all nodes (use by migrate)
2909 sub check_storage_availability {
2910     my ($storecfg, $conf, $node) = @_;
2911
2912     foreach_drive($conf, sub {
2913         my ($ds, $drive) = @_;
2914
2915         my $volid = $drive->{file};
2916         return if !$volid;
2917
2918         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2919         return if !$sid;
2920
2921         # check if storage is available on both nodes
2922         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2923         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2924    });
2925 }
2926
2927 # list nodes where all VM images are available (used by has_feature API)
2928 sub shared_nodes {
2929     my ($conf, $storecfg) = @_;
2930
2931     my $nodelist = PVE::Cluster::get_nodelist();
2932     my $nodehash = { map { $_ => 1 } @$nodelist };
2933     my $nodename = PVE::INotify::nodename();
2934
2935     foreach_drive($conf, sub {
2936         my ($ds, $drive) = @_;
2937
2938         my $volid = $drive->{file};
2939         return if !$volid;
2940
2941         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2942         if ($storeid) {
2943             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2944             if ($scfg->{disable}) {
2945                 $nodehash = {};
2946             } elsif (my $avail = $scfg->{nodes}) {
2947                 foreach my $node (keys %$nodehash) {
2948                     delete $nodehash->{$node} if !$avail->{$node};
2949                 }
2950             } elsif (!$scfg->{shared}) {
2951                 foreach my $node (keys %$nodehash) {
2952                     delete $nodehash->{$node} if $node ne $nodename
2953                 }
2954             }
2955         }
2956     });
2957
2958     return $nodehash
2959 }
2960
2961 sub check_local_storage_availability {
2962     my ($conf, $storecfg) = @_;
2963
2964     my $nodelist = PVE::Cluster::get_nodelist();
2965     my $nodehash = { map { $_ => {} } @$nodelist };
2966
2967     foreach_drive($conf, sub {
2968         my ($ds, $drive) = @_;
2969
2970         my $volid = $drive->{file};
2971         return if !$volid;
2972
2973         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2974         if ($storeid) {
2975             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2976
2977             if ($scfg->{disable}) {
2978                 foreach my $node (keys %$nodehash) {
2979                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2980                 }
2981             } elsif (my $avail = $scfg->{nodes}) {
2982                 foreach my $node (keys %$nodehash) {
2983                     if (!$avail->{$node}) {
2984                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2985                     }
2986                 }
2987             }
2988         }
2989     });
2990
2991     foreach my $node (values %$nodehash) {
2992         if (my $unavail = $node->{unavailable_storages}) {
2993             $node->{unavailable_storages} = [ sort keys %$unavail ];
2994         }
2995     }
2996
2997     return $nodehash
2998 }
2999
3000 sub check_cmdline {
3001     my ($pidfile, $pid) = @_;
3002
3003     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3004     if (defined($fh)) {
3005         my $line = <$fh>;
3006         $fh->close;
3007         return undef if !$line;
3008         my @param = split(/\0/, $line);
3009
3010         my $cmd = $param[0];
3011         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3012
3013         for (my $i = 0; $i < scalar (@param); $i++) {
3014             my $p = $param[$i];
3015             next if !$p;
3016             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3017                 my $p = $param[$i+1];
3018                 return 1 if $p && ($p eq $pidfile);
3019                 return undef;
3020             }
3021         }
3022     }
3023     return undef;
3024 }
3025
3026 sub check_running {
3027     my ($vmid, $nocheck, $node) = @_;
3028
3029     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3030
3031     die "unable to find configuration file for VM $vmid - no such machine\n"
3032         if !$nocheck && ! -f $filename;
3033
3034     my $pidfile = pidfile_name($vmid);
3035
3036     if (my $fd = IO::File->new("<$pidfile")) {
3037         my $st = stat($fd);
3038         my $line = <$fd>;
3039         close($fd);
3040
3041         my $mtime = $st->mtime;
3042         if ($mtime > time()) {
3043             warn "file '$filename' modified in future\n";
3044         }
3045
3046         if ($line =~ m/^(\d+)$/) {
3047             my $pid = $1;
3048             if (check_cmdline($pidfile, $pid)) {
3049                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3050                     return $pid;
3051                 }
3052             }
3053         }
3054     }
3055
3056     return undef;
3057 }
3058
3059 sub vzlist {
3060
3061     my $vzlist = config_list();
3062
3063     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3064
3065     while (defined(my $de = $fd->read)) {
3066         next if $de !~ m/^(\d+)\.pid$/;
3067         my $vmid = $1;
3068         next if !defined($vzlist->{$vmid});
3069         if (my $pid = check_running($vmid)) {
3070             $vzlist->{$vmid}->{pid} = $pid;
3071         }
3072     }
3073
3074     return $vzlist;
3075 }
3076
3077 sub disksize {
3078     my ($storecfg, $conf) = @_;
3079
3080     my $bootdisk = $conf->{bootdisk};
3081     return undef if !$bootdisk;
3082     return undef if !is_valid_drivename($bootdisk);
3083
3084     return undef if !$conf->{$bootdisk};
3085
3086     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3087     return undef if !defined($drive);
3088
3089     return undef if drive_is_cdrom($drive);
3090
3091     my $volid = $drive->{file};
3092     return undef if !$volid;
3093
3094     return $drive->{size};
3095 }
3096
3097 our $vmstatus_return_properties = {
3098     vmid => get_standard_option('pve-vmid'),
3099     status => {
3100         description => "Qemu process status.",
3101         type => 'string',
3102         enum => ['stopped', 'running'],
3103     },
3104     maxmem => {
3105         description => "Maximum memory in bytes.",
3106         type => 'integer',
3107         optional => 1,
3108         renderer => 'bytes',
3109     },
3110     maxdisk => {
3111         description => "Root disk size in bytes.",
3112         type => 'integer',
3113         optional => 1,
3114         renderer => 'bytes',
3115     },
3116     name => {
3117         description => "VM name.",
3118         type => 'string',
3119         optional => 1,
3120     },
3121     qmpstatus => {
3122         description => "Qemu QMP agent status.",
3123         type => 'string',
3124         optional => 1,
3125     },
3126     pid => {
3127         description => "PID of running qemu process.",
3128         type => 'integer',
3129         optional => 1,
3130     },
3131     uptime => {
3132         description => "Uptime.",
3133         type => 'integer',
3134         optional => 1,
3135         renderer => 'duration',
3136     },
3137     cpus => {
3138         description => "Maximum usable CPUs.",
3139         type => 'number',
3140         optional => 1,
3141     },
3142     lock => {
3143         description => "The current config lock, if any.",
3144         type => 'string',
3145         optional => 1,
3146     }
3147 };
3148
3149 my $last_proc_pid_stat;
3150
3151 # get VM status information
3152 # This must be fast and should not block ($full == false)
3153 # We only query KVM using QMP if $full == true (this can be slow)
3154 sub vmstatus {
3155     my ($opt_vmid, $full) = @_;
3156
3157     my $res = {};
3158
3159     my $storecfg = PVE::Storage::config();
3160
3161     my $list = vzlist();
3162     my $defaults = load_defaults();
3163
3164     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3165
3166     my $cpucount = $cpuinfo->{cpus} || 1;
3167
3168     foreach my $vmid (keys %$list) {
3169         next if $opt_vmid && ($vmid ne $opt_vmid);
3170
3171         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3172         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3173
3174         my $d = { vmid => $vmid };
3175         $d->{pid} = $list->{$vmid}->{pid};
3176
3177         # fixme: better status?
3178         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3179
3180         my $size = disksize($storecfg, $conf);
3181         if (defined($size)) {
3182             $d->{disk} = 0; # no info available
3183             $d->{maxdisk} = $size;
3184         } else {
3185             $d->{disk} = 0;
3186             $d->{maxdisk} = 0;
3187         }
3188
3189         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3190             * ($conf->{cores} || $defaults->{cores});
3191         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3192         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3193
3194         $d->{name} = $conf->{name} || "VM $vmid";
3195         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3196             : $defaults->{memory}*(1024*1024);
3197
3198         if ($conf->{balloon}) {
3199             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3200             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3201                 : $defaults->{shares};
3202         }
3203
3204         $d->{uptime} = 0;
3205         $d->{cpu} = 0;
3206         $d->{mem} = 0;
3207
3208         $d->{netout} = 0;
3209         $d->{netin} = 0;
3210
3211         $d->{diskread} = 0;
3212         $d->{diskwrite} = 0;
3213
3214         $d->{template} = PVE::QemuConfig->is_template($conf);
3215
3216         $d->{serial} = 1 if conf_has_serial($conf);
3217         $d->{lock} = $conf->{lock} if $conf->{lock};
3218
3219         $res->{$vmid} = $d;
3220     }
3221
3222     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3223     foreach my $dev (keys %$netdev) {
3224         next if $dev !~ m/^tap([1-9]\d*)i/;
3225         my $vmid = $1;
3226         my $d = $res->{$vmid};
3227         next if !$d;
3228
3229         $d->{netout} += $netdev->{$dev}->{receive};
3230         $d->{netin} += $netdev->{$dev}->{transmit};
3231
3232         if ($full) {
3233             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3234             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3235         }
3236
3237     }
3238
3239     my $ctime = gettimeofday;
3240
3241     foreach my $vmid (keys %$list) {
3242
3243         my $d = $res->{$vmid};
3244         my $pid = $d->{pid};
3245         next if !$pid;
3246
3247         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3248         next if !$pstat; # not running
3249
3250         my $used = $pstat->{utime} + $pstat->{stime};
3251
3252         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3253
3254         if ($pstat->{vsize}) {
3255             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3256         }
3257
3258         my $old = $last_proc_pid_stat->{$pid};
3259         if (!$old) {
3260             $last_proc_pid_stat->{$pid} = {
3261                 time => $ctime,
3262                 used => $used,
3263                 cpu => 0,
3264             };
3265             next;
3266         }
3267
3268         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3269
3270         if ($dtime > 1000) {
3271             my $dutime = $used -  $old->{used};
3272
3273             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3274             $last_proc_pid_stat->{$pid} = {
3275                 time => $ctime,
3276                 used => $used,
3277                 cpu => $d->{cpu},
3278             };
3279         } else {
3280             $d->{cpu} = $old->{cpu};
3281         }
3282     }
3283
3284     return $res if !$full;
3285
3286     my $qmpclient = PVE::QMPClient->new();
3287
3288     my $ballooncb = sub {
3289         my ($vmid, $resp) = @_;
3290
3291         my $info = $resp->{'return'};
3292         return if !$info->{max_mem};
3293
3294         my $d = $res->{$vmid};
3295
3296         # use memory assigned to VM
3297         $d->{maxmem} = $info->{max_mem};
3298         $d->{balloon} = $info->{actual};
3299
3300         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3301             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3302             $d->{freemem} = $info->{free_mem};
3303         }
3304
3305         $d->{ballooninfo} = $info;
3306     };
3307
3308     my $blockstatscb = sub {
3309         my ($vmid, $resp) = @_;
3310         my $data = $resp->{'return'} || [];
3311         my $totalrdbytes = 0;
3312         my $totalwrbytes = 0;
3313
3314         for my $blockstat (@$data) {
3315             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3316             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3317
3318             $blockstat->{device} =~ s/drive-//;
3319             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3320         }
3321         $res->{$vmid}->{diskread} = $totalrdbytes;
3322         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3323     };
3324
3325     my $statuscb = sub {
3326         my ($vmid, $resp) = @_;
3327
3328         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3329         # this fails if ballon driver is not loaded, so this must be
3330         # the last commnand (following command are aborted if this fails).
3331         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3332
3333         my $status = 'unknown';
3334         if (!defined($status = $resp->{'return'}->{status})) {
3335             warn "unable to get VM status\n";
3336             return;
3337         }
3338
3339         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3340     };
3341
3342     foreach my $vmid (keys %$list) {
3343         next if $opt_vmid && ($vmid ne $opt_vmid);
3344         next if !$res->{$vmid}->{pid}; # not running
3345         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3346     }
3347
3348     $qmpclient->queue_execute(undef, 2);
3349
3350     foreach my $vmid (keys %$list) {
3351         next if $opt_vmid && ($vmid ne $opt_vmid);
3352         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3353     }
3354
3355     return $res;
3356 }
3357
3358 sub foreach_drive {
3359     my ($conf, $func, @param) = @_;
3360
3361     foreach my $ds (valid_drive_names()) {
3362         next if !defined($conf->{$ds});
3363
3364         my $drive = parse_drive($ds, $conf->{$ds});
3365         next if !$drive;
3366
3367         &$func($ds, $drive, @param);
3368     }
3369 }
3370
3371 sub foreach_volid {
3372     my ($conf, $func, @param) = @_;
3373
3374     my $volhash = {};
3375
3376     my $test_volid = sub {
3377         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3378
3379         return if !$volid;
3380
3381         $volhash->{$volid}->{cdrom} //= 1;
3382         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3383
3384         $volhash->{$volid}->{replicate} //= 0;
3385         $volhash->{$volid}->{replicate} = 1 if $replicate;
3386
3387         $volhash->{$volid}->{shared} //= 0;
3388         $volhash->{$volid}->{shared} = 1 if $shared;
3389
3390         $volhash->{$volid}->{referenced_in_config} //= 0;
3391         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3392
3393         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3394             if defined($snapname);
3395         $volhash->{$volid}->{size} = $size if $size;
3396     };
3397
3398     foreach_drive($conf, sub {
3399         my ($ds, $drive) = @_;
3400         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3401     });
3402
3403     foreach my $snapname (keys %{$conf->{snapshots}}) {
3404         my $snap = $conf->{snapshots}->{$snapname};
3405         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3406         foreach_drive($snap, sub {
3407             my ($ds, $drive) = @_;
3408             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3409         });
3410     }
3411
3412     foreach my $volid (keys %$volhash) {
3413         &$func($volid, $volhash->{$volid}, @param);
3414     }
3415 }
3416
3417 sub conf_has_serial {
3418     my ($conf) = @_;
3419
3420     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3421         if ($conf->{"serial$i"}) {
3422             return 1;
3423         }
3424     }
3425
3426     return 0;
3427 }
3428
3429 sub conf_has_audio {
3430     my ($conf, $id) = @_;
3431
3432     $id //= 0;
3433     my $audio = $conf->{"audio$id"};
3434     return undef if !defined($audio);
3435
3436     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3437     my $audiodriver = $audioproperties->{driver} // 'spice';
3438
3439     return {
3440         dev => $audioproperties->{device},
3441         dev_id => "audiodev$id",
3442         backend => $audiodriver,
3443         backend_id => "$audiodriver-backend${id}",
3444     };
3445 }
3446
3447 sub vga_conf_has_spice {
3448     my ($vga) = @_;
3449
3450     my $vgaconf = parse_vga($vga);
3451     my $vgatype = $vgaconf->{type};
3452     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3453
3454     return $1 || 1;
3455 }
3456
3457 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3458 sub get_host_arch() {
3459     $host_arch = (POSIX::uname())[4] if !$host_arch;
3460     return $host_arch;
3461 }
3462
3463 sub is_native($) {
3464     my ($arch) = @_;
3465     return get_host_arch() eq $arch;
3466 }
3467
3468 my $default_machines = {
3469     x86_64 => 'pc',
3470     aarch64 => 'virt',
3471 };
3472
3473 sub get_basic_machine_info {
3474     my ($conf, $forcemachine) = @_;
3475
3476     my $arch = $conf->{arch} // get_host_arch();
3477     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3478     return ($arch, $machine);
3479 }
3480
3481 sub get_ovmf_files($) {
3482     my ($arch) = @_;
3483
3484     my $ovmf = $OVMF->{$arch}
3485         or die "no OVMF images known for architecture '$arch'\n";
3486
3487     return @$ovmf;
3488 }
3489
3490 my $Arch2Qemu = {
3491     aarch64 => '/usr/bin/qemu-system-aarch64',
3492     x86_64 => '/usr/bin/qemu-system-x86_64',
3493 };
3494 sub get_command_for_arch($) {
3495     my ($arch) = @_;
3496     return '/usr/bin/kvm' if is_native($arch);
3497
3498     my $cmd = $Arch2Qemu->{$arch}
3499         or die "don't know how to emulate architecture '$arch'\n";
3500     return $cmd;
3501 }
3502
3503 sub get_cpu_options {
3504     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3505
3506     my $cpuFlags = [];
3507     my $ostype = $conf->{ostype};
3508
3509     my $cpu = $kvm ? "kvm64" : "qemu64";
3510     if ($arch eq 'aarch64') {
3511         $cpu = 'cortex-a57';
3512     }
3513     my $hv_vendor_id;
3514     if (my $cputype = $conf->{cpu}) {
3515         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3516             or die "Cannot parse cpu description: $cputype\n";
3517         $cpu = $cpuconf->{cputype};
3518         $kvm_off = 1 if $cpuconf->{hidden};
3519         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3520
3521         if (defined(my $flags = $cpuconf->{flags})) {
3522             push @$cpuFlags, split(";", $flags);
3523         }
3524     }
3525
3526     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3527
3528     push @$cpuFlags , '-x2apic'
3529         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3530
3531     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3532
3533     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3534
3535     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3536
3537         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3538         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3539     }
3540
3541     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3542
3543     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3544
3545     push @$cpuFlags, 'kvm=off' if $kvm_off;
3546
3547     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3548         push @$cpuFlags, "vendor=${cpu_vendor}"
3549             if $cpu_vendor ne 'default';
3550     } elsif ($arch ne 'aarch64') {
3551         die "internal error"; # should not happen
3552     }
3553
3554     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3555
3556     return ('-cpu', $cpu);
3557 }
3558
3559 sub config_to_command {
3560     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3561
3562     my $cmd = [];
3563     my $globalFlags = [];
3564     my $machineFlags = [];
3565     my $rtcFlags = [];
3566     my $devices = [];
3567     my $pciaddr = '';
3568     my $bridges = {};
3569     my $kvmver = kvm_user_version();
3570     my $vernum = 0; # unknown
3571     my $ostype = $conf->{ostype};
3572     my $winversion = windows_version($ostype);
3573     my $kvm = $conf->{kvm};
3574
3575     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3576     $kvm //= 1 if is_native($arch);
3577
3578     if ($kvm) {
3579         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3580             if !defined kvm_version();
3581     }
3582
3583     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3584         $vernum = $1*1000000+$2*1000;
3585     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3586         $vernum = $1*1000000+$2*1000+$3;
3587     }
3588
3589     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3590
3591     my $have_ovz = -f '/proc/vz/vestat';
3592
3593     my $q35 = machine_type_is_q35($conf);
3594     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3595     my $use_old_bios_files = undef;
3596     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3597
3598     my $cpuunits = defined($conf->{cpuunits}) ?
3599             $conf->{cpuunits} : $defaults->{cpuunits};
3600
3601     push @$cmd, get_command_for_arch($arch);
3602
3603     push @$cmd, '-id', $vmid;
3604
3605     my $vmname = $conf->{name} || "vm$vmid";
3606
3607     push @$cmd, '-name', $vmname;
3608
3609     my $use_virtio = 0;
3610
3611     my $qmpsocket = qmp_socket($vmid);
3612     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3613     push @$cmd, '-mon', "chardev=qmp,mode=control";
3614
3615     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3616         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3617         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3618     }
3619
3620     push @$cmd, '-pidfile' , pidfile_name($vmid);
3621
3622     push @$cmd, '-daemonize';
3623
3624     if ($conf->{smbios1}) {
3625         my $smbios_conf = parse_smbios1($conf->{smbios1});
3626         if ($smbios_conf->{base64}) {
3627             # Do not pass base64 flag to qemu
3628             delete $smbios_conf->{base64};
3629             my $smbios_string = "";
3630             foreach my $key (keys %$smbios_conf) {
3631                 my $value;
3632                 if ($key eq "uuid") {
3633                     $value = $smbios_conf->{uuid}
3634                 } else {
3635                     $value = decode_base64($smbios_conf->{$key});
3636                 }
3637                 # qemu accepts any binary data, only commas need escaping by double comma
3638                 $value =~ s/,/,,/g;
3639                 $smbios_string .= "," . $key . "=" . $value if $value;
3640             }
3641             push @$cmd, '-smbios', "type=1" . $smbios_string;
3642         } else {
3643             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3644         }
3645     }
3646
3647     if ($conf->{vmgenid}) {
3648         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3649     }
3650
3651     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3652     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3653         die "uefi base image not found\n" if ! -f $ovmf_code;
3654
3655         my $path;
3656         my $format;
3657         if (my $efidisk = $conf->{efidisk0}) {
3658             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3659             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3660             $format = $d->{format};
3661             if ($storeid) {
3662                 $path = PVE::Storage::path($storecfg, $d->{file});
3663                 if (!defined($format)) {
3664                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3665                     $format = qemu_img_format($scfg, $volname);
3666                 }
3667             } else {
3668                 $path = $d->{file};
3669                 die "efidisk format must be specified\n"
3670                     if !defined($format);
3671             }
3672         } else {
3673             warn "no efidisk configured! Using temporary efivars disk.\n";
3674             $path = "/tmp/$vmid-ovmf.fd";
3675             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3676             $format = 'raw';
3677         }
3678
3679         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3680         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3681     }
3682
3683     # load q35 config
3684     if ($q35) {
3685         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3686         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3687             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3688         } else {
3689             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3690         }
3691     }
3692
3693     # add usb controllers
3694     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3695     push @$devices, @usbcontrollers if @usbcontrollers;
3696     my $vga = parse_vga($conf->{vga});
3697
3698     my $qxlnum = vga_conf_has_spice($conf->{vga});
3699     $vga->{type} = 'qxl' if $qxlnum;
3700
3701     if (!$vga->{type}) {
3702         if ($arch eq 'aarch64') {
3703             $vga->{type} = 'virtio';
3704         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3705             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3706         } else {
3707             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3708         }
3709     }
3710
3711     # enable absolute mouse coordinates (needed by vnc)
3712     my $tablet;
3713     if (defined($conf->{tablet})) {
3714         $tablet = $conf->{tablet};
3715     } else {
3716         $tablet = $defaults->{tablet};
3717         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3718         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3719     }
3720
3721     if ($tablet) {
3722         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3723         my $kbd = print_keyboarddevice_full($conf, $arch);
3724         push @$devices, '-device', $kbd if defined($kbd);
3725     }
3726
3727     my $kvm_off = 0;
3728     my $gpu_passthrough;
3729
3730     # host pci devices
3731     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3732         my $d = parse_hostpci($conf->{"hostpci$i"});
3733         next if !$d;
3734
3735         my $pcie = $d->{pcie};
3736         if ($pcie) {
3737             die "q35 machine model is not enabled" if !$q35;
3738             # win7 wants to have the pcie devices directly on the pcie bus
3739             # instead of in the root port
3740             if ($winversion == 7) {
3741                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3742             } else {
3743                 $pciaddr = print_pcie_addr("hostpci$i");
3744             }
3745         } else {
3746             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3747         }
3748
3749         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3750         my $romfile = $d->{romfile};
3751
3752         my $xvga = '';
3753         if ($d->{'x-vga'}) {
3754             $xvga = ',x-vga=on';
3755             $kvm_off = 1;
3756             $vga->{type} = 'none' if !defined($conf->{vga});
3757             $gpu_passthrough = 1;
3758
3759             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3760                 $xvga = "";
3761             }
3762         }
3763         my $pcidevices = $d->{pciid};
3764         my $multifunction = 1 if @$pcidevices > 1;
3765         my $sysfspath;
3766         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3767             my $id = $pcidevices->[0]->{id};
3768             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3769             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3770         } elsif ($d->{mdev}) {
3771             warn "ignoring mediated device with multifunction device\n";
3772         }
3773
3774         my $j=0;
3775         foreach my $pcidevice (@$pcidevices) {
3776
3777             my $id = "hostpci$i";
3778             $id .= ".$j" if $multifunction;
3779             my $addr = $pciaddr;
3780             $addr .= ".$j" if $multifunction;
3781             my $devicestr = "vfio-pci";
3782             if ($sysfspath) {
3783                 $devicestr .= ",sysfsdev=$sysfspath";
3784             } else {
3785                 $devicestr .= ",host=$pcidevice->{id}";
3786             }
3787             $devicestr .= ",id=$id$addr";
3788
3789             if($j == 0){
3790                 $devicestr .= "$rombar$xvga";
3791                 $devicestr .= ",multifunction=on" if $multifunction;
3792                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3793             }
3794
3795             push @$devices, '-device', $devicestr;
3796             $j++;
3797         }
3798     }
3799
3800     # usb devices
3801     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3802     push @$devices, @usbdevices if @usbdevices;
3803     # serial devices
3804     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3805         if (my $path = $conf->{"serial$i"}) {
3806             if ($path eq 'socket') {
3807                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3808                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3809                 # On aarch64, serial0 is the UART device. Qemu only allows
3810                 # connecting UART devices via the '-serial' command line, as
3811                 # the device has a fixed slot on the hardware...
3812                 if ($arch eq 'aarch64' && $i == 0) {
3813                     push @$devices, '-serial', "chardev:serial$i";
3814                 } else {
3815                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3816                 }
3817             } else {
3818                 die "no such serial device\n" if ! -c $path;
3819                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3820                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3821             }
3822         }
3823     }
3824
3825     # parallel devices
3826     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3827         if (my $path = $conf->{"parallel$i"}) {
3828             die "no such parallel device\n" if ! -c $path;
3829             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3830             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3831             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3832         }
3833     }
3834
3835     if (my $audio = conf_has_audio($conf)) {
3836
3837         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3838
3839         my $id = $audio->{dev_id};
3840         if ($audio->{dev} eq 'AC97') {
3841             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3842         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3843             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3844             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3845             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3846         } else {
3847             die "unkown audio device '$audio->{dev}', implement me!";
3848         }
3849
3850         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3851     }
3852
3853     my $sockets = 1;
3854     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3855     $sockets = $conf->{sockets} if  $conf->{sockets};
3856
3857     my $cores = $conf->{cores} || 1;
3858
3859     my $maxcpus = $sockets * $cores;
3860
3861     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3862
3863     my $allowed_vcpus = $cpuinfo->{cpus};
3864
3865     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3866         if ($allowed_vcpus < $maxcpus);
3867
3868     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3869
3870         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3871         for (my $i = 2; $i <= $vcpus; $i++)  {
3872             my $cpustr = print_cpu_device($conf,$i);
3873             push @$cmd, '-device', $cpustr;
3874         }
3875
3876     } else {
3877
3878         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3879     }
3880     push @$cmd, '-nodefaults';
3881
3882     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3883
3884     my $bootindex_hash = {};
3885     my $i = 1;
3886     foreach my $o (split(//, $bootorder)) {
3887         $bootindex_hash->{$o} = $i*100;
3888         $i++;
3889     }
3890
3891     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3892
3893     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3894
3895     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3896
3897     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3898         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3899         my $socket = vnc_socket($vmid);
3900         push @$cmd,  '-vnc', "unix:$socket,password";
3901     } else {
3902         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3903         push @$cmd, '-nographic';
3904     }
3905
3906     # time drift fix
3907     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3908
3909     my $useLocaltime = $conf->{localtime};
3910
3911     if ($winversion >= 5) { # windows
3912         $useLocaltime = 1 if !defined($conf->{localtime});
3913
3914         # use time drift fix when acpi is enabled
3915         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3916             $tdf = 1 if !defined($conf->{tdf});
3917         }
3918     }
3919
3920     if ($winversion >= 6) {
3921         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3922         push @$cmd, '-no-hpet';
3923     }
3924
3925     push @$rtcFlags, 'driftfix=slew' if $tdf;
3926
3927     if (!$kvm) {
3928         push @$machineFlags, 'accel=tcg';
3929     }
3930
3931     if ($machine_type) {
3932         push @$machineFlags, "type=${machine_type}";
3933     }
3934
3935     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3936         push @$rtcFlags, "base=$conf->{startdate}";
3937     } elsif ($useLocaltime) {
3938         push @$rtcFlags, 'base=localtime';
3939     }
3940
3941     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3942
3943     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3944
3945     push @$cmd, '-S' if $conf->{freeze};
3946
3947     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3948
3949     if (parse_guest_agent($conf)->{enabled}) {
3950         my $qgasocket = qmp_socket($vmid, 1);
3951         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3952         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3953         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3954         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3955     }
3956
3957     my $spice_port;
3958
3959     if ($qxlnum) {
3960         if ($qxlnum > 1) {
3961             if ($winversion){
3962                 for(my $i = 1; $i < $qxlnum; $i++){
3963                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3964                 }
3965             } else {
3966                 # assume other OS works like Linux
3967                 my ($ram, $vram) = ("134217728", "67108864");
3968                 if ($vga->{memory}) {
3969                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3970                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3971                 }
3972                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3973                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3974             }
3975         }
3976
3977         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3978
3979         my $nodename = PVE::INotify::nodename();
3980         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3981         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3982         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3983         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3984         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3985
3986         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3987
3988         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3989         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3990         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3991     }
3992
3993     # enable balloon by default, unless explicitly disabled
3994     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3995         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3996         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3997     }
3998
3999     if ($conf->{watchdog}) {
4000         my $wdopts = parse_watchdog($conf->{watchdog});
4001         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4002         my $watchdog = $wdopts->{model} || 'i6300esb';
4003         push @$devices, '-device', "$watchdog$pciaddr";
4004         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4005     }
4006
4007     my $vollist = [];
4008     my $scsicontroller = {};
4009     my $ahcicontroller = {};
4010     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4011
4012     # Add iscsi initiator name if available
4013     if (my $initiator = get_initiator_name()) {
4014         push @$devices, '-iscsi', "initiator-name=$initiator";
4015     }
4016
4017     foreach_drive($conf, sub {
4018         my ($ds, $drive) = @_;
4019
4020         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4021             push @$vollist, $drive->{file};
4022         }
4023
4024         # ignore efidisk here, already added in bios/fw handling code above
4025         return if $drive->{interface} eq 'efidisk';
4026
4027         $use_virtio = 1 if $ds =~ m/^virtio/;
4028
4029         if (drive_is_cdrom ($drive)) {
4030             if ($bootindex_hash->{d}) {
4031                 $drive->{bootindex} = $bootindex_hash->{d};
4032                 $bootindex_hash->{d} += 1;
4033             }
4034         } else {
4035             if ($bootindex_hash->{c}) {
4036                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4037                 $bootindex_hash->{c} += 1;
4038             }
4039         }
4040
4041         if($drive->{interface} eq 'virtio'){
4042            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4043         }
4044
4045         if ($drive->{interface} eq 'scsi') {
4046
4047             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4048
4049             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4050             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4051
4052             my $iothread = '';
4053             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4054                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4055                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4056             } elsif ($drive->{iothread}) {
4057                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4058             }
4059
4060             my $queues = '';
4061             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4062                 $queues = ",num_queues=$drive->{queues}";
4063             }
4064
4065             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4066             $scsicontroller->{$controller}=1;
4067         }
4068
4069         if ($drive->{interface} eq 'sata') {
4070            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4071            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4072            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4073            $ahcicontroller->{$controller}=1;
4074         }
4075
4076         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4077         push @$devices, '-drive',$drive_cmd;
4078         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4079     });
4080
4081     for (my $i = 0; $i < $MAX_NETS; $i++) {
4082          next if !$conf->{"net$i"};
4083          my $d = parse_net($conf->{"net$i"});
4084          next if !$d;
4085
4086          $use_virtio = 1 if $d->{model} eq 'virtio';
4087
4088          if ($bootindex_hash->{n}) {
4089             $d->{bootindex} = $bootindex_hash->{n};
4090             $bootindex_hash->{n} += 1;
4091          }
4092
4093          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4094          push @$devices, '-netdev', $netdevfull;
4095
4096          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4097          push @$devices, '-device', $netdevicefull;
4098     }
4099
4100     if ($conf->{ivshmem}) {
4101         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4102
4103         my $bus;
4104         if ($q35) {
4105             $bus = print_pcie_addr("ivshmem");
4106         } else {
4107             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4108         }
4109
4110         my $ivshmem_name = $ivshmem->{name} // $vmid;
4111         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4112
4113         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4114         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4115     }
4116
4117     if (!$q35) {
4118         # add pci bridges
4119         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4120            $bridges->{1} = 1;
4121            $bridges->{2} = 1;
4122         }
4123
4124         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4125
4126         while (my ($k, $v) = each %$bridges) {
4127             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4128             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4129         }
4130     }
4131
4132     push @$cmd, @$devices;
4133     push @$cmd, '-rtc', join(',', @$rtcFlags)
4134         if scalar(@$rtcFlags);
4135     push @$cmd, '-machine', join(',', @$machineFlags)
4136         if scalar(@$machineFlags);
4137     push @$cmd, '-global', join(',', @$globalFlags)
4138         if scalar(@$globalFlags);
4139
4140     if (my $vmstate = $conf->{vmstate}) {
4141         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4142         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4143         push @$cmd, '-loadstate', $statepath;
4144     }
4145
4146     # add custom args
4147     if ($conf->{args}) {
4148         my $aa = PVE::Tools::split_args($conf->{args});
4149         push @$cmd, @$aa;
4150     }
4151
4152     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4153 }
4154
4155 sub vnc_socket {
4156     my ($vmid) = @_;
4157     return "${var_run_tmpdir}/$vmid.vnc";
4158 }
4159
4160 sub spice_port {
4161     my ($vmid) = @_;
4162
4163     my $res = vm_mon_cmd($vmid, 'query-spice');
4164
4165     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4166 }
4167
4168 sub qmp_socket {
4169     my ($vmid, $qga, $name) = @_;
4170     my $sockettype = $qga ? 'qga' : 'qmp';
4171     my $ext = $name ? '-'.$name : '';
4172     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4173 }
4174
4175 sub pidfile_name {
4176     my ($vmid) = @_;
4177     return "${var_run_tmpdir}/$vmid.pid";
4178 }
4179
4180 sub vm_devices_list {
4181     my ($vmid) = @_;
4182
4183     my $res = vm_mon_cmd($vmid, 'query-pci');
4184     my $devices_to_check = [];
4185     my $devices = {};
4186     foreach my $pcibus (@$res) {
4187         push @$devices_to_check, @{$pcibus->{devices}},
4188     }
4189
4190     while (@$devices_to_check) {
4191         my $to_check = [];
4192         for my $d (@$devices_to_check) {
4193             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4194             next if !$d->{'pci_bridge'};
4195
4196             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4197             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4198         }
4199         $devices_to_check = $to_check;
4200     }
4201
4202     my $resblock = vm_mon_cmd($vmid, 'query-block');
4203     foreach my $block (@$resblock) {
4204         if($block->{device} =~ m/^drive-(\S+)/){
4205                 $devices->{$1} = 1;
4206         }
4207     }
4208
4209     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4210     foreach my $mice (@$resmice) {
4211         if ($mice->{name} eq 'QEMU HID Tablet') {
4212             $devices->{tablet} = 1;
4213             last;
4214         }
4215     }
4216
4217     # for usb devices there is no query-usb
4218     # but we can iterate over the entries in
4219     # qom-list path=/machine/peripheral
4220     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4221     foreach my $per (@$resperipheral) {
4222         if ($per->{name} =~ m/^usb\d+$/) {
4223             $devices->{$per->{name}} = 1;
4224         }
4225     }
4226
4227     return $devices;
4228 }
4229
4230 sub vm_deviceplug {
4231     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4232
4233     my $q35 = machine_type_is_q35($conf);
4234
4235     my $devices_list = vm_devices_list($vmid);
4236     return 1 if defined($devices_list->{$deviceid});
4237
4238     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4239
4240     if ($deviceid eq 'tablet') {
4241
4242         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4243
4244     } elsif ($deviceid eq 'keyboard') {
4245
4246         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4247
4248     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4249
4250         die "usb hotplug currently not reliable\n";
4251         # since we can't reliably hot unplug all added usb devices
4252         # and usb passthrough disables live migration
4253         # we disable usb hotplugging for now
4254         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4255
4256     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4257
4258         qemu_iothread_add($vmid, $deviceid, $device);
4259
4260         qemu_driveadd($storecfg, $vmid, $device);
4261         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4262
4263         qemu_deviceadd($vmid, $devicefull);
4264         eval { qemu_deviceaddverify($vmid, $deviceid); };
4265         if (my $err = $@) {
4266             eval { qemu_drivedel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270
4271     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4272
4273
4274         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4275         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4276         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4277
4278         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4279
4280         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4281             qemu_iothread_add($vmid, $deviceid, $device);
4282             $devicefull .= ",iothread=iothread-$deviceid";
4283         }
4284
4285         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4286             $devicefull .= ",num_queues=$device->{queues}";
4287         }
4288
4289         qemu_deviceadd($vmid, $devicefull);
4290         qemu_deviceaddverify($vmid, $deviceid);
4291
4292     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4293
4294         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4295         qemu_driveadd($storecfg, $vmid, $device);
4296
4297         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4298         eval { qemu_deviceadd($vmid, $devicefull); };
4299         if (my $err = $@) {
4300             eval { qemu_drivedel($vmid, $deviceid); };
4301             warn $@ if $@;
4302             die $err;
4303         }
4304
4305     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4306
4307         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4308
4309         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4310         my $use_old_bios_files = undef;
4311         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4312
4313         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4314         qemu_deviceadd($vmid, $netdevicefull);
4315         eval {
4316             qemu_deviceaddverify($vmid, $deviceid);
4317             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4318         };
4319         if (my $err = $@) {
4320             eval { qemu_netdevdel($vmid, $deviceid); };
4321             warn $@ if $@;
4322             die $err;
4323         }
4324
4325     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4326
4327         my $bridgeid = $2;
4328         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4329         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4330
4331         qemu_deviceadd($vmid, $devicefull);
4332         qemu_deviceaddverify($vmid, $deviceid);
4333
4334     } else {
4335         die "can't hotplug device '$deviceid'\n";
4336     }
4337
4338     return 1;
4339 }
4340
4341 # fixme: this should raise exceptions on error!
4342 sub vm_deviceunplug {
4343     my ($vmid, $conf, $deviceid) = @_;
4344
4345     my $devices_list = vm_devices_list($vmid);
4346     return 1 if !defined($devices_list->{$deviceid});
4347
4348     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4349
4350     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4351
4352         qemu_devicedel($vmid, $deviceid);
4353
4354     } elsif ($deviceid =~ m/^usb\d+$/) {
4355
4356         die "usb hotplug currently not reliable\n";
4357         # when unplugging usb devices this way,
4358         # there may be remaining usb controllers/hubs
4359         # so we disable it for now
4360         qemu_devicedel($vmid, $deviceid);
4361         qemu_devicedelverify($vmid, $deviceid);
4362
4363     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4364
4365         qemu_devicedel($vmid, $deviceid);
4366         qemu_devicedelverify($vmid, $deviceid);
4367         qemu_drivedel($vmid, $deviceid);
4368         qemu_iothread_del($conf, $vmid, $deviceid);
4369
4370     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4371
4372         qemu_devicedel($vmid, $deviceid);
4373         qemu_devicedelverify($vmid, $deviceid);
4374         qemu_iothread_del($conf, $vmid, $deviceid);
4375
4376     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4377
4378         qemu_devicedel($vmid, $deviceid);
4379         qemu_drivedel($vmid, $deviceid);
4380         qemu_deletescsihw($conf, $vmid, $deviceid);
4381
4382     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4383
4384         qemu_devicedel($vmid, $deviceid);
4385         qemu_devicedelverify($vmid, $deviceid);
4386         qemu_netdevdel($vmid, $deviceid);
4387
4388     } else {
4389         die "can't unplug device '$deviceid'\n";
4390     }
4391
4392     return 1;
4393 }
4394
4395 sub qemu_deviceadd {
4396     my ($vmid, $devicefull) = @_;
4397
4398     $devicefull = "driver=".$devicefull;
4399     my %options =  split(/[=,]/, $devicefull);
4400
4401     vm_mon_cmd($vmid, "device_add" , %options);
4402 }
4403
4404 sub qemu_devicedel {
4405     my ($vmid, $deviceid) = @_;
4406
4407     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4408 }
4409
4410 sub qemu_iothread_add {
4411     my($vmid, $deviceid, $device) = @_;
4412
4413     if ($device->{iothread}) {
4414         my $iothreads = vm_iothreads_list($vmid);
4415         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4416     }
4417 }
4418
4419 sub qemu_iothread_del {
4420     my($conf, $vmid, $deviceid) = @_;
4421
4422     my $confid = $deviceid;
4423     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4424         $confid = 'scsi' . $1;
4425     }
4426     my $device = parse_drive($confid, $conf->{$confid});
4427     if ($device->{iothread}) {
4428         my $iothreads = vm_iothreads_list($vmid);
4429         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4430     }
4431 }
4432
4433 sub qemu_objectadd {
4434     my($vmid, $objectid, $qomtype) = @_;
4435
4436     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4437
4438     return 1;
4439 }
4440
4441 sub qemu_objectdel {
4442     my($vmid, $objectid) = @_;
4443
4444     vm_mon_cmd($vmid, "object-del", id => $objectid);
4445
4446     return 1;
4447 }
4448
4449 sub qemu_driveadd {
4450     my ($storecfg, $vmid, $device) = @_;
4451
4452     my $drive = print_drive_full($storecfg, $vmid, $device);
4453     $drive =~ s/\\/\\\\/g;
4454     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4455
4456     # If the command succeeds qemu prints: "OK"
4457     return 1 if $ret =~ m/OK/s;
4458
4459     die "adding drive failed: $ret\n";
4460 }
4461
4462 sub qemu_drivedel {
4463     my($vmid, $deviceid) = @_;
4464
4465     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4466     $ret =~ s/^\s+//;
4467
4468     return 1 if $ret eq "";
4469
4470     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4471     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4472
4473     die "deleting drive $deviceid failed : $ret\n";
4474 }
4475
4476 sub qemu_deviceaddverify {
4477     my ($vmid, $deviceid) = @_;
4478
4479     for (my $i = 0; $i <= 5; $i++) {
4480          my $devices_list = vm_devices_list($vmid);
4481          return 1 if defined($devices_list->{$deviceid});
4482          sleep 1;
4483     }
4484
4485     die "error on hotplug device '$deviceid'\n";
4486 }
4487
4488
4489 sub qemu_devicedelverify {
4490     my ($vmid, $deviceid) = @_;
4491
4492     # need to verify that the device is correctly removed as device_del
4493     # is async and empty return is not reliable
4494
4495     for (my $i = 0; $i <= 5; $i++) {
4496          my $devices_list = vm_devices_list($vmid);
4497          return 1 if !defined($devices_list->{$deviceid});
4498          sleep 1;
4499     }
4500
4501     die "error on hot-unplugging device '$deviceid'\n";
4502 }
4503
4504 sub qemu_findorcreatescsihw {
4505     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4506
4507     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4508
4509     my $scsihwid="$controller_prefix$controller";
4510     my $devices_list = vm_devices_list($vmid);
4511
4512     if(!defined($devices_list->{$scsihwid})) {
4513         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4514     }
4515
4516     return 1;
4517 }
4518
4519 sub qemu_deletescsihw {
4520     my ($conf, $vmid, $opt) = @_;
4521
4522     my $device = parse_drive($opt, $conf->{$opt});
4523
4524     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4525         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4526         return 1;
4527     }
4528
4529     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4530
4531     my $devices_list = vm_devices_list($vmid);
4532     foreach my $opt (keys %{$devices_list}) {
4533         if (PVE::QemuServer::is_valid_drivename($opt)) {
4534             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4535             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4536                 return 1;
4537             }
4538         }
4539     }
4540
4541     my $scsihwid="scsihw$controller";
4542
4543     vm_deviceunplug($vmid, $conf, $scsihwid);
4544
4545     return 1;
4546 }
4547
4548 sub qemu_add_pci_bridge {
4549     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4550
4551     my $bridges = {};
4552
4553     my $bridgeid;
4554
4555     print_pci_addr($device, $bridges, $arch, $machine_type);
4556
4557     while (my ($k, $v) = each %$bridges) {
4558         $bridgeid = $k;
4559     }
4560     return 1 if !defined($bridgeid) || $bridgeid < 1;
4561
4562     my $bridge = "pci.$bridgeid";
4563     my $devices_list = vm_devices_list($vmid);
4564
4565     if (!defined($devices_list->{$bridge})) {
4566         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4567     }
4568
4569     return 1;
4570 }
4571
4572 sub qemu_set_link_status {
4573     my ($vmid, $device, $up) = @_;
4574
4575     vm_mon_cmd($vmid, "set_link", name => $device,
4576                up => $up ? JSON::true : JSON::false);
4577 }
4578
4579 sub qemu_netdevadd {
4580     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4581
4582     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4583     my %options =  split(/[=,]/, $netdev);
4584
4585     vm_mon_cmd($vmid, "netdev_add",  %options);
4586     return 1;
4587 }
4588
4589 sub qemu_netdevdel {
4590     my ($vmid, $deviceid) = @_;
4591
4592     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4593 }
4594
4595 sub qemu_usb_hotplug {
4596     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4597
4598     return if !$device;
4599
4600     # remove the old one first
4601     vm_deviceunplug($vmid, $conf, $deviceid);
4602
4603     # check if xhci controller is necessary and available
4604     if ($device->{usb3}) {
4605
4606         my $devicelist = vm_devices_list($vmid);
4607
4608         if (!$devicelist->{xhci}) {
4609             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4610             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4611         }
4612     }
4613     my $d = parse_usb_device($device->{host});
4614     $d->{usb3} = $device->{usb3};
4615
4616     # add the new one
4617     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4618 }
4619
4620 sub qemu_cpu_hotplug {
4621     my ($vmid, $conf, $vcpus) = @_;
4622
4623     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4624
4625     my $sockets = 1;
4626     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4627     $sockets = $conf->{sockets} if  $conf->{sockets};
4628     my $cores = $conf->{cores} || 1;
4629     my $maxcpus = $sockets * $cores;
4630
4631     $vcpus = $maxcpus if !$vcpus;
4632
4633     die "you can't add more vcpus than maxcpus\n"
4634         if $vcpus > $maxcpus;
4635
4636     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4637
4638     if ($vcpus < $currentvcpus) {
4639
4640         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4641
4642             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4643                 qemu_devicedel($vmid, "cpu$i");
4644                 my $retry = 0;
4645                 my $currentrunningvcpus = undef;
4646                 while (1) {
4647                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4648                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4649                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4650                     $retry++;
4651                     sleep 1;
4652                 }
4653                 #update conf after each succesfull cpu unplug
4654                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4655                 PVE::QemuConfig->write_config($vmid, $conf);
4656             }
4657         } else {
4658             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4659         }
4660
4661         return;
4662     }
4663
4664     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4665     die "vcpus in running vm does not match its configuration\n"
4666         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4667
4668     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4669
4670         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4671             my $cpustr = print_cpu_device($conf, $i);
4672             qemu_deviceadd($vmid, $cpustr);
4673
4674             my $retry = 0;
4675             my $currentrunningvcpus = undef;
4676             while (1) {
4677                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4678                 last if scalar(@{$currentrunningvcpus}) == $i;
4679                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4680                 sleep 1;
4681                 $retry++;
4682             }
4683             #update conf after each succesfull cpu hotplug
4684             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4685             PVE::QemuConfig->write_config($vmid, $conf);
4686         }
4687     } else {
4688
4689         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4690             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4691         }
4692     }
4693 }
4694
4695 sub qemu_block_set_io_throttle {
4696     my ($vmid, $deviceid,
4697         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4698         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4699         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4700         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4701
4702     return if !check_running($vmid) ;
4703
4704     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4705         bps => int($bps),
4706         bps_rd => int($bps_rd),
4707         bps_wr => int($bps_wr),
4708         iops => int($iops),
4709         iops_rd => int($iops_rd),
4710         iops_wr => int($iops_wr),
4711         bps_max => int($bps_max),
4712         bps_rd_max => int($bps_rd_max),
4713         bps_wr_max => int($bps_wr_max),
4714         iops_max => int($iops_max),
4715         iops_rd_max => int($iops_rd_max),
4716         iops_wr_max => int($iops_wr_max),
4717         bps_max_length => int($bps_max_length),
4718         bps_rd_max_length => int($bps_rd_max_length),
4719         bps_wr_max_length => int($bps_wr_max_length),
4720         iops_max_length => int($iops_max_length),
4721         iops_rd_max_length => int($iops_rd_max_length),
4722         iops_wr_max_length => int($iops_wr_max_length),
4723     );
4724
4725 }
4726
4727 # old code, only used to shutdown old VM after update
4728 sub __read_avail {
4729     my ($fh, $timeout) = @_;
4730
4731     my $sel = new IO::Select;
4732     $sel->add($fh);
4733
4734     my $res = '';
4735     my $buf;
4736
4737     my @ready;
4738     while (scalar (@ready = $sel->can_read($timeout))) {
4739         my $count;
4740         if ($count = $fh->sysread($buf, 8192)) {
4741             if ($buf =~ /^(.*)\(qemu\) $/s) {
4742                 $res .= $1;
4743                 last;
4744             } else {
4745                 $res .= $buf;
4746             }
4747         } else {
4748             if (!defined($count)) {
4749                 die "$!\n";
4750             }
4751             last;
4752         }
4753     }
4754
4755     die "monitor read timeout\n" if !scalar(@ready);
4756
4757     return $res;
4758 }
4759
4760 sub qemu_block_resize {
4761     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4762
4763     my $running = check_running($vmid);
4764
4765     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4766
4767     return if !$running;
4768
4769     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4770
4771 }
4772
4773 sub qemu_volume_snapshot {
4774     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4775
4776     my $running = check_running($vmid);
4777
4778     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4779         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4780     } else {
4781         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4782     }
4783 }
4784
4785 sub qemu_volume_snapshot_delete {
4786     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4787
4788     my $running = check_running($vmid);
4789
4790     if($running) {
4791
4792         $running = undef;
4793         my $conf = PVE::QemuConfig->load_config($vmid);
4794         foreach_drive($conf, sub {
4795             my ($ds, $drive) = @_;
4796             $running = 1 if $drive->{file} eq $volid;
4797         });
4798     }
4799
4800     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4801         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4802     } else {
4803         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4804     }
4805 }
4806
4807 sub set_migration_caps {
4808     my ($vmid) = @_;
4809
4810     my $cap_ref = [];
4811
4812     my $enabled_cap = {
4813         "auto-converge" => 1,
4814         "xbzrle" => 1,
4815         "x-rdma-pin-all" => 0,
4816         "zero-blocks" => 0,
4817         "compress" => 0
4818     };
4819
4820     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4821
4822     for my $supported_capability (@$supported_capabilities) {
4823         push @$cap_ref, {
4824             capability => $supported_capability->{capability},
4825             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4826         };
4827     }
4828
4829     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4830 }
4831
4832 my $fast_plug_option = {
4833     'lock' => 1,
4834     'name' => 1,
4835     'onboot' => 1,
4836     'shares' => 1,
4837     'startup' => 1,
4838     'description' => 1,
4839     'protection' => 1,
4840     'vmstatestorage' => 1,
4841     'hookscript' => 1,
4842 };
4843
4844 # hotplug changes in [PENDING]
4845 # $selection hash can be used to only apply specified options, for
4846 # example: { cores => 1 } (only apply changed 'cores')
4847 # $errors ref is used to return error messages
4848 sub vmconfig_hotplug_pending {
4849     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4850
4851     my $defaults = load_defaults();
4852     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4853
4854     # commit values which do not have any impact on running VM first
4855     # Note: those option cannot raise errors, we we do not care about
4856     # $selection and always apply them.
4857
4858     my $add_error = sub {
4859         my ($opt, $msg) = @_;
4860         $errors->{$opt} = "hotplug problem - $msg";
4861     };
4862
4863     my $changes = 0;
4864     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4865         if ($fast_plug_option->{$opt}) {
4866             $conf->{$opt} = $conf->{pending}->{$opt};
4867             delete $conf->{pending}->{$opt};
4868             $changes = 1;
4869         }
4870     }
4871
4872     if ($changes) {
4873         PVE::QemuConfig->write_config($vmid, $conf);
4874         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4875     }
4876
4877     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4878
4879     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4880     while (my ($opt, $force) = each %$pending_delete_hash) {
4881         next if $selection && !$selection->{$opt};
4882         eval {
4883             if ($opt eq 'hotplug') {
4884                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4885             } elsif ($opt eq 'tablet') {
4886                 die "skip\n" if !$hotplug_features->{usb};
4887                 if ($defaults->{tablet}) {
4888                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4889                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4890                         if $arch eq 'aarch64';
4891                 } else {
4892                     vm_deviceunplug($vmid, $conf, 'tablet');
4893                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4894                 }
4895             } elsif ($opt =~ m/^usb\d+/) {
4896                 die "skip\n";
4897                 # since we cannot reliably hot unplug usb devices
4898                 # we are disabling it
4899                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4900                 vm_deviceunplug($vmid, $conf, $opt);
4901             } elsif ($opt eq 'vcpus') {
4902                 die "skip\n" if !$hotplug_features->{cpu};
4903                 qemu_cpu_hotplug($vmid, $conf, undef);
4904             } elsif ($opt eq 'balloon') {
4905                 # enable balloon device is not hotpluggable
4906                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4907                 # here we reset the ballooning value to memory
4908                 my $balloon = $conf->{memory} || $defaults->{memory};
4909                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4910             } elsif ($fast_plug_option->{$opt}) {
4911                 # do nothing
4912             } elsif ($opt =~ m/^net(\d+)$/) {
4913                 die "skip\n" if !$hotplug_features->{network};
4914                 vm_deviceunplug($vmid, $conf, $opt);
4915             } elsif (is_valid_drivename($opt)) {
4916                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4917                 vm_deviceunplug($vmid, $conf, $opt);
4918                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4919             } elsif ($opt =~ m/^memory$/) {
4920                 die "skip\n" if !$hotplug_features->{memory};
4921                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4922             } elsif ($opt eq 'cpuunits') {
4923                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4924             } elsif ($opt eq 'cpulimit') {
4925                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4926             } else {
4927                 die "skip\n";
4928             }
4929         };
4930         if (my $err = $@) {
4931             &$add_error($opt, $err) if $err ne "skip\n";
4932         } else {
4933             # save new config if hotplug was successful
4934             delete $conf->{$opt};
4935             vmconfig_undelete_pending_option($conf, $opt);
4936             PVE::QemuConfig->write_config($vmid, $conf);
4937             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4938         }
4939     }
4940
4941     my $apply_pending_cloudinit;
4942     $apply_pending_cloudinit = sub {
4943         my ($key, $value) = @_;
4944         $apply_pending_cloudinit = sub {}; # once is enough
4945
4946         my @cloudinit_opts = keys %$confdesc_cloudinit;
4947         foreach my $opt (keys %{$conf->{pending}}) {
4948             next if !grep { $_ eq $opt } @cloudinit_opts;
4949             $conf->{$opt} = delete $conf->{pending}->{$opt};
4950         }
4951
4952         my $new_conf = { %$conf };
4953         $new_conf->{$key} = $value;
4954         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4955     };
4956
4957     foreach my $opt (keys %{$conf->{pending}}) {
4958         next if $selection && !$selection->{$opt};
4959         my $value = $conf->{pending}->{$opt};
4960         eval {
4961             if ($opt eq 'hotplug') {
4962                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4963             } elsif ($opt eq 'tablet') {
4964                 die "skip\n" if !$hotplug_features->{usb};
4965                 if ($value == 1) {
4966                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4967                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4968                         if $arch eq 'aarch64';
4969                 } elsif ($value == 0) {
4970                     vm_deviceunplug($vmid, $conf, 'tablet');
4971                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4972                 }
4973             } elsif ($opt =~ m/^usb\d+$/) {
4974                 die "skip\n";
4975                 # since we cannot reliably hot unplug usb devices
4976                 # we are disabling it
4977                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4978                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4979                 die "skip\n" if !$d;
4980                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4981             } elsif ($opt eq 'vcpus') {
4982                 die "skip\n" if !$hotplug_features->{cpu};
4983                 qemu_cpu_hotplug($vmid, $conf, $value);
4984             } elsif ($opt eq 'balloon') {
4985                 # enable/disable balloning device is not hotpluggable
4986                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4987                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4988                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4989
4990                 # allow manual ballooning if shares is set to zero
4991                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4992                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4993                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4994                 }
4995             } elsif ($opt =~ m/^net(\d+)$/) {
4996                 # some changes can be done without hotplug
4997                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4998                                     $vmid, $opt, $value, $arch, $machine_type);
4999             } elsif (is_valid_drivename($opt)) {
5000                 # some changes can be done without hotplug
5001                 my $drive = parse_drive($opt, $value);
5002                 if (drive_is_cloudinit($drive)) {
5003                     &$apply_pending_cloudinit($opt, $value);
5004                 }
5005                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5006                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5007             } elsif ($opt =~ m/^memory$/) { #dimms
5008                 die "skip\n" if !$hotplug_features->{memory};
5009                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5010             } elsif ($opt eq 'cpuunits') {
5011                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5012             } elsif ($opt eq 'cpulimit') {
5013                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5014                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5015             } else {
5016                 die "skip\n";  # skip non-hot-pluggable options
5017             }
5018         };
5019         if (my $err = $@) {
5020             &$add_error($opt, $err) if $err ne "skip\n";
5021         } else {
5022             # save new config if hotplug was successful
5023             $conf->{$opt} = $value;
5024             delete $conf->{pending}->{$opt};
5025             PVE::QemuConfig->write_config($vmid, $conf);
5026             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5027         }
5028     }
5029 }
5030
5031 sub try_deallocate_drive {
5032     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5033
5034     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5035         my $volid = $drive->{file};
5036         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5037             my $sid = PVE::Storage::parse_volume_id($volid);
5038             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5039
5040             # check if the disk is really unused
5041             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5042                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5043             PVE::Storage::vdisk_free($storecfg, $volid);
5044             return 1;
5045         } else {
5046             # If vm is not owner of this disk remove from config
5047             return 1;
5048         }
5049     }
5050
5051     return undef;
5052 }
5053
5054 sub vmconfig_delete_or_detach_drive {
5055     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5056
5057     my $drive = parse_drive($opt, $conf->{$opt});
5058
5059     my $rpcenv = PVE::RPCEnvironment::get();
5060     my $authuser = $rpcenv->get_user();
5061
5062     if ($force) {
5063         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5064         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5065     } else {
5066         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5067     }
5068 }
5069
5070 sub vmconfig_apply_pending {
5071     my ($vmid, $conf, $storecfg) = @_;
5072
5073     # cold plug
5074
5075     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5076     while (my ($opt, $force) = each %$pending_delete_hash) {
5077         die "internal error" if $opt =~ m/^unused/;
5078         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5079         if (!defined($conf->{$opt})) {
5080             vmconfig_undelete_pending_option($conf, $opt);
5081             PVE::QemuConfig->write_config($vmid, $conf);
5082         } elsif (is_valid_drivename($opt)) {
5083             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5084             vmconfig_undelete_pending_option($conf, $opt);
5085             delete $conf->{$opt};
5086             PVE::QemuConfig->write_config($vmid, $conf);
5087         } else {
5088             vmconfig_undelete_pending_option($conf, $opt);
5089             delete $conf->{$opt};
5090             PVE::QemuConfig->write_config($vmid, $conf);
5091         }
5092     }
5093
5094     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5095
5096     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5097         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5098
5099         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5100             # skip if nothing changed
5101         } elsif (is_valid_drivename($opt)) {
5102             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5103                 if defined($conf->{$opt});
5104             $conf->{$opt} = $conf->{pending}->{$opt};
5105         } else {
5106             $conf->{$opt} = $conf->{pending}->{$opt};
5107         }
5108
5109         delete $conf->{pending}->{$opt};
5110         PVE::QemuConfig->write_config($vmid, $conf);
5111     }
5112 }
5113
5114 my $safe_num_ne = sub {
5115     my ($a, $b) = @_;
5116
5117     return 0 if !defined($a) && !defined($b);
5118     return 1 if !defined($a);
5119     return 1 if !defined($b);
5120
5121     return $a != $b;
5122 };
5123
5124 my $safe_string_ne = sub {
5125     my ($a, $b) = @_;
5126
5127     return 0 if !defined($a) && !defined($b);
5128     return 1 if !defined($a);
5129     return 1 if !defined($b);
5130
5131     return $a ne $b;
5132 };
5133
5134 sub vmconfig_update_net {
5135     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5136
5137     my $newnet = parse_net($value);
5138
5139     if ($conf->{$opt}) {
5140         my $oldnet = parse_net($conf->{$opt});
5141
5142         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5143             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5144             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5145             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5146
5147             # for non online change, we try to hot-unplug
5148             die "skip\n" if !$hotplug;
5149             vm_deviceunplug($vmid, $conf, $opt);
5150         } else {
5151
5152             die "internal error" if $opt !~ m/net(\d+)/;
5153             my $iface = "tap${vmid}i$1";
5154
5155             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5156                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5157                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5158                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5159                 PVE::Network::tap_unplug($iface);
5160                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5161             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5162                 # Rate can be applied on its own but any change above needs to
5163                 # include the rate in tap_plug since OVS resets everything.
5164                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5165             }
5166
5167             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5168                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5169             }
5170
5171             return 1;
5172         }
5173     }
5174
5175     if ($hotplug) {
5176         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5177     } else {
5178         die "skip\n";
5179     }
5180 }
5181
5182 sub vmconfig_update_disk {
5183     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5184
5185     # fixme: do we need force?
5186
5187     my $drive = parse_drive($opt, $value);
5188
5189     if ($conf->{$opt}) {
5190
5191         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5192
5193             my $media = $drive->{media} || 'disk';
5194             my $oldmedia = $old_drive->{media} || 'disk';
5195             die "unable to change media type\n" if $media ne $oldmedia;
5196
5197             if (!drive_is_cdrom($old_drive)) {
5198
5199                 if ($drive->{file} ne $old_drive->{file}) {
5200
5201                     die "skip\n" if !$hotplug;
5202
5203                     # unplug and register as unused
5204                     vm_deviceunplug($vmid, $conf, $opt);
5205                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5206
5207                 } else {
5208                     # update existing disk
5209
5210                     # skip non hotpluggable value
5211                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5212                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5213                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5214                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5215                         die "skip\n";
5216                     }
5217
5218                     # apply throttle
5219                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5220                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5221                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5222                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5223                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5224                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5225                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5226                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5227                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5228                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5229                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5230                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5231                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5232                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5233                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5234                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5235                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5236                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5237
5238                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5239                                                    ($drive->{mbps} || 0)*1024*1024,
5240                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5241                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5242                                                    $drive->{iops} || 0,
5243                                                    $drive->{iops_rd} || 0,
5244                                                    $drive->{iops_wr} || 0,
5245                                                    ($drive->{mbps_max} || 0)*1024*1024,
5246                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5247                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5248                                                    $drive->{iops_max} || 0,
5249                                                    $drive->{iops_rd_max} || 0,
5250                                                    $drive->{iops_wr_max} || 0,
5251                                                    $drive->{bps_max_length} || 1,
5252                                                    $drive->{bps_rd_max_length} || 1,
5253                                                    $drive->{bps_wr_max_length} || 1,
5254                                                    $drive->{iops_max_length} || 1,
5255                                                    $drive->{iops_rd_max_length} || 1,
5256                                                    $drive->{iops_wr_max_length} || 1);
5257
5258                     }
5259
5260                     return 1;
5261                 }
5262
5263             } else { # cdrom
5264
5265                 if ($drive->{file} eq 'none') {
5266                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5267                     if (drive_is_cloudinit($old_drive)) {
5268                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5269                     }
5270                 } else {
5271                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5272                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5273                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5274                 }
5275
5276                 return 1;
5277             }
5278         }
5279     }
5280
5281     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5282     # hotplug new disks
5283     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5284     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5285 }
5286
5287 sub vm_start {
5288     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5289         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5290
5291     PVE::QemuConfig->lock_config($vmid, sub {
5292         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5293
5294         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5295
5296         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5297
5298         PVE::QemuConfig->check_lock($conf)
5299             if !($skiplock || $is_suspended);
5300
5301         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5302
5303         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5304             vmconfig_apply_pending($vmid, $conf, $storecfg);
5305             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5306         }
5307
5308         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5309
5310         my $defaults = load_defaults();
5311
5312         # set environment variable useful inside network script
5313         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5314
5315         my $local_volumes = {};
5316
5317         if ($targetstorage) {
5318             foreach_drive($conf, sub {
5319                 my ($ds, $drive) = @_;
5320
5321                 return if drive_is_cdrom($drive);
5322
5323                 my $volid = $drive->{file};
5324
5325                 return if !$volid;
5326
5327                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5328
5329                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5330                 return if $scfg->{shared};
5331                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5332             });
5333
5334             my $format = undef;
5335
5336             foreach my $opt (sort keys %$local_volumes) {
5337
5338                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5339                 my $drive = parse_drive($opt, $conf->{$opt});
5340
5341                 #if remote storage is specified, use default format
5342                 if ($targetstorage && $targetstorage ne "1") {
5343                     $storeid = $targetstorage;
5344                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5345                     $format = $defFormat;
5346                 } else {
5347                     #else we use same format than original
5348                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5349                     $format = qemu_img_format($scfg, $volid);
5350                 }
5351
5352                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5353                 my $newdrive = $drive;
5354                 $newdrive->{format} = $format;
5355                 $newdrive->{file} = $newvolid;
5356                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5357                 $local_volumes->{$opt} = $drivestr;
5358                 #pass drive to conf for command line
5359                 $conf->{$opt} = $drivestr;
5360             }
5361         }
5362
5363         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5364
5365         if ($is_suspended) {
5366             # enforce machine type on suspended vm to ensure HW compatibility
5367             $forcemachine = $conf->{runningmachine};
5368             print "Resuming suspended VM\n";
5369         }
5370
5371         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5372
5373         my $migrate_port = 0;
5374         my $migrate_uri;
5375         if ($statefile) {
5376             if ($statefile eq 'tcp') {
5377                 my $localip = "localhost";
5378                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5379                 my $nodename = PVE::INotify::nodename();
5380
5381                 if (!defined($migration_type)) {
5382                     if (defined($datacenterconf->{migration}->{type})) {
5383                         $migration_type = $datacenterconf->{migration}->{type};
5384                     } else {
5385                         $migration_type = 'secure';
5386                     }
5387                 }
5388
5389                 if ($migration_type eq 'insecure') {
5390                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5391                     if ($migrate_network_addr) {
5392                         $localip = $migrate_network_addr;
5393                     } else {
5394                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5395                     }
5396
5397                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5398                 }
5399
5400                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5401                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5402                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5403                 push @$cmd, '-incoming', $migrate_uri;
5404                 push @$cmd, '-S';
5405
5406             } elsif ($statefile eq 'unix') {
5407                 # should be default for secure migrations as a ssh TCP forward
5408                 # tunnel is not deterministic reliable ready and fails regurarly
5409                 # to set up in time, so use UNIX socket forwards
5410                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5411                 unlink $socket_addr;
5412
5413                 $migrate_uri = "unix:$socket_addr";
5414
5415                 push @$cmd, '-incoming', $migrate_uri;
5416                 push @$cmd, '-S';
5417
5418             } else {
5419                 push @$cmd, '-loadstate', $statefile;
5420             }
5421         } elsif ($paused) {
5422             push @$cmd, '-S';
5423         }
5424
5425         # host pci devices
5426         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5427           my $d = parse_hostpci($conf->{"hostpci$i"});
5428           next if !$d;
5429           my $pcidevices = $d->{pciid};
5430           foreach my $pcidevice (@$pcidevices) {
5431                 my $pciid = $pcidevice->{id};
5432
5433                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5434                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5435                 die "no pci device info for device '$pciid'\n" if !$info;
5436
5437                 if ($d->{mdev}) {
5438                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5439                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5440                 } else {
5441                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5442                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5443                     die "can't reset pci device '$pciid'\n"
5444                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5445                 }
5446           }
5447         }
5448
5449         PVE::Storage::activate_volumes($storecfg, $vollist);
5450
5451         eval {
5452             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5453                 outfunc => sub {}, errfunc => sub {});
5454         };
5455         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5456         # timeout should be more than enough here...
5457         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5458
5459         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5460                                                   : $defaults->{cpuunits};
5461
5462         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5463         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5464
5465         my %properties = (
5466             Slice => 'qemu.slice',
5467             KillMode => 'none',
5468             CPUShares => $cpuunits
5469         );
5470
5471         if (my $cpulimit = $conf->{cpulimit}) {
5472             $properties{CPUQuota} = int($cpulimit * 100);
5473         }
5474         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5475
5476         my $run_qemu = sub {
5477             PVE::Tools::run_fork sub {
5478                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5479                 run_command($cmd, %run_params);
5480             };
5481         };
5482
5483         if ($conf->{hugepages}) {
5484
5485             my $code = sub {
5486                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5487                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5488
5489                 PVE::QemuServer::Memory::hugepages_mount();
5490                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5491
5492                 eval { $run_qemu->() };
5493                 if (my $err = $@) {
5494                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5495                     die $err;
5496                 }
5497
5498                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5499             };
5500             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5501
5502         } else {
5503             eval { $run_qemu->() };
5504         }
5505
5506         if (my $err = $@) {
5507             # deactivate volumes if start fails
5508             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5509             die "start failed: $err";
5510         }
5511
5512         print "migration listens on $migrate_uri\n" if $migrate_uri;
5513
5514         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5515             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5516             warn $@ if $@;
5517         }
5518
5519         #start nbd server for storage migration
5520         if ($targetstorage) {
5521             my $nodename = PVE::INotify::nodename();
5522             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5523             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5524             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5525             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5526
5527             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5528
5529             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5530
5531             foreach my $opt (sort keys %$local_volumes) {
5532                 my $volid = $local_volumes->{$opt};
5533                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5534                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5535                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5536             }
5537         }
5538
5539         if ($migratedfrom) {
5540             eval {
5541                 set_migration_caps($vmid);
5542             };
5543             warn $@ if $@;
5544
5545             if ($spice_port) {
5546                 print "spice listens on port $spice_port\n";
5547                 if ($spice_ticket) {
5548                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5549                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5550                 }
5551             }
5552
5553         } else {
5554             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5555                 if !$statefile && $conf->{balloon};
5556
5557             foreach my $opt (keys %$conf) {
5558                 next if $opt !~  m/^net\d+$/;
5559                 my $nicconf = parse_net($conf->{$opt});
5560                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5561             }
5562         }
5563
5564         vm_mon_cmd_nocheck($vmid, 'qom-set',
5565                     path => "machine/peripheral/balloon0",
5566                     property => "guest-stats-polling-interval",
5567                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5568
5569         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5570             print "Resumed VM, removing state\n";
5571             delete $conf->@{qw(lock vmstate runningmachine)};
5572             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5573             PVE::Storage::vdisk_free($storecfg, $vmstate);
5574             PVE::QemuConfig->write_config($vmid, $conf);
5575         }
5576
5577         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5578     });
5579 }
5580
5581 sub vm_mon_cmd {
5582     my ($vmid, $execute, %params) = @_;
5583
5584     my $cmd = { execute => $execute, arguments => \%params };
5585     vm_qmp_command($vmid, $cmd);
5586 }
5587
5588 sub vm_mon_cmd_nocheck {
5589     my ($vmid, $execute, %params) = @_;
5590
5591     my $cmd = { execute => $execute, arguments => \%params };
5592     vm_qmp_command($vmid, $cmd, 1);
5593 }
5594
5595 sub vm_qmp_command {
5596     my ($vmid, $cmd, $nocheck) = @_;
5597
5598     my $res;
5599
5600     my $timeout;
5601     if ($cmd->{arguments}) {
5602         $timeout = delete $cmd->{arguments}->{timeout};
5603     }
5604
5605     eval {
5606         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5607         my $sname = qmp_socket($vmid);
5608         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5609             my $qmpclient = PVE::QMPClient->new();
5610
5611             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5612         } else {
5613             die "unable to open monitor socket\n";
5614         }
5615     };
5616     if (my $err = $@) {
5617         syslog("err", "VM $vmid qmp command failed - $err");
5618         die $err;
5619     }
5620
5621     return $res;
5622 }
5623
5624 sub vm_human_monitor_command {
5625     my ($vmid, $cmdline) = @_;
5626
5627     my $cmd = {
5628         execute => 'human-monitor-command',
5629         arguments => { 'command-line' => $cmdline},
5630     };
5631
5632     return vm_qmp_command($vmid, $cmd);
5633 }
5634
5635 sub vm_commandline {
5636     my ($storecfg, $vmid, $snapname) = @_;
5637
5638     my $conf = PVE::QemuConfig->load_config($vmid);
5639
5640     if ($snapname) {
5641         my $snapshot = $conf->{snapshots}->{$snapname};
5642         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5643
5644         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5645
5646         $conf = $snapshot;
5647     }
5648
5649     my $defaults = load_defaults();
5650
5651     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5652
5653     return PVE::Tools::cmd2string($cmd);
5654 }
5655
5656 sub vm_reset {
5657     my ($vmid, $skiplock) = @_;
5658
5659     PVE::QemuConfig->lock_config($vmid, sub {
5660
5661         my $conf = PVE::QemuConfig->load_config($vmid);
5662
5663         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5664
5665         vm_mon_cmd($vmid, "system_reset");
5666     });
5667 }
5668
5669 sub get_vm_volumes {
5670     my ($conf) = @_;
5671
5672     my $vollist = [];
5673     foreach_volid($conf, sub {
5674         my ($volid, $attr) = @_;
5675
5676         return if $volid =~ m|^/|;
5677
5678         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5679         return if !$sid;
5680
5681         push @$vollist, $volid;
5682     });
5683
5684     return $vollist;
5685 }
5686
5687 sub vm_stop_cleanup {
5688     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5689
5690     eval {
5691
5692         if (!$keepActive) {
5693             my $vollist = get_vm_volumes($conf);
5694             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5695         }
5696
5697         foreach my $ext (qw(mon qmp pid vnc qga)) {
5698             unlink "/var/run/qemu-server/${vmid}.$ext";
5699         }
5700
5701         if ($conf->{ivshmem}) {
5702             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5703             # just delete it for now, VMs which have this already open do not
5704             # are affected, but new VMs will get a separated one. If this
5705             # becomes an issue we either add some sort of ref-counting or just
5706             # add a "don't delete on stop" flag to the ivshmem format.
5707             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5708         }
5709
5710         foreach my $key (keys %$conf) {
5711             next if $key !~ m/^hostpci(\d+)$/;
5712             my $hostpciindex = $1;
5713             my $d = parse_hostpci($conf->{$key});
5714             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5715
5716             foreach my $pci (@{$d->{pciid}}) {
5717                 my $pciid = $pci->{id};
5718                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5719             }
5720         }
5721
5722         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5723     };
5724     warn $@ if $@; # avoid errors - just warn
5725 }
5726
5727 # Note: use $nockeck to skip tests if VM configuration file exists.
5728 # We need that when migration VMs to other nodes (files already moved)
5729 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5730 sub vm_stop {
5731     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5732
5733     $force = 1 if !defined($force) && !$shutdown;
5734
5735     if ($migratedfrom){
5736         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5737         kill 15, $pid if $pid;
5738         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5739         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5740         return;
5741     }
5742
5743     PVE::QemuConfig->lock_config($vmid, sub {
5744
5745         my $pid = check_running($vmid, $nocheck);
5746         return if !$pid;
5747
5748         my $conf;
5749         if (!$nocheck) {
5750             $conf = PVE::QemuConfig->load_config($vmid);
5751             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5752             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5753                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5754                 $timeout = $opts->{down} if $opts->{down};
5755             }
5756             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5757         }
5758
5759         eval {
5760             if ($shutdown) {
5761                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5762                     vm_qmp_command($vmid, {
5763                         execute => "guest-shutdown",
5764                         arguments => { timeout => $timeout }
5765                     }, $nocheck);
5766                 } else {
5767                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5768                 }
5769             } else {
5770                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5771             }
5772         };
5773         my $err = $@;
5774
5775         if (!$err) {
5776             $timeout = 60 if !defined($timeout);
5777
5778             my $count = 0;
5779             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5780                 $count++;
5781                 sleep 1;
5782             }
5783
5784             if ($count >= $timeout) {
5785                 if ($force) {
5786                     warn "VM still running - terminating now with SIGTERM\n";
5787                     kill 15, $pid;
5788                 } else {
5789                     die "VM quit/powerdown failed - got timeout\n";
5790                 }
5791             } else {
5792                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5793                 return;
5794             }
5795         } else {
5796             if ($force) {
5797                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5798                 kill 15, $pid;
5799             } else {
5800                 die "VM quit/powerdown failed\n";
5801             }
5802         }
5803
5804         # wait again
5805         $timeout = 10;
5806
5807         my $count = 0;
5808         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5809             $count++;
5810             sleep 1;
5811         }
5812
5813         if ($count >= $timeout) {
5814             warn "VM still running - terminating now with SIGKILL\n";
5815             kill 9, $pid;
5816             sleep 1;
5817         }
5818
5819         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5820    });
5821 }
5822
5823 sub vm_suspend {
5824     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5825
5826     my $conf;
5827     my $path;
5828     my $storecfg;
5829     my $vmstate;
5830
5831     PVE::QemuConfig->lock_config($vmid, sub {
5832
5833         $conf = PVE::QemuConfig->load_config($vmid);
5834
5835         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5836         PVE::QemuConfig->check_lock($conf)
5837             if !($skiplock || $is_backing_up);
5838
5839         die "cannot suspend to disk during backup\n"
5840             if $is_backing_up && $includestate;
5841
5842         if ($includestate) {
5843             $conf->{lock} = 'suspending';
5844             my $date = strftime("%Y-%m-%d", localtime(time()));
5845             $storecfg = PVE::Storage::config();
5846             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5847             $path = PVE::Storage::path($storecfg, $vmstate);
5848             PVE::QemuConfig->write_config($vmid, $conf);
5849         } else {
5850             vm_mon_cmd($vmid, "stop");
5851         }
5852     });
5853
5854     if ($includestate) {
5855         # save vm state
5856         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5857
5858         eval {
5859             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5860             for(;;) {
5861                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5862                 if (!$state->{status}) {
5863                     die "savevm not active\n";
5864                 } elsif ($state->{status} eq 'active') {
5865                     sleep(1);
5866                     next;
5867                 } elsif ($state->{status} eq 'completed') {
5868                     print "State saved, quitting\n";
5869                     last;
5870                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5871                     die "query-savevm failed with error '$state->{error}'\n"
5872                 } else {
5873                     die "query-savevm returned status '$state->{status}'\n";
5874                 }
5875             }
5876         };
5877         my $err = $@;
5878
5879         PVE::QemuConfig->lock_config($vmid, sub {
5880             $conf = PVE::QemuConfig->load_config($vmid);
5881             if ($err) {
5882                 # cleanup, but leave suspending lock, to indicate something went wrong
5883                 eval {
5884                     vm_mon_cmd($vmid, "savevm-end");
5885                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5886                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5887                     delete $conf->@{qw(vmstate runningmachine)};
5888                     PVE::QemuConfig->write_config($vmid, $conf);
5889                 };
5890                 warn $@ if $@;
5891                 die $err;
5892             }
5893
5894             die "lock changed unexpectedly\n"
5895                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5896
5897             vm_qmp_command($vmid, { execute => "quit" });
5898             $conf->{lock} = 'suspended';
5899             PVE::QemuConfig->write_config($vmid, $conf);
5900         });
5901     }
5902 }
5903
5904 sub vm_resume {
5905     my ($vmid, $skiplock, $nocheck) = @_;
5906
5907     PVE::QemuConfig->lock_config($vmid, sub {
5908         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5909         my $res = $vm_mon_cmd->($vmid, 'query-status');
5910         my $resume_cmd = 'cont';
5911
5912         if ($res->{status} && $res->{status} eq 'suspended') {
5913             $resume_cmd = 'system_wakeup';
5914         }
5915
5916         if (!$nocheck) {
5917
5918             my $conf = PVE::QemuConfig->load_config($vmid);
5919
5920             PVE::QemuConfig->check_lock($conf)
5921                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5922         }
5923
5924         $vm_mon_cmd->($vmid, $resume_cmd);
5925     });
5926 }
5927
5928 sub vm_sendkey {
5929     my ($vmid, $skiplock, $key) = @_;
5930
5931     PVE::QemuConfig->lock_config($vmid, sub {
5932
5933         my $conf = PVE::QemuConfig->load_config($vmid);
5934
5935         # there is no qmp command, so we use the human monitor command
5936         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5937         die $res if $res ne '';
5938     });
5939 }
5940
5941 sub vm_destroy {
5942     my ($storecfg, $vmid, $skiplock) = @_;
5943
5944     PVE::QemuConfig->lock_config($vmid, sub {
5945
5946         my $conf = PVE::QemuConfig->load_config($vmid);
5947
5948         if (!check_running($vmid)) {
5949             destroy_vm($storecfg, $vmid, undef, $skiplock);
5950         } else {
5951             die "VM $vmid is running - destroy failed\n";
5952         }
5953     });
5954 }
5955
5956 # vzdump restore implementaion
5957
5958 sub tar_archive_read_firstfile {
5959     my $archive = shift;
5960
5961     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5962
5963     # try to detect archive type first
5964     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5965         die "unable to open file '$archive'\n";
5966     my $firstfile = <$fh>;
5967     kill 15, $pid;
5968     close $fh;
5969
5970     die "ERROR: archive contaions no data\n" if !$firstfile;
5971     chomp $firstfile;
5972
5973     return $firstfile;
5974 }
5975
5976 sub tar_restore_cleanup {
5977     my ($storecfg, $statfile) = @_;
5978
5979     print STDERR "starting cleanup\n";
5980
5981     if (my $fd = IO::File->new($statfile, "r")) {
5982         while (defined(my $line = <$fd>)) {
5983             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5984                 my $volid = $2;
5985                 eval {
5986                     if ($volid =~ m|^/|) {
5987                         unlink $volid || die 'unlink failed\n';
5988                     } else {
5989                         PVE::Storage::vdisk_free($storecfg, $volid);
5990                     }
5991                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5992                 };
5993                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5994             } else {
5995                 print STDERR "unable to parse line in statfile - $line";
5996             }
5997         }
5998         $fd->close();
5999     }
6000 }
6001
6002 sub restore_archive {
6003     my ($archive, $vmid, $user, $opts) = @_;
6004
6005     my $format = $opts->{format};
6006     my $comp;
6007
6008     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6009         $format = 'tar' if !$format;
6010         $comp = 'gzip';
6011     } elsif ($archive =~ m/\.tar$/) {
6012         $format = 'tar' if !$format;
6013     } elsif ($archive =~ m/.tar.lzo$/) {
6014         $format = 'tar' if !$format;
6015         $comp = 'lzop';
6016     } elsif ($archive =~ m/\.vma$/) {
6017         $format = 'vma' if !$format;
6018     } elsif ($archive =~ m/\.vma\.gz$/) {
6019         $format = 'vma' if !$format;
6020         $comp = 'gzip';
6021     } elsif ($archive =~ m/\.vma\.lzo$/) {
6022         $format = 'vma' if !$format;
6023         $comp = 'lzop';
6024     } else {
6025         $format = 'vma' if !$format; # default
6026     }
6027
6028     # try to detect archive format
6029     if ($format eq 'tar') {
6030         return restore_tar_archive($archive, $vmid, $user, $opts);
6031     } else {
6032         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6033     }
6034 }
6035
6036 sub restore_update_config_line {
6037     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6038
6039     return if $line =~ m/^\#qmdump\#/;
6040     return if $line =~ m/^\#vzdump\#/;
6041     return if $line =~ m/^lock:/;
6042     return if $line =~ m/^unused\d+:/;
6043     return if $line =~ m/^parent:/;
6044
6045     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6046     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6047         # try to convert old 1.X settings
6048         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6049         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6050             my ($model, $macaddr) = split(/\=/, $devconfig);
6051             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6052             my $net = {
6053                 model => $model,
6054                 bridge => "vmbr$ind",
6055                 macaddr => $macaddr,
6056             };
6057             my $netstr = print_net($net);
6058
6059             print $outfd "net$cookie->{netcount}: $netstr\n";
6060             $cookie->{netcount}++;
6061         }
6062     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6063         my ($id, $netstr) = ($1, $2);
6064         my $net = parse_net($netstr);
6065         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6066         $netstr = print_net($net);
6067         print $outfd "$id: $netstr\n";
6068     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6069         my $virtdev = $1;
6070         my $value = $3;
6071         my $di = parse_drive($virtdev, $value);
6072         if (defined($di->{backup}) && !$di->{backup}) {
6073             print $outfd "#$line";
6074         } elsif ($map->{$virtdev}) {
6075             delete $di->{format}; # format can change on restore
6076             $di->{file} = $map->{$virtdev};
6077             $value = print_drive($vmid, $di);
6078             print $outfd "$virtdev: $value\n";
6079         } else {
6080             print $outfd $line;
6081         }
6082     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6083         my $vmgenid = $1;
6084         if ($vmgenid ne '0') {
6085             # always generate a new vmgenid if there was a valid one setup
6086             $vmgenid = generate_uuid();
6087         }
6088         print $outfd "vmgenid: $vmgenid\n";
6089     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6090         my ($uuid, $uuid_str);
6091         UUID::generate($uuid);
6092         UUID::unparse($uuid, $uuid_str);
6093         my $smbios1 = parse_smbios1($2);
6094         $smbios1->{uuid} = $uuid_str;
6095         print $outfd $1.print_smbios1($smbios1)."\n";
6096     } else {
6097         print $outfd $line;
6098     }
6099 }
6100
6101 sub scan_volids {
6102     my ($cfg, $vmid) = @_;
6103
6104     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6105
6106     my $volid_hash = {};
6107     foreach my $storeid (keys %$info) {
6108         foreach my $item (@{$info->{$storeid}}) {
6109             next if !($item->{volid} && $item->{size});
6110             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6111             $volid_hash->{$item->{volid}} = $item;
6112         }
6113     }
6114
6115     return $volid_hash;
6116 }
6117
6118 sub is_volume_in_use {
6119     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6120
6121     my $path = PVE::Storage::path($storecfg, $volid);
6122
6123     my $scan_config = sub {
6124         my ($cref, $snapname) = @_;
6125
6126         foreach my $key (keys %$cref) {
6127             my $value = $cref->{$key};
6128             if (is_valid_drivename($key)) {
6129                 next if $skip_drive && $key eq $skip_drive;
6130                 my $drive = parse_drive($key, $value);
6131                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6132                 return 1 if $volid eq $drive->{file};
6133                 if ($drive->{file} =~ m!^/!) {
6134                     return 1 if $drive->{file} eq $path;
6135                 } else {
6136                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6137                     next if !$storeid;
6138                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6139                     next if !$scfg;
6140                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6141                 }
6142             }
6143         }
6144
6145         return 0;
6146     };
6147
6148     return 1 if &$scan_config($conf);
6149
6150     undef $skip_drive;
6151
6152     foreach my $snapname (keys %{$conf->{snapshots}}) {
6153         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6154     }
6155
6156     return 0;
6157 }
6158
6159 sub update_disksize {
6160     my ($vmid, $conf, $volid_hash) = @_;
6161
6162     my $changes;
6163     my $prefix = "VM $vmid:";
6164
6165     # used and unused disks
6166     my $referenced = {};
6167
6168     # Note: it is allowed to define multiple storages with same path (alias), so
6169     # we need to check both 'volid' and real 'path' (two different volid can point
6170     # to the same path).
6171
6172     my $referencedpath = {};
6173
6174     # update size info
6175     foreach my $opt (keys %$conf) {
6176         if (is_valid_drivename($opt)) {
6177             my $drive = parse_drive($opt, $conf->{$opt});
6178             my $volid = $drive->{file};
6179             next if !$volid;
6180
6181             $referenced->{$volid} = 1;
6182             if ($volid_hash->{$volid} &&
6183                 (my $path = $volid_hash->{$volid}->{path})) {
6184                 $referencedpath->{$path} = 1;
6185             }
6186
6187             next if drive_is_cdrom($drive);
6188             next if !$volid_hash->{$volid};
6189
6190             $drive->{size} = $volid_hash->{$volid}->{size};
6191             my $new = print_drive($vmid, $drive);
6192             if ($new ne $conf->{$opt}) {
6193                 $changes = 1;
6194                 $conf->{$opt} = $new;
6195                 print "$prefix update disk '$opt' information.\n";
6196             }
6197         }
6198     }
6199
6200     # remove 'unusedX' entry if volume is used
6201     foreach my $opt (keys %$conf) {
6202         next if $opt !~ m/^unused\d+$/;
6203         my $volid = $conf->{$opt};
6204         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6205         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6206             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6207             $changes = 1;
6208             delete $conf->{$opt};
6209         }
6210
6211         $referenced->{$volid} = 1;
6212         $referencedpath->{$path} = 1 if $path;
6213     }
6214
6215     foreach my $volid (sort keys %$volid_hash) {
6216         next if $volid =~ m/vm-$vmid-state-/;
6217         next if $referenced->{$volid};
6218         my $path = $volid_hash->{$volid}->{path};
6219         next if !$path; # just to be sure
6220         next if $referencedpath->{$path};
6221         $changes = 1;
6222         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6223         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6224         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6225     }
6226
6227     return $changes;
6228 }
6229
6230 sub rescan {
6231     my ($vmid, $nolock, $dryrun) = @_;
6232
6233     my $cfg = PVE::Storage::config();
6234
6235     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6236     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6237     foreach my $stor (keys %{$cfg->{ids}}) {
6238         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6239     }
6240
6241     print "rescan volumes...\n";
6242     my $volid_hash = scan_volids($cfg, $vmid);
6243
6244     my $updatefn =  sub {
6245         my ($vmid) = @_;
6246
6247         my $conf = PVE::QemuConfig->load_config($vmid);
6248
6249         PVE::QemuConfig->check_lock($conf);
6250
6251         my $vm_volids = {};
6252         foreach my $volid (keys %$volid_hash) {
6253             my $info = $volid_hash->{$volid};
6254             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6255         }
6256
6257         my $changes = update_disksize($vmid, $conf, $vm_volids);
6258
6259         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6260     };
6261
6262     if (defined($vmid)) {
6263         if ($nolock) {
6264             &$updatefn($vmid);
6265         } else {
6266             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6267         }
6268     } else {
6269         my $vmlist = config_list();
6270         foreach my $vmid (keys %$vmlist) {
6271             if ($nolock) {
6272                 &$updatefn($vmid);
6273             } else {
6274                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6275             }
6276         }
6277     }
6278 }
6279
6280 sub restore_vma_archive {
6281     my ($archive, $vmid, $user, $opts, $comp) = @_;
6282
6283     my $readfrom = $archive;
6284
6285     my $cfg = PVE::Storage::config();
6286     my $commands = [];
6287     my $bwlimit = $opts->{bwlimit};
6288
6289     my $dbg_cmdstring = '';
6290     my $add_pipe = sub {
6291         my ($cmd) = @_;
6292         push @$commands, $cmd;
6293         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6294         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6295         $readfrom = '-';
6296     };
6297
6298     my $input = undef;
6299     if ($archive eq '-') {
6300         $input = '<&STDIN';
6301     } else {
6302         # If we use a backup from a PVE defined storage we also consider that
6303         # storage's rate limit:
6304         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6305         if (defined($volid)) {
6306             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6307             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6308             if ($readlimit) {
6309                 print STDERR "applying read rate limit: $readlimit\n";
6310                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6311                 $add_pipe->($cstream);
6312             }
6313         }
6314     }
6315
6316     if ($comp) {
6317         my $cmd;
6318         if ($comp eq 'gzip') {
6319             $cmd = ['zcat', $readfrom];
6320         } elsif ($comp eq 'lzop') {
6321             $cmd = ['lzop', '-d', '-c', $readfrom];
6322         } else {
6323             die "unknown compression method '$comp'\n";
6324         }
6325         $add_pipe->($cmd);
6326     }
6327
6328     my $tmpdir = "/var/tmp/vzdumptmp$$";
6329     rmtree $tmpdir;
6330
6331     # disable interrupts (always do cleanups)
6332     local $SIG{INT} =
6333         local $SIG{TERM} =
6334         local $SIG{QUIT} =
6335         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6336
6337     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6338     POSIX::mkfifo($mapfifo, 0600);
6339     my $fifofh;
6340
6341     my $openfifo = sub {
6342         open($fifofh, '>', $mapfifo) || die $!;
6343     };
6344
6345     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6346
6347     my $oldtimeout;
6348     my $timeout = 5;
6349
6350     my $devinfo = {};
6351
6352     my $rpcenv = PVE::RPCEnvironment::get();
6353
6354     my $conffile = PVE::QemuConfig->config_file($vmid);
6355     my $tmpfn = "$conffile.$$.tmp";
6356
6357     # Note: $oldconf is undef if VM does not exists
6358     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6359     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6360
6361     my %storage_limits;
6362
6363     my $print_devmap = sub {
6364         my $virtdev_hash = {};
6365
6366         my $cfgfn = "$tmpdir/qemu-server.conf";
6367
6368         # we can read the config - that is already extracted
6369         my $fh = IO::File->new($cfgfn, "r") ||
6370             "unable to read qemu-server.conf - $!\n";
6371
6372         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6373         if (-f $fwcfgfn) {
6374             my $pve_firewall_dir = '/etc/pve/firewall';
6375             mkdir $pve_firewall_dir; # make sure the dir exists
6376             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6377         }
6378
6379         while (defined(my $line = <$fh>)) {
6380             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6381                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6382                 die "archive does not contain data for drive '$virtdev'\n"
6383                     if !$devinfo->{$devname};
6384                 if (defined($opts->{storage})) {
6385                     $storeid = $opts->{storage} || 'local';
6386                 } elsif (!$storeid) {
6387                     $storeid = 'local';
6388                 }
6389                 $format = 'raw' if !$format;
6390                 $devinfo->{$devname}->{devname} = $devname;
6391                 $devinfo->{$devname}->{virtdev} = $virtdev;
6392                 $devinfo->{$devname}->{format} = $format;
6393                 $devinfo->{$devname}->{storeid} = $storeid;
6394
6395                 # check permission on storage
6396                 my $pool = $opts->{pool}; # todo: do we need that?
6397                 if ($user ne 'root@pam') {
6398                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6399                 }
6400
6401                 $storage_limits{$storeid} = $bwlimit;
6402
6403                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6404             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6405                 my $virtdev = $1;
6406                 my $drive = parse_drive($virtdev, $2);
6407                 if (drive_is_cloudinit($drive)) {
6408                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6409                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6410                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6411
6412                     my $d = {
6413                         format => $format,
6414                         storeid => $opts->{storage} // $storeid,
6415                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6416                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6417                         name => "vm-$vmid-cloudinit",
6418                         is_cloudinit => 1,
6419                     };
6420                     $virtdev_hash->{$virtdev} = $d;
6421                 }
6422             }
6423         }
6424
6425         foreach my $key (keys %storage_limits) {
6426             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6427             next if !$limit;
6428             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6429             $storage_limits{$key} = $limit * 1024;
6430         }
6431
6432         foreach my $devname (keys %$devinfo) {
6433             die "found no device mapping information for device '$devname'\n"
6434                 if !$devinfo->{$devname}->{virtdev};
6435         }
6436
6437         # create empty/temp config
6438         if ($oldconf) {
6439             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6440             foreach_drive($oldconf, sub {
6441                 my ($ds, $drive) = @_;
6442
6443                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6444
6445                 my $volid = $drive->{file};
6446                 return if !$volid || $volid =~ m|^/|;
6447
6448                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6449                 return if !$path || !$owner || ($owner != $vmid);
6450
6451                 # Note: only delete disk we want to restore
6452                 # other volumes will become unused
6453                 if ($virtdev_hash->{$ds}) {
6454                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6455                     if (my $err = $@) {
6456                         warn $err;
6457                     }
6458                 }
6459             });
6460
6461             # delete vmstate files, after the restore we have no snapshots anymore
6462             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6463                 my $snap = $oldconf->{snapshots}->{$snapname};
6464                 if ($snap->{vmstate}) {
6465                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6466                     if (my $err = $@) {
6467                         warn $err;
6468                     }
6469                 }
6470             }
6471         }
6472
6473         my $map = {};
6474         foreach my $virtdev (sort keys %$virtdev_hash) {
6475             my $d = $virtdev_hash->{$virtdev};
6476             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6477             my $storeid = $d->{storeid};
6478             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6479
6480             my $map_opts = '';
6481             if (my $limit = $storage_limits{$storeid}) {
6482                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6483             }
6484
6485             # test if requested format is supported
6486             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6487             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6488             $d->{format} = $defFormat if !$supported;
6489
6490             my $name;
6491             if ($d->{is_cloudinit}) {
6492                 $name = $d->{name};
6493                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6494             }
6495
6496             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6497             print STDERR "new volume ID is '$volid'\n";
6498             $d->{volid} = $volid;
6499
6500             PVE::Storage::activate_volumes($cfg, [$volid]);
6501
6502             my $write_zeros = 1;
6503             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6504                 $write_zeros = 0;
6505             }
6506
6507             if (!$d->{is_cloudinit}) {
6508                 my $path = PVE::Storage::path($cfg, $volid);
6509
6510                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6511
6512                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6513             }
6514             $map->{$virtdev} = $volid;
6515         }
6516
6517         $fh->seek(0, 0) || die "seek failed - $!\n";
6518
6519         my $outfd = new IO::File ($tmpfn, "w") ||
6520             die "unable to write config for VM $vmid\n";
6521
6522         my $cookie = { netcount => 0 };
6523         while (defined(my $line = <$fh>)) {
6524             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6525         }
6526
6527         $fh->close();
6528         $outfd->close();
6529     };
6530
6531     eval {
6532         # enable interrupts
6533         local $SIG{INT} =
6534             local $SIG{TERM} =
6535             local $SIG{QUIT} =
6536             local $SIG{HUP} =
6537             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6538         local $SIG{ALRM} = sub { die "got timeout\n"; };
6539
6540         $oldtimeout = alarm($timeout);
6541
6542         my $parser = sub {
6543             my $line = shift;
6544
6545             print "$line\n";
6546
6547             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6548                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6549                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6550             } elsif ($line =~ m/^CTIME: /) {
6551                 # we correctly received the vma config, so we can disable
6552                 # the timeout now for disk allocation (set to 10 minutes, so
6553                 # that we always timeout if something goes wrong)
6554                 alarm(600);
6555                 &$print_devmap();
6556                 print $fifofh "done\n";
6557                 my $tmp = $oldtimeout || 0;
6558                 $oldtimeout = undef;
6559                 alarm($tmp);
6560                 close($fifofh);
6561             }
6562         };
6563
6564         print "restore vma archive: $dbg_cmdstring\n";
6565         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6566     };
6567     my $err = $@;
6568
6569     alarm($oldtimeout) if $oldtimeout;
6570
6571     my $vollist = [];
6572     foreach my $devname (keys %$devinfo) {
6573         my $volid = $devinfo->{$devname}->{volid};
6574         push @$vollist, $volid if $volid;
6575     }
6576
6577     PVE::Storage::deactivate_volumes($cfg, $vollist);
6578
6579     unlink $mapfifo;
6580
6581     if ($err) {
6582         rmtree $tmpdir;
6583         unlink $tmpfn;
6584
6585         foreach my $devname (keys %$devinfo) {
6586             my $volid = $devinfo->{$devname}->{volid};
6587             next if !$volid;
6588             eval {
6589                 if ($volid =~ m|^/|) {
6590                     unlink $volid || die 'unlink failed\n';
6591                 } else {
6592                     PVE::Storage::vdisk_free($cfg, $volid);
6593                 }
6594                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6595             };
6596             print STDERR "unable to cleanup '$volid' - $@" if $@;
6597         }
6598         die $err;
6599     }
6600
6601     rmtree $tmpdir;
6602
6603     rename($tmpfn, $conffile) ||
6604         die "unable to commit configuration file '$conffile'\n";
6605
6606     PVE::Cluster::cfs_update(); # make sure we read new file
6607
6608     eval { rescan($vmid, 1); };
6609     warn $@ if $@;
6610 }
6611
6612 sub restore_tar_archive {
6613     my ($archive, $vmid, $user, $opts) = @_;
6614
6615     if ($archive ne '-') {
6616         my $firstfile = tar_archive_read_firstfile($archive);
6617         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6618             if $firstfile ne 'qemu-server.conf';
6619     }
6620
6621     my $storecfg = PVE::Storage::config();
6622
6623     # destroy existing data - keep empty config
6624     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6625     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6626
6627     my $tocmd = "/usr/lib/qemu-server/qmextract";
6628
6629     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6630     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6631     $tocmd .= ' --prealloc' if $opts->{prealloc};
6632     $tocmd .= ' --info' if $opts->{info};
6633
6634     # tar option "xf" does not autodetect compression when read from STDIN,
6635     # so we pipe to zcat
6636     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6637         PVE::Tools::shellquote("--to-command=$tocmd");
6638
6639     my $tmpdir = "/var/tmp/vzdumptmp$$";
6640     mkpath $tmpdir;
6641
6642     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6643     local $ENV{VZDUMP_VMID} = $vmid;
6644     local $ENV{VZDUMP_USER} = $user;
6645
6646     my $conffile = PVE::QemuConfig->config_file($vmid);
6647     my $tmpfn = "$conffile.$$.tmp";
6648
6649     # disable interrupts (always do cleanups)
6650     local $SIG{INT} =
6651         local $SIG{TERM} =
6652         local $SIG{QUIT} =
6653         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6654
6655     eval {
6656         # enable interrupts
6657         local $SIG{INT} =
6658             local $SIG{TERM} =
6659             local $SIG{QUIT} =
6660             local $SIG{HUP} =
6661             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6662
6663         if ($archive eq '-') {
6664             print "extracting archive from STDIN\n";
6665             run_command($cmd, input => "<&STDIN");
6666         } else {
6667             print "extracting archive '$archive'\n";
6668             run_command($cmd);
6669         }
6670
6671         return if $opts->{info};
6672
6673         # read new mapping
6674         my $map = {};
6675         my $statfile = "$tmpdir/qmrestore.stat";
6676         if (my $fd = IO::File->new($statfile, "r")) {
6677             while (defined (my $line = <$fd>)) {
6678                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6679                     $map->{$1} = $2 if $1;
6680                 } else {
6681                     print STDERR "unable to parse line in statfile - $line\n";
6682                 }
6683             }
6684             $fd->close();
6685         }
6686
6687         my $confsrc = "$tmpdir/qemu-server.conf";
6688
6689         my $srcfd = new IO::File($confsrc, "r") ||
6690             die "unable to open file '$confsrc'\n";
6691
6692         my $outfd = new IO::File ($tmpfn, "w") ||
6693             die "unable to write config for VM $vmid\n";
6694
6695         my $cookie = { netcount => 0 };
6696         while (defined (my $line = <$srcfd>)) {
6697             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6698         }
6699
6700         $srcfd->close();
6701         $outfd->close();
6702     };
6703     my $err = $@;
6704
6705     if ($err) {
6706
6707         unlink $tmpfn;
6708
6709         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6710
6711         die $err;
6712     }
6713
6714     rmtree $tmpdir;
6715
6716     rename $tmpfn, $conffile ||
6717         die "unable to commit configuration file '$conffile'\n";
6718
6719     PVE::Cluster::cfs_update(); # make sure we read new file
6720
6721     eval { rescan($vmid, 1); };
6722     warn $@ if $@;
6723 };
6724
6725 sub foreach_storage_used_by_vm {
6726     my ($conf, $func) = @_;
6727
6728     my $sidhash = {};
6729
6730     foreach_drive($conf, sub {
6731         my ($ds, $drive) = @_;
6732         return if drive_is_cdrom($drive);
6733
6734         my $volid = $drive->{file};
6735
6736         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6737         $sidhash->{$sid} = $sid if $sid;
6738     });
6739
6740     foreach my $sid (sort keys %$sidhash) {
6741         &$func($sid);
6742     }
6743 }
6744
6745 sub do_snapshots_with_qemu {
6746     my ($storecfg, $volid) = @_;
6747
6748     my $storage_name = PVE::Storage::parse_volume_id($volid);
6749     my $scfg = $storecfg->{ids}->{$storage_name};
6750
6751     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6752         return 1;
6753     }
6754
6755     if ($volid =~ m/\.(qcow2|qed)$/){
6756         return 1;
6757     }
6758
6759     return undef;
6760 }
6761
6762 sub qga_check_running {
6763     my ($vmid, $nowarn) = @_;
6764
6765     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6766     if ($@) {
6767         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6768         return 0;
6769     }
6770     return 1;
6771 }
6772
6773 sub template_create {
6774     my ($vmid, $conf, $disk) = @_;
6775
6776     my $storecfg = PVE::Storage::config();
6777
6778     foreach_drive($conf, sub {
6779         my ($ds, $drive) = @_;
6780
6781         return if drive_is_cdrom($drive);
6782         return if $disk && $ds ne $disk;
6783
6784         my $volid = $drive->{file};
6785         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6786
6787         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6788         $drive->{file} = $voliddst;
6789         $conf->{$ds} = print_drive($vmid, $drive);
6790         PVE::QemuConfig->write_config($vmid, $conf);
6791     });
6792 }
6793
6794 sub convert_iscsi_path {
6795     my ($path) = @_;
6796
6797     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6798         my $portal = $1;
6799         my $target = $2;
6800         my $lun = $3;
6801
6802         my $initiator_name = get_initiator_name();
6803
6804         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6805                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6806     }
6807
6808     die "cannot convert iscsi path '$path', unkown format\n";
6809 }
6810
6811 sub qemu_img_convert {
6812     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6813
6814     my $storecfg = PVE::Storage::config();
6815     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6816     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6817
6818     if ($src_storeid && $dst_storeid) {
6819
6820         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6821
6822         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6823         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6824
6825         my $src_format = qemu_img_format($src_scfg, $src_volname);
6826         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6827
6828         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6829         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6830
6831         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6832         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6833
6834         my $cmd = [];
6835         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6836         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6837         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6838         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6839
6840         if ($src_is_iscsi) {
6841             push @$cmd, '--image-opts';
6842             $src_path = convert_iscsi_path($src_path);
6843         } else {
6844             push @$cmd, '-f', $src_format;
6845         }
6846
6847         if ($dst_is_iscsi) {
6848             push @$cmd, '--target-image-opts';
6849             $dst_path = convert_iscsi_path($dst_path);
6850         } else {
6851             push @$cmd, '-O', $dst_format;
6852         }
6853
6854         push @$cmd, $src_path;
6855
6856         if (!$dst_is_iscsi && $is_zero_initialized) {
6857             push @$cmd, "zeroinit:$dst_path";
6858         } else {
6859             push @$cmd, $dst_path;
6860         }
6861
6862         my $parser = sub {
6863             my $line = shift;
6864             if($line =~ m/\((\S+)\/100\%\)/){
6865                 my $percent = $1;
6866                 my $transferred = int($size * $percent / 100);
6867                 my $remaining = $size - $transferred;
6868
6869                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6870             }
6871
6872         };
6873
6874         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6875         my $err = $@;
6876         die "copy failed: $err" if $err;
6877     }
6878 }
6879
6880 sub qemu_img_format {
6881     my ($scfg, $volname) = @_;
6882
6883     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6884         return $1;
6885     } else {
6886         return "raw";
6887     }
6888 }
6889
6890 sub qemu_drive_mirror {
6891     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6892
6893     $jobs = {} if !$jobs;
6894
6895     my $qemu_target;
6896     my $format;
6897     $jobs->{"drive-$drive"} = {};
6898
6899     if ($dst_volid =~ /^nbd:/) {
6900         $qemu_target = $dst_volid;
6901         $format = "nbd";
6902     } else {
6903         my $storecfg = PVE::Storage::config();
6904         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6905
6906         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6907
6908         $format = qemu_img_format($dst_scfg, $dst_volname);
6909
6910         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6911
6912         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6913     }
6914
6915     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6916     $opts->{format} = $format if $format;
6917
6918     if (defined($bwlimit)) {
6919         $opts->{speed} = $bwlimit * 1024;
6920         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6921     } else {
6922         print "drive mirror is starting for drive-$drive\n";
6923     }
6924
6925     # if a job already runs for this device we get an error, catch it for cleanup
6926     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6927     if (my $err = $@) {
6928         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6929         warn "$@\n" if $@;
6930         die "mirroring error: $err\n";
6931     }
6932
6933     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6934 }
6935
6936 sub qemu_drive_mirror_monitor {
6937     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6938
6939     eval {
6940         my $err_complete = 0;
6941
6942         while (1) {
6943             die "storage migration timed out\n" if $err_complete > 300;
6944
6945             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6946
6947             my $running_mirror_jobs = {};
6948             foreach my $stat (@$stats) {
6949                 next if $stat->{type} ne 'mirror';
6950                 $running_mirror_jobs->{$stat->{device}} = $stat;
6951             }
6952
6953             my $readycounter = 0;
6954
6955             foreach my $job (keys %$jobs) {
6956
6957                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6958                     print "$job : finished\n";
6959                     delete $jobs->{$job};
6960                     next;
6961                 }
6962
6963                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6964
6965                 my $busy = $running_mirror_jobs->{$job}->{busy};
6966                 my $ready = $running_mirror_jobs->{$job}->{ready};
6967                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6968                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6969                     my $remaining = $total - $transferred;
6970                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6971
6972                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6973                 }
6974
6975                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6976             }
6977
6978             last if scalar(keys %$jobs) == 0;
6979
6980             if ($readycounter == scalar(keys %$jobs)) {
6981                 print "all mirroring jobs are ready \n";
6982                 last if $skipcomplete; #do the complete later
6983
6984                 if ($vmiddst && $vmiddst != $vmid) {
6985                     my $agent_running = $qga && qga_check_running($vmid);
6986                     if ($agent_running) {
6987                         print "freeze filesystem\n";
6988                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6989                     } else {
6990                         print "suspend vm\n";
6991                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6992                     }
6993
6994                     # if we clone a disk for a new target vm, we don't switch the disk
6995                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6996
6997                     if ($agent_running) {
6998                         print "unfreeze filesystem\n";
6999                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7000                     } else {
7001                         print "resume vm\n";
7002                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7003                     }
7004
7005                     last;
7006                 } else {
7007
7008                     foreach my $job (keys %$jobs) {
7009                         # try to switch the disk if source and destination are on the same guest
7010                         print "$job: Completing block job...\n";
7011
7012                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7013                         if ($@ =~ m/cannot be completed/) {
7014                             print "$job: Block job cannot be completed, try again.\n";
7015                             $err_complete++;
7016                         }else {
7017                             print "$job: Completed successfully.\n";
7018                             $jobs->{$job}->{complete} = 1;
7019                         }
7020                     }
7021                 }
7022             }
7023             sleep 1;
7024         }
7025     };
7026     my $err = $@;
7027
7028     if ($err) {
7029         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7030         die "mirroring error: $err";
7031     }
7032
7033 }
7034
7035 sub qemu_blockjobs_cancel {
7036     my ($vmid, $jobs) = @_;
7037
7038     foreach my $job (keys %$jobs) {
7039         print "$job: Cancelling block job\n";
7040         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7041         $jobs->{$job}->{cancel} = 1;
7042     }
7043
7044     while (1) {
7045         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7046
7047         my $running_jobs = {};
7048         foreach my $stat (@$stats) {
7049             $running_jobs->{$stat->{device}} = $stat;
7050         }
7051
7052         foreach my $job (keys %$jobs) {
7053
7054             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7055                 print "$job: Done.\n";
7056                 delete $jobs->{$job};
7057             }
7058         }
7059
7060         last if scalar(keys %$jobs) == 0;
7061
7062         sleep 1;
7063     }
7064 }
7065
7066 sub clone_disk {
7067     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7068         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7069
7070     my $newvolid;
7071
7072     if (!$full) {
7073         print "create linked clone of drive $drivename ($drive->{file})\n";
7074         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7075         push @$newvollist, $newvolid;
7076     } else {
7077
7078         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7079         $storeid = $storage if $storage;
7080
7081         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7082         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7083
7084         print "create full clone of drive $drivename ($drive->{file})\n";
7085         my $name = undef;
7086         if (drive_is_cloudinit($drive)) {
7087             $name = "vm-$newvmid-cloudinit";
7088             $snapname = undef;
7089             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7090             if ($dst_format ne 'raw') {
7091                 $name .= ".$dst_format";
7092             }
7093         }
7094         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7095         push @$newvollist, $newvolid;
7096
7097         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7098
7099         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7100         if (!$running || $snapname) {
7101             # TODO: handle bwlimits
7102             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7103         } else {
7104
7105             my $kvmver = get_running_qemu_version ($vmid);
7106             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7107                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7108                     if $drive->{iothread};
7109             }
7110
7111             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7112         }
7113     }
7114
7115     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7116
7117     my $disk = $drive;
7118     $disk->{format} = undef;
7119     $disk->{file} = $newvolid;
7120     $disk->{size} = $size;
7121
7122     return $disk;
7123 }
7124
7125 # this only works if VM is running
7126 sub get_current_qemu_machine {
7127     my ($vmid) = @_;
7128
7129     my $cmd = { execute => 'query-machines', arguments => {} };
7130     my $res = vm_qmp_command($vmid, $cmd);
7131
7132     my ($current, $default);
7133     foreach my $e (@$res) {
7134         $default = $e->{name} if $e->{'is-default'};
7135         $current = $e->{name} if $e->{'is-current'};
7136     }
7137
7138     # fallback to the default machine if current is not supported by qemu
7139     return $current || $default || 'pc';
7140 }
7141
7142 sub get_running_qemu_version {
7143     my ($vmid) = @_;
7144     my $cmd = { execute => 'query-version', arguments => {} };
7145     my $res = vm_qmp_command($vmid, $cmd);
7146     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7147 }
7148
7149 sub qemu_machine_feature_enabled {
7150     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7151
7152     my $current_major;
7153     my $current_minor;
7154
7155     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7156
7157         $current_major = $3;
7158         $current_minor = $4;
7159
7160     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7161
7162         $current_major = $1;
7163         $current_minor = $2;
7164     }
7165
7166     return 1 if $current_major > $version_major ||
7167                 ($current_major == $version_major &&
7168                  $current_minor >= $version_minor);
7169 }
7170
7171 sub qemu_machine_pxe {
7172     my ($vmid, $conf, $machine) = @_;
7173
7174     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7175
7176     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7177         $machine .= '.pxe';
7178     }
7179
7180     return $machine;
7181 }
7182
7183 sub qemu_use_old_bios_files {
7184     my ($machine_type) = @_;
7185
7186     return if !$machine_type;
7187
7188     my $use_old_bios_files = undef;
7189
7190     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7191         $machine_type = $1;
7192         $use_old_bios_files = 1;
7193     } else {
7194         my $kvmver = kvm_user_version();
7195         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7196         # load new efi bios files on migration. So this hack is required to allow
7197         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7198         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7199         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7200     }
7201
7202     return ($use_old_bios_files, $machine_type);
7203 }
7204
7205 sub create_efidisk($$$$$) {
7206     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7207
7208     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7209     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7210
7211     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7212     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7213     PVE::Storage::activate_volumes($storecfg, [$volid]);
7214
7215     my $path = PVE::Storage::path($storecfg, $volid);
7216     eval {
7217         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7218     };
7219     die "Copying EFI vars image failed: $@" if $@;
7220
7221     return ($volid, $vars_size);
7222 }
7223
7224 sub vm_iothreads_list {
7225     my ($vmid) = @_;
7226
7227     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7228
7229     my $iothreads = {};
7230     foreach my $iothread (@$res) {
7231         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7232     }
7233
7234     return $iothreads;
7235 }
7236
7237 sub scsihw_infos {
7238     my ($conf, $drive) = @_;
7239
7240     my $maxdev = 0;
7241
7242     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7243         $maxdev = 7;
7244     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7245         $maxdev = 1;
7246     } else {
7247         $maxdev = 256;
7248     }
7249
7250     my $controller = int($drive->{index} / $maxdev);
7251     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7252
7253     return ($maxdev, $controller, $controller_prefix);
7254 }
7255
7256 sub add_hyperv_enlightenments {
7257     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7258
7259     return if $winversion < 6;
7260     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7261
7262     if ($gpu_passthrough || defined($hv_vendor_id)) {
7263         $hv_vendor_id //= 'proxmox';
7264         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7265     }
7266
7267     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7268         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7269         push @$cpuFlags , 'hv_vapic';
7270         push @$cpuFlags , 'hv_time';
7271     } else {
7272         push @$cpuFlags , 'hv_spinlocks=0xffff';
7273     }
7274
7275     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7276         push @$cpuFlags , 'hv_reset';
7277         push @$cpuFlags , 'hv_vpindex';
7278         push @$cpuFlags , 'hv_runtime';
7279     }
7280
7281     if ($winversion >= 7) {
7282         push @$cpuFlags , 'hv_relaxed';
7283
7284         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7285             push @$cpuFlags , 'hv_synic';
7286             push @$cpuFlags , 'hv_stimer';
7287         }
7288
7289         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7290             push @$cpuFlags , 'hv_ipi';
7291         }
7292     }
7293 }
7294
7295 sub windows_version {
7296     my ($ostype) = @_;
7297
7298     return 0 if !$ostype;
7299
7300     my $winversion = 0;
7301
7302     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7303         $winversion = 5;
7304     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7305         $winversion = 6;
7306     } elsif ($ostype =~ m/^win(\d+)$/) {
7307         $winversion = $1;
7308     }
7309
7310     return $winversion;
7311 }
7312
7313 sub resolve_dst_disk_format {
7314         my ($storecfg, $storeid, $src_volname, $format) = @_;
7315         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7316
7317         if (!$format) {
7318             # if no target format is specified, use the source disk format as hint
7319             if ($src_volname) {
7320                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7321                 $format = qemu_img_format($scfg, $src_volname);
7322             } else {
7323                 return $defFormat;
7324             }
7325         }
7326
7327         # test if requested format is supported - else use default
7328         my $supported = grep { $_ eq $format } @$validFormats;
7329         $format = $defFormat if !$supported;
7330         return $format;
7331 }
7332
7333 sub resolve_first_disk {
7334     my $conf = shift;
7335     my @disks = PVE::QemuServer::valid_drive_names();
7336     my $firstdisk;
7337     foreach my $ds (reverse @disks) {
7338         next if !$conf->{$ds};
7339         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7340         next if PVE::QemuServer::drive_is_cdrom($disk);
7341         $firstdisk = $ds;
7342     }
7343     return $firstdisk;
7344 }
7345
7346 sub generate_uuid {
7347     my ($uuid, $uuid_str);
7348     UUID::generate($uuid);
7349     UUID::unparse($uuid, $uuid_str);
7350     return $uuid_str;
7351 }
7352
7353 sub generate_smbios1_uuid {
7354     return "uuid=".generate_uuid();
7355 }
7356
7357 sub nbd_stop {
7358     my ($vmid) = @_;
7359
7360     vm_mon_cmd($vmid, 'nbd-server-stop');
7361 }
7362
7363 # bash completion helper
7364
7365 sub complete_backup_archives {
7366     my ($cmdname, $pname, $cvalue) = @_;
7367
7368     my $cfg = PVE::Storage::config();
7369
7370     my $storeid;
7371
7372     if ($cvalue =~ m/^([^:]+):/) {
7373         $storeid = $1;
7374     }
7375
7376     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7377
7378     my $res = [];
7379     foreach my $id (keys %$data) {
7380         foreach my $item (@{$data->{$id}}) {
7381             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7382             push @$res, $item->{volid} if defined($item->{volid});
7383         }
7384     }
7385
7386     return $res;
7387 }
7388
7389 my $complete_vmid_full = sub {
7390     my ($running) = @_;
7391
7392     my $idlist = vmstatus();
7393
7394     my $res = [];
7395
7396     foreach my $id (keys %$idlist) {
7397         my $d = $idlist->{$id};
7398         if (defined($running)) {
7399             next if $d->{template};
7400             next if $running && $d->{status} ne 'running';
7401             next if !$running && $d->{status} eq 'running';
7402         }
7403         push @$res, $id;
7404
7405     }
7406     return $res;
7407 };
7408
7409 sub complete_vmid {
7410     return &$complete_vmid_full();
7411 }
7412
7413 sub complete_vmid_stopped {
7414     return &$complete_vmid_full(0);
7415 }
7416
7417 sub complete_vmid_running {
7418     return &$complete_vmid_full(1);
7419 }
7420
7421 sub complete_storage {
7422
7423     my $cfg = PVE::Storage::config();
7424     my $ids = $cfg->{ids};
7425
7426     my $res = [];
7427     foreach my $sid (keys %$ids) {
7428         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7429         next if !$ids->{$sid}->{content}->{images};
7430         push @$res, $sid;
7431     }
7432
7433     return $res;
7434 }
7435
7436 1;