PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs'
 182 );
 183 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 184
 185 my $cpu_fmt = {
 186     cputype => {
 187         description => "Emulated CPU type.",
 188         type => 'string',
 189         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 190         default => 'kvm64',
 191         default_key => 1,
 192     },
 193     hidden => {
 194         description => "Do not identify as a KVM virtual machine.",
 195         type => 'boolean',
 196         optional => 1,
 197         default => 0
 198     },
 199     'hv-vendor-id' => {
 200         type => 'string',
 201         pattern => qr/[a-zA-Z0-9]{1,12}/,
 202         format_description => 'vendor-id',
 203         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 204         optional => 1,
 205     },
 206     flags => {
 207         description => "List of additional CPU flags separated by ';'."
 208                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 209                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 210         format_description => '+FLAG[;-FLAG...]',
 211         type => 'string',
 212         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 213         optional => 1,
 214     },
 215 };
 216
 217 my $watchdog_fmt = {
 218     model => {
 219         default_key => 1,
 220         type => 'string',
 221         enum => [qw(i6300esb ib700)],
 222         description => "Watchdog type to emulate.",
 223         default => 'i6300esb',
 224         optional => 1,
 225     },
 226     action => {
 227         type => 'string',
 228         enum => [qw(reset shutdown poweroff pause debug none)],
 229         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 230         optional => 1,
 231     },
 232 };
 233 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 234
 235 my $agent_fmt = {
 236     enabled => {
 237         description => "Enable/disable Qemu GuestAgent.",
 238         type => 'boolean',
 239         default => 0,
 240         default_key => 1,
 241     },
 242     fstrim_cloned_disks => {
 243         description => "Run fstrim after cloning/moving a disk.",
 244         type => 'boolean',
 245         optional => 1,
 246         default => 0
 247     },
 248 };
 249
 250 my $vga_fmt = {
 251     type => {
 252         description => "Select the VGA type.",
 253         type => 'string',
 254         default => 'std',
 255         optional => 1,
 256         default_key => 1,
 257         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 258     },
 259     memory => {
 260         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 261         type => 'integer',
 262         optional => 1,
 263         minimum => 4,
 264         maximum => 512,
 265     },
 266 };
 267
 268 my $ivshmem_fmt = {
 269     size => {
 270         type => 'integer',
 271         minimum => 1,
 272         description => "The size of the file in MB.",
 273     },
 274     name => {
 275         type => 'string',
 276         pattern => '[a-zA-Z0-9\-]+',
 277         optional => 1,
 278         format_description => 'string',
 279         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 280     },
 281 };
 282
 283 my $audio_fmt = {
 284     device => {
 285         type => 'string',
 286         enum => [qw(ich9-intel-hda intel-hda AC97)],
 287         description =>  "Configure an audio device."
 288     },
 289     driver =>  {
 290         type => 'string',
 291         enum => ['spice'],
 292         default => 'spice',
 293         optional => 1,
 294         description => "Driver backend for the audio device."
 295     },
 296 };
 297
 298 my $confdesc = {
 299     onboot => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Specifies whether a VM will be started during system bootup.",
 303         default => 0,
 304     },
 305     autostart => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Automatic restart after crash (currently ignored).",
 309         default => 0,
 310     },
 311     hotplug => {
 312         optional => 1,
 313         type => 'string', format => 'pve-hotplug-features',
 314         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 315         default => 'network,disk,usb',
 316     },
 317     reboot => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 321         default => 1,
 322     },
 323     lock => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Lock/unlock the VM.",
 327         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 328     },
 329     cpulimit => {
 330         optional => 1,
 331         type => 'number',
 332         description => "Limit of CPU usage.",
 333         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 334         minimum => 0,
 335         maximum => 128,
 336         default => 0,
 337     },
 338     cpuunits => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "CPU weight for a VM.",
 342         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 343         minimum => 2,
 344         maximum => 262144,
 345         default => 1024,
 346     },
 347     memory => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 351         minimum => 16,
 352         default => 512,
 353     },
 354     balloon => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 358         minimum => 0,
 359     },
 360     shares => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 364         minimum => 0,
 365         maximum => 50000,
 366         default => 1000,
 367     },
 368     keyboard => {
 369         optional => 1,
 370         type => 'string',
 371         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 372                        "It should not be necessary to set it.",
 373         enum => PVE::Tools::kvmkeymaplist(),
 374         default => undef,
 375     },
 376     name => {
 377         optional => 1,
 378         type => 'string', format => 'dns-name',
 379         description => "Set a name for the VM. Only used on the configuration web interface.",
 380     },
 381     scsihw => {
 382         optional => 1,
 383         type => 'string',
 384         description => "SCSI controller model",
 385         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 386         default => 'lsi',
 387     },
 388     description => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6/3.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string',
 420         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 421         pattern => '[acdn]{1,4}',
 422         default => 'cdn',
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     vcpus => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "Number of hotplugged vcpus.",
 467         minimum => 1,
 468         default => 0,
 469     },
 470     acpi => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable ACPI.",
 474         default => 1,
 475     },
 476     agent => {
 477         optional => 1,
 478         description => "Enable/disable Qemu GuestAgent and its properties.",
 479         type => 'string',
 480         format => $agent_fmt,
 481     },
 482     kvm => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Enable/disable KVM hardware virtualization.",
 486         default => 1,
 487     },
 488     tdf => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable time drift fix.",
 492         default => 0,
 493     },
 494     localtime => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 498     },
 499     freeze => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 503     },
 504     vga => {
 505         optional => 1,
 506         type => 'string', format => $vga_fmt,
 507         description => "Configure the VGA hardware.",
 508         verbose_description => "Configure the VGA Hardware. If you want to use ".
 509             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 510             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 511             "is 'std' for all OS types besides some Windows versions (XP and " .
 512             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 513             "display server. For win* OS you can select how many independent " .
 514             "displays you want, Linux guests can add displays them self.\n".
 515             "You can also run without any graphic card, using a serial device as terminal.",
 516     },
 517     watchdog => {
 518         optional => 1,
 519         type => 'string', format => 'pve-qm-watchdog',
 520         description => "Create a virtual hardware watchdog device.",
 521         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 522             " (by a guest action), the watchdog must be periodically polled " .
 523             "by an agent inside the guest or else the watchdog will reset " .
 524             "the guest (or execute the respective action specified)",
 525     },
 526     startdate => {
 527         optional => 1,
 528         type => 'string',
 529         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 530         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 531         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 532         default => 'now',
 533     },
 534     startup =>  get_standard_option('pve-startup-order'),
 535     template => {
 536         optional => 1,
 537         type => 'boolean',
 538         description => "Enable/disable Template.",
 539         default => 0,
 540     },
 541     args => {
 542         optional => 1,
 543         type => 'string',
 544         description => "Arbitrary arguments passed to kvm.",
 545         verbose_description => <<EODESCR,
 546 Arbitrary arguments passed to kvm, for example:
 547
 548 args: -no-reboot -no-hpet
 549
 550 NOTE: this option is for experts only.
 551 EODESCR
 552     },
 553     tablet => {
 554         optional => 1,
 555         type => 'boolean',
 556         default => 1,
 557         description => "Enable/disable the USB tablet device.",
 558         verbose_description => "Enable/disable the USB tablet device. This device is " .
 559             "usually needed to allow absolute mouse positioning with VNC. " .
 560             "Else the mouse runs out of sync with normal VNC clients. " .
 561             "If you're running lots of console-only guests on one host, " .
 562             "you may consider disabling this to save some context switches. " .
 563             "This is turned off by default if you use spice (-vga=qxl).",
 564     },
 565     migrate_speed => {
 566         optional => 1,
 567         type => 'integer',
 568         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 569         minimum => 0,
 570         default => 0,
 571     },
 572     migrate_downtime => {
 573         optional => 1,
 574         type => 'number',
 575         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 576         minimum => 0,
 577         default => 0.1,
 578     },
 579     cdrom => {
 580         optional => 1,
 581         type => 'string', format => 'pve-qm-ide',
 582         typetext => '<volume>',
 583         description => "This is an alias for option -ide2",
 584     },
 585     cpu => {
 586         optional => 1,
 587         description => "Emulated CPU type.",
 588         type => 'string',
 589         format => $cpu_fmt,
 590     },
 591     parent => get_standard_option('pve-snapshot-name', {
 592         optional => 1,
 593         description => "Parent snapshot name. This is used internally, and should not be modified.",
 594     }),
 595     snaptime => {
 596         optional => 1,
 597         description => "Timestamp for snapshots.",
 598         type => 'integer',
 599         minimum => 0,
 600     },
 601     vmstate => {
 602         optional => 1,
 603         type => 'string', format => 'pve-volume-id',
 604         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 605     },
 606     vmstatestorage => get_standard_option('pve-storage-id', {
 607         description => "Default storage for VM state volumes/files.",
 608         optional => 1,
 609     }),
 610     runningmachine => get_standard_option('pve-qemu-machine', {
 611         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 612     }),
 613     machine => get_standard_option('pve-qemu-machine'),
 614     arch => {
 615         description => "Virtual processor architecture. Defaults to the host.",
 616         optional => 1,
 617         type => 'string',
 618         enum => [qw(x86_64 aarch64)],
 619     },
 620     smbios1 => {
 621         description => "Specify SMBIOS type 1 fields.",
 622         type => 'string', format => 'pve-qm-smbios1',
 623         maxLength => 512,
 624         optional => 1,
 625     },
 626     protection => {
 627         optional => 1,
 628         type => 'boolean',
 629         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 630         default => 0,
 631     },
 632     bios => {
 633         optional => 1,
 634         type => 'string',
 635         enum => [ qw(seabios ovmf) ],
 636         description => "Select BIOS implementation.",
 637         default => 'seabios',
 638     },
 639     vmgenid => {
 640         type => 'string',
 641         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 642         format_description => 'UUID',
 643         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 644         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 645             " 128-bit integer value identifier to the guest OS. This allows to".
 646             " notify the guest operating system when the virtual machine is".
 647             " executed with a different configuration (e.g. snapshot execution".
 648             " or creation from a template). The guest operating system notices".
 649             " the change, and is then able to react as appropriate by marking".
 650             " its copies of distributed databases as dirty, re-initializing its".
 651             " random number generator, etc.\n".
 652             "Note that auto-creation only works when done throug API/CLI create".
 653             " or update methods, but not when manually editing the config file.",
 654         default => "1 (autogenerated)",
 655         optional => 1,
 656     },
 657     hookscript => {
 658         type => 'string',
 659         format => 'pve-volume-id',
 660         optional => 1,
 661         description => "Script that will be executed during various steps in the vms lifetime.",
 662     },
 663     ivshmem => {
 664         type => 'string',
 665         format => $ivshmem_fmt,
 666         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 667         optional => 1,
 668     },
 669     audio0 => {
 670         type => 'string',
 671         format => $audio_fmt,
 672         description => "Configure a audio device, useful in combination with QXL/Spice.",
 673         optional => 1
 674     },
 675 };
 676
 677 my $cicustom_fmt = {
 678     meta => {
 679         type => 'string',
 680         optional => 1,
 681         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 682         format => 'pve-volume-id',
 683         format_description => 'volume',
 684     },
 685     network => {
 686         type => 'string',
 687         optional => 1,
 688         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 689         format => 'pve-volume-id',
 690         format_description => 'volume',
 691     },
 692     user => {
 693         type => 'string',
 694         optional => 1,
 695         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 696         format => 'pve-volume-id',
 697         format_description => 'volume',
 698     },
 699 };
 700 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 701
 702 my $confdesc_cloudinit = {
 703     citype => {
 704         optional => 1,
 705         type => 'string',
 706         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 707         enum => ['configdrive2', 'nocloud'],
 708     },
 709     ciuser => {
 710         optional => 1,
 711         type => 'string',
 712         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 713     },
 714     cipassword => {
 715         optional => 1,
 716         type => 'string',
 717         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 718     },
 719     cicustom => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 723         format => 'pve-qm-cicustom',
 724     },
 725     searchdomain => {
 726         optional => 1,
 727         type => 'string',
 728         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 729     },
 730     nameserver => {
 731         optional => 1,
 732         type => 'string', format => 'address-list',
 733         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     sshkeys => {
 736         optional => 1,
 737         type => 'string',
 738         format => 'urlencoded',
 739         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 740     },
 741 };
 742
 743 # what about other qemu settings ?
 744 #cpu => 'string',
 745 #machine => 'string',
 746 #fda => 'file',
 747 #fdb => 'file',
 748 #mtdblock => 'file',
 749 #sd => 'file',
 750 #pflash => 'file',
 751 #snapshot => 'bool',
 752 #bootp => 'file',
 753 ##tftp => 'dir',
 754 ##smb => 'dir',
 755 #kernel => 'file',
 756 #append => 'string',
 757 #initrd => 'file',
 758 ##soundhw => 'string',
 759
 760 while (my ($k, $v) = each %$confdesc) {
 761     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 762 }
 763
 764 my $MAX_IDE_DISKS = 4;
 765 my $MAX_SCSI_DISKS = 14;
 766 my $MAX_VIRTIO_DISKS = 16;
 767 my $MAX_SATA_DISKS = 6;
 768 my $MAX_USB_DEVICES = 5;
 769 my $MAX_NETS = 32;
 770 my $MAX_UNUSED_DISKS = 256;
 771 my $MAX_HOSTPCI_DEVICES = 4;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         optional => 1,
 850     },
 851     queues => {
 852         type => 'integer',
 853         minimum => 0, maximum => 16,
 854         description => 'Number of packet queues to be used on the device.',
 855         optional => 1,
 856     },
 857     rate => {
 858         type => 'number',
 859         minimum => 0,
 860         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 861         optional => 1,
 862     },
 863     tag => {
 864         type => 'integer',
 865         minimum => 1, maximum => 4094,
 866         description => 'VLAN tag to apply to packets on this interface.',
 867         optional => 1,
 868     },
 869     trunks => {
 870         type => 'string',
 871         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 872         description => 'VLAN trunks to pass through this interface.',
 873         format_description => 'vlanid[;vlanid...]',
 874         optional => 1,
 875     },
 876     firewall => {
 877         type => 'boolean',
 878         description => 'Whether this interface should be protected by the firewall.',
 879         optional => 1,
 880     },
 881     link_down => {
 882         type => 'boolean',
 883         description => 'Whether this interface should be disconnected (like pulling the plug).',
 884         optional => 1,
 885     },
 886 };
 887
 888 my $netdesc = {
 889     optional => 1,
 890     type => 'string', format => $net_fmt,
 891     description => "Specify network devices.",
 892 };
 893
 894 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 895
 896 my $ipconfig_fmt = {
 897     ip => {
 898         type => 'string',
 899         format => 'pve-ipv4-config',
 900         format_description => 'IPv4Format/CIDR',
 901         description => 'IPv4 address in CIDR format.',
 902         optional => 1,
 903         default => 'dhcp',
 904     },
 905     gw => {
 906         type => 'string',
 907         format => 'ipv4',
 908         format_description => 'GatewayIPv4',
 909         description => 'Default gateway for IPv4 traffic.',
 910         optional => 1,
 911         requires => 'ip',
 912     },
 913     ip6 => {
 914         type => 'string',
 915         format => 'pve-ipv6-config',
 916         format_description => 'IPv6Format/CIDR',
 917         description => 'IPv6 address in CIDR format.',
 918         optional => 1,
 919         default => 'dhcp',
 920     },
 921     gw6 => {
 922         type => 'string',
 923         format => 'ipv6',
 924         format_description => 'GatewayIPv6',
 925         description => 'Default gateway for IPv6 traffic.',
 926         optional => 1,
 927         requires => 'ip6',
 928     },
 929 };
 930 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 931 my $ipconfigdesc = {
 932     optional => 1,
 933     type => 'string', format => 'pve-qm-ipconfig',
 934     description => <<'EODESCR',
 935 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 936
 937 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 938
 939 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 940 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 941
 942 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 943 EODESCR
 944 };
 945 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 946
 947 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 948     $confdesc->{"net$i"} = $netdesc;
 949     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 950 }
 951
 952 foreach my $key (keys %$confdesc_cloudinit) {
 953     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 954 }
 955
 956 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 957 sub verify_volume_id_or_qm_path {
 958     my ($volid, $noerr) = @_;
 959
 960     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 961         return $volid;
 962     }
 963
 964     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 965     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 966     if ($@) {
 967         return undef if $noerr;
 968         die $@;
 969     }
 970     return $volid;
 971 }
 972
 973 my $drivename_hash;
 974
 975 my %drivedesc_base = (
 976     volume => { alias => 'file' },
 977     file => {
 978         type => 'string',
 979         format => 'pve-volume-id-or-qm-path',
 980         default_key => 1,
 981         format_description => 'volume',
 982         description => "The drive's backing volume.",
 983     },
 984     media => {
 985         type => 'string',
 986         enum => [qw(cdrom disk)],
 987         description => "The drive's media type.",
 988         default => 'disk',
 989         optional => 1
 990     },
 991     cyls => {
 992         type => 'integer',
 993         description => "Force the drive's physical geometry to have a specific cylinder count.",
 994         optional => 1
 995     },
 996     heads => {
 997         type => 'integer',
 998         description => "Force the drive's physical geometry to have a specific head count.",
 999         optional => 1
1000     },
1001     secs => {
1002         type => 'integer',
1003         description => "Force the drive's physical geometry to have a specific sector count.",
1004         optional => 1
1005     },
1006     trans => {
1007         type => 'string',
1008         enum => [qw(none lba auto)],
1009         description => "Force disk geometry bios translation mode.",
1010         optional => 1,
1011     },
1012     snapshot => {
1013         type => 'boolean',
1014         description => "Controls qemu's snapshot mode feature."
1015             . " If activated, changes made to the disk are temporary and will"
1016             . " be discarded when the VM is shutdown.",
1017         optional => 1,
1018     },
1019     cache => {
1020         type => 'string',
1021         enum => [qw(none writethrough writeback unsafe directsync)],
1022         description => "The drive's cache mode",
1023         optional => 1,
1024     },
1025     format => get_standard_option('pve-qm-image-format'),
1026     size => {
1027         type => 'string',
1028         format => 'disk-size',
1029         format_description => 'DiskSize',
1030         description => "Disk size. This is purely informational and has no effect.",
1031         optional => 1,
1032     },
1033     backup => {
1034         type => 'boolean',
1035         description => "Whether the drive should be included when making backups.",
1036         optional => 1,
1037     },
1038     replicate => {
1039         type => 'boolean',
1040         description => 'Whether the drive should considered for replication jobs.',
1041         optional => 1,
1042         default => 1,
1043     },
1044     rerror => {
1045         type => 'string',
1046         enum => [qw(ignore report stop)],
1047         description => 'Read error action.',
1048         optional => 1,
1049     },
1050     werror => {
1051         type => 'string',
1052         enum => [qw(enospc ignore report stop)],
1053         description => 'Write error action.',
1054         optional => 1,
1055     },
1056     aio => {
1057         type => 'string',
1058         enum => [qw(native threads)],
1059         description => 'AIO type to use.',
1060         optional => 1,
1061     },
1062     discard => {
1063         type => 'string',
1064         enum => [qw(ignore on)],
1065         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1066         optional => 1,
1067     },
1068     detect_zeroes => {
1069         type => 'boolean',
1070         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1071         optional => 1,
1072     },
1073     serial => {
1074         type => 'string',
1075         format => 'urlencoded',
1076         format_description => 'serial',
1077         maxLength => 20*3, # *3 since it's %xx url enoded
1078         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1079         optional => 1,
1080     },
1081     shared => {
1082         type => 'boolean',
1083         description => 'Mark this locally-managed volume as available on all nodes',
1084         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1085         optional => 1,
1086         default => 0,
1087     }
1088 );
1089
1090 my %iothread_fmt = ( iothread => {
1091         type => 'boolean',
1092         description => "Whether to use iothreads for this drive",
1093         optional => 1,
1094 });
1095
1096 my %model_fmt = (
1097     model => {
1098         type => 'string',
1099         format => 'urlencoded',
1100         format_description => 'model',
1101         maxLength => 40*3, # *3 since it's %xx url enoded
1102         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1103         optional => 1,
1104     },
1105 );
1106
1107 my %queues_fmt = (
1108     queues => {
1109         type => 'integer',
1110         description => "Number of queues.",
1111         minimum => 2,
1112         optional => 1
1113     }
1114 );
1115
1116 my %scsiblock_fmt = (
1117     scsiblock => {
1118         type => 'boolean',
1119         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1120         optional => 1,
1121         default => 0,
1122     },
1123 );
1124
1125 my %ssd_fmt = (
1126     ssd => {
1127         type => 'boolean',
1128         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %wwn_fmt = (
1134     wwn => {
1135         type => 'string',
1136         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1137         format_description => 'wwn',
1138         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1139         optional => 1,
1140     },
1141 );
1142
1143 my $add_throttle_desc = sub {
1144     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1145     my $d = {
1146         type => $type,
1147         format_description => $unit,
1148         description => "Maximum $what in $longunit.",
1149         optional => 1,
1150     };
1151     $d->{minimum} = $minimum if defined($minimum);
1152     $drivedesc_base{$key} = $d;
1153 };
1154 # throughput: (leaky bucket)
1155 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1156 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1157 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1158 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1159 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1160 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1161 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1162 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1163 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1164
1165 # pools: (pool of IO before throttling starts taking effect)
1166 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1167 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1168 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1169 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1170 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1171 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1172
1173 # burst lengths
1174 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1175 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1176 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1177 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1178 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1179 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1180
1181 # legacy support
1182 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1183 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1184 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1185 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1186
1187 my $ide_fmt = {
1188     %drivedesc_base,
1189     %model_fmt,
1190     %ssd_fmt,
1191     %wwn_fmt,
1192 };
1193 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1194
1195 my $idedesc = {
1196     optional => 1,
1197     type => 'string', format => $ide_fmt,
1198     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1199 };
1200 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1201
1202 my $scsi_fmt = {
1203     %drivedesc_base,
1204     %iothread_fmt,
1205     %queues_fmt,
1206     %scsiblock_fmt,
1207     %ssd_fmt,
1208     %wwn_fmt,
1209 };
1210 my $scsidesc = {
1211     optional => 1,
1212     type => 'string', format => $scsi_fmt,
1213     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1214 };
1215 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1216
1217 my $sata_fmt = {
1218     %drivedesc_base,
1219     %ssd_fmt,
1220     %wwn_fmt,
1221 };
1222 my $satadesc = {
1223     optional => 1,
1224     type => 'string', format => $sata_fmt,
1225     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1226 };
1227 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1228
1229 my $virtio_fmt = {
1230     %drivedesc_base,
1231     %iothread_fmt,
1232 };
1233 my $virtiodesc = {
1234     optional => 1,
1235     type => 'string', format => $virtio_fmt,
1236     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1237 };
1238 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1239
1240 my $alldrive_fmt = {
1241     %drivedesc_base,
1242     %iothread_fmt,
1243     %model_fmt,
1244     %queues_fmt,
1245     %scsiblock_fmt,
1246     %ssd_fmt,
1247     %wwn_fmt,
1248 };
1249
1250 my $efidisk_fmt = {
1251     volume => { alias => 'file' },
1252     file => {
1253         type => 'string',
1254         format => 'pve-volume-id-or-qm-path',
1255         default_key => 1,
1256         format_description => 'volume',
1257         description => "The drive's backing volume.",
1258     },
1259     format => get_standard_option('pve-qm-image-format'),
1260     size => {
1261         type => 'string',
1262         format => 'disk-size',
1263         format_description => 'DiskSize',
1264         description => "Disk size. This is purely informational and has no effect.",
1265         optional => 1,
1266     },
1267 };
1268
1269 my $efidisk_desc = {
1270     optional => 1,
1271     type => 'string', format => $efidisk_fmt,
1272     description => "Configure a Disk for storing EFI vars",
1273 };
1274
1275 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1276
1277 my $usb_fmt = {
1278     host => {
1279         default_key => 1,
1280         type => 'string', format => 'pve-qm-usb-device',
1281         format_description => 'HOSTUSBDEVICE|spice',
1282         description => <<EODESCR,
1283 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1284
1285  'bus-port(.port)*' (decimal numbers) or
1286  'vendor_id:product_id' (hexadeciaml numbers) or
1287  'spice'
1288
1289 You can use the 'lsusb -t' command to list existing usb devices.
1290
1291 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1292
1293 The value 'spice' can be used to add a usb redirection devices for spice.
1294 EODESCR
1295     },
1296     usb3 => {
1297         optional => 1,
1298         type => 'boolean',
1299         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1300         default => 0,
1301     },
1302 };
1303
1304 my $usbdesc = {
1305     optional => 1,
1306     type => 'string', format => $usb_fmt,
1307     description => "Configure an USB device (n is 0 to 4).",
1308 };
1309 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1310
1311 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1312 my $hostpci_fmt = {
1313     host => {
1314         default_key => 1,
1315         type => 'string',
1316         pattern => qr/$PCIRE(;$PCIRE)*/,
1317         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1318         description => <<EODESCR,
1319 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1320 of PCI virtual functions of the host. HOSTPCIID syntax is:
1321
1322 'bus:dev.func' (hexadecimal numbers)
1323
1324 You can us the 'lspci' command to list existing PCI devices.
1325 EODESCR
1326     },
1327     rombar => {
1328         type => 'boolean',
1329         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1330         optional => 1,
1331         default => 1,
1332     },
1333     romfile => {
1334         type => 'string',
1335         pattern => '[^,;]+',
1336         format_description => 'string',
1337         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1338         optional => 1,
1339     },
1340     pcie => {
1341         type => 'boolean',
1342         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1343         optional => 1,
1344         default => 0,
1345     },
1346     'x-vga' => {
1347         type => 'boolean',
1348         description =>  "Enable vfio-vga device support.",
1349         optional => 1,
1350         default => 0,
1351     },
1352     'mdev' => {
1353         type => 'string',
1354         format_description => 'string',
1355         pattern => '[^/\.:]+',
1356         optional => 1,
1357         description => <<EODESCR
1358 The type of mediated device to use.
1359 An instance of this type will be created on startup of the VM and
1360 will be cleaned up when the VM stops.
1361 EODESCR
1362     }
1363 };
1364 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1365
1366 my $hostpcidesc = {
1367         optional => 1,
1368         type => 'string', format => 'pve-qm-hostpci',
1369         description => "Map host PCI devices into guest.",
1370         verbose_description =>  <<EODESCR,
1371 Map host PCI devices into guest.
1372
1373 NOTE: This option allows direct access to host hardware. So it is no longer
1374 possible to migrate such machines - use with special care.
1375
1376 CAUTION: Experimental! User reported problems with this option.
1377 EODESCR
1378 };
1379 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1380
1381 my $serialdesc = {
1382         optional => 1,
1383         type => 'string',
1384         pattern => '(/dev/.+|socket)',
1385         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1386         verbose_description =>  <<EODESCR,
1387 Create a serial device inside the VM (n is 0 to 3), and pass through a
1388 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1389 host side (use 'qm terminal' to open a terminal connection).
1390
1391 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1392
1393 CAUTION: Experimental! User reported problems with this option.
1394 EODESCR
1395 };
1396
1397 my $paralleldesc= {
1398         optional => 1,
1399         type => 'string',
1400         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1401         description =>  "Map host parallel devices (n is 0 to 2).",
1402         verbose_description =>  <<EODESCR,
1403 Map host parallel devices (n is 0 to 2).
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1406
1407 CAUTION: Experimental! User reported problems with this option.
1408 EODESCR
1409 };
1410
1411 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1412     $confdesc->{"parallel$i"} = $paralleldesc;
1413 }
1414
1415 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1416     $confdesc->{"serial$i"} = $serialdesc;
1417 }
1418
1419 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1420     $confdesc->{"hostpci$i"} = $hostpcidesc;
1421 }
1422
1423 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1424     $drivename_hash->{"ide$i"} = 1;
1425     $confdesc->{"ide$i"} = $idedesc;
1426 }
1427
1428 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1429     $drivename_hash->{"sata$i"} = 1;
1430     $confdesc->{"sata$i"} = $satadesc;
1431 }
1432
1433 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1434     $drivename_hash->{"scsi$i"} = 1;
1435     $confdesc->{"scsi$i"} = $scsidesc ;
1436 }
1437
1438 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1439     $drivename_hash->{"virtio$i"} = 1;
1440     $confdesc->{"virtio$i"} = $virtiodesc;
1441 }
1442
1443 $drivename_hash->{efidisk0} = 1;
1444 $confdesc->{efidisk0} = $efidisk_desc;
1445
1446 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1447     $confdesc->{"usb$i"} = $usbdesc;
1448 }
1449
1450 my $unuseddesc = {
1451     optional => 1,
1452     type => 'string', format => 'pve-volume-id',
1453     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1454 };
1455
1456 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1457     $confdesc->{"unused$i"} = $unuseddesc;
1458 }
1459
1460 my $kvm_api_version = 0;
1461
1462 sub kvm_version {
1463     return $kvm_api_version if $kvm_api_version;
1464
1465     open my $fh, '<', '/dev/kvm'
1466         or return undef;
1467
1468     # 0xae00 => KVM_GET_API_VERSION
1469     $kvm_api_version = ioctl($fh, 0xae00, 0);
1470
1471     return $kvm_api_version;
1472 }
1473
1474 my $kvm_user_version = {};
1475 my $kvm_mtime = {};
1476
1477 sub kvm_user_version {
1478     my ($binary) = @_;
1479
1480     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1481     my $st = stat($binary);
1482
1483     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1484     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1485         $cachedmtime == $st->mtime;
1486
1487     $kvm_user_version->{$binary} = 'unknown';
1488     $kvm_mtime->{$binary} = $st->mtime;
1489
1490     my $code = sub {
1491         my $line = shift;
1492         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1493             $kvm_user_version->{$binary} = $2;
1494         }
1495     };
1496
1497     eval { run_command([$binary, '--version'], outfunc => $code); };
1498     warn $@ if $@;
1499
1500     return $kvm_user_version->{$binary};
1501
1502 }
1503
1504 sub kernel_has_vhost_net {
1505     return -c '/dev/vhost-net';
1506 }
1507
1508 sub valid_drive_names {
1509     # order is important - used to autoselect boot disk
1510     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1511             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1512             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1513             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1514             'efidisk0');
1515 }
1516
1517 sub is_valid_drivename {
1518     my $dev = shift;
1519
1520     return defined($drivename_hash->{$dev});
1521 }
1522
1523 sub option_exists {
1524     my $key = shift;
1525     return defined($confdesc->{$key});
1526 }
1527
1528 sub nic_models {
1529     return $nic_model_list;
1530 }
1531
1532 sub os_list_description {
1533
1534     return {
1535         other => 'Other',
1536         wxp => 'Windows XP',
1537         w2k => 'Windows 2000',
1538         w2k3 =>, 'Windows 2003',
1539         w2k8 => 'Windows 2008',
1540         wvista => 'Windows Vista',
1541         win7 => 'Windows 7',
1542         win8 => 'Windows 8/2012',
1543         win10 => 'Windows 10/2016',
1544         l24 => 'Linux 2.4',
1545         l26 => 'Linux 2.6',
1546     };
1547 }
1548
1549 my $cdrom_path;
1550
1551 sub get_cdrom_path {
1552
1553     return  $cdrom_path if $cdrom_path;
1554
1555     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1556     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1557     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1558 }
1559
1560 sub get_iso_path {
1561     my ($storecfg, $vmid, $cdrom) = @_;
1562
1563     if ($cdrom eq 'cdrom') {
1564         return get_cdrom_path();
1565     } elsif ($cdrom eq 'none') {
1566         return '';
1567     } elsif ($cdrom =~ m|^/|) {
1568         return $cdrom;
1569     } else {
1570         return PVE::Storage::path($storecfg, $cdrom);
1571     }
1572 }
1573
1574 # try to convert old style file names to volume IDs
1575 sub filename_to_volume_id {
1576     my ($vmid, $file, $media) = @_;
1577
1578      if (!($file eq 'none' || $file eq 'cdrom' ||
1579           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1580
1581         return undef if $file =~ m|/|;
1582
1583         if ($media && $media eq 'cdrom') {
1584             $file = "local:iso/$file";
1585         } else {
1586             $file = "local:$vmid/$file";
1587         }
1588     }
1589
1590     return $file;
1591 }
1592
1593 sub verify_media_type {
1594     my ($opt, $vtype, $media) = @_;
1595
1596     return if !$media;
1597
1598     my $etype;
1599     if ($media eq 'disk') {
1600         $etype = 'images';
1601     } elsif ($media eq 'cdrom') {
1602         $etype = 'iso';
1603     } else {
1604         die "internal error";
1605     }
1606
1607     return if ($vtype eq $etype);
1608
1609     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1610 }
1611
1612 sub cleanup_drive_path {
1613     my ($opt, $storecfg, $drive) = @_;
1614
1615     # try to convert filesystem paths to volume IDs
1616
1617     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1618         ($drive->{file} !~ m|^/dev/.+|) &&
1619         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1620         ($drive->{file} !~ m/^\d+$/)) {
1621         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1622         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1623         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1624         verify_media_type($opt, $vtype, $drive->{media});
1625         $drive->{file} = $volid;
1626     }
1627
1628     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1629 }
1630
1631 sub parse_hotplug_features {
1632     my ($data) = @_;
1633
1634     my $res = {};
1635
1636     return $res if $data eq '0';
1637
1638     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1639
1640     foreach my $feature (PVE::Tools::split_list($data)) {
1641         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1642             $res->{$1} = 1;
1643         } else {
1644             die "invalid hotplug feature '$feature'\n";
1645         }
1646     }
1647     return $res;
1648 }
1649
1650 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1651 sub pve_verify_hotplug_features {
1652     my ($value, $noerr) = @_;
1653
1654     return $value if parse_hotplug_features($value);
1655
1656     return undef if $noerr;
1657
1658     die "unable to parse hotplug option\n";
1659 }
1660
1661 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1662 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1663 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1664 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1665 #        [,iothread=on][,serial=serial][,model=model]
1666
1667 sub parse_drive {
1668     my ($key, $data) = @_;
1669
1670     my ($interface, $index);
1671
1672     if ($key =~ m/^([^\d]+)(\d+)$/) {
1673         $interface = $1;
1674         $index = $2;
1675     } else {
1676         return undef;
1677     }
1678
1679     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1680                                      : $confdesc->{$key}->{format};
1681     if (!$desc) {
1682         warn "invalid drive key: $key\n";
1683         return undef;
1684     }
1685     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1686     return undef if !$res;
1687     $res->{interface} = $interface;
1688     $res->{index} = $index;
1689
1690     my $error = 0;
1691     foreach my $opt (qw(bps bps_rd bps_wr)) {
1692         if (my $bps = defined(delete $res->{$opt})) {
1693             if (defined($res->{"m$opt"})) {
1694                 warn "both $opt and m$opt specified\n";
1695                 ++$error;
1696                 next;
1697             }
1698             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1699         }
1700     }
1701
1702     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1703     for my $requirement (
1704         [mbps_max => 'mbps'],
1705         [mbps_rd_max => 'mbps_rd'],
1706         [mbps_wr_max => 'mbps_wr'],
1707         [miops_max => 'miops'],
1708         [miops_rd_max => 'miops_rd'],
1709         [miops_wr_max => 'miops_wr'],
1710         [bps_max_length => 'mbps_max'],
1711         [bps_rd_max_length => 'mbps_rd_max'],
1712         [bps_wr_max_length => 'mbps_wr_max'],
1713         [iops_max_length => 'iops_max'],
1714         [iops_rd_max_length => 'iops_rd_max'],
1715         [iops_wr_max_length => 'iops_wr_max']) {
1716         my ($option, $requires) = @$requirement;
1717         if ($res->{$option} && !$res->{$requires}) {
1718             warn "$option requires $requires\n";
1719             ++$error;
1720         }
1721     }
1722
1723     return undef if $error;
1724
1725     return undef if $res->{mbps_rd} && $res->{mbps};
1726     return undef if $res->{mbps_wr} && $res->{mbps};
1727     return undef if $res->{iops_rd} && $res->{iops};
1728     return undef if $res->{iops_wr} && $res->{iops};
1729
1730     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1731         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1732         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1733         return undef if $res->{interface} eq 'virtio';
1734     }
1735
1736     if (my $size = $res->{size}) {
1737         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1738     }
1739
1740     return $res;
1741 }
1742
1743 sub print_drive {
1744     my ($vmid, $drive) = @_;
1745     my $data = { %$drive };
1746     delete $data->{$_} for qw(index interface);
1747     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1748 }
1749
1750 sub scsi_inquiry {
1751     my($fh, $noerr) = @_;
1752
1753     my $SG_IO = 0x2285;
1754     my $SG_GET_VERSION_NUM = 0x2282;
1755
1756     my $versionbuf = "\x00" x 8;
1757     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1758     if (!$ret) {
1759         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1760         return undef;
1761     }
1762     my $version = unpack("I", $versionbuf);
1763     if ($version < 30000) {
1764         die "scsi generic interface too old\n"  if !$noerr;
1765         return undef;
1766     }
1767
1768     my $buf = "\x00" x 36;
1769     my $sensebuf = "\x00" x 8;
1770     my $cmd = pack("C x3 C x1", 0x12, 36);
1771
1772     # see /usr/include/scsi/sg.h
1773     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1774
1775     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1776                       length($sensebuf), 0, length($buf), $buf,
1777                       $cmd, $sensebuf, 6000);
1778
1779     $ret = ioctl($fh, $SG_IO, $packet);
1780     if (!$ret) {
1781         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1782         return undef;
1783     }
1784
1785     my @res = unpack($sg_io_hdr_t, $packet);
1786     if ($res[17] || $res[18]) {
1787         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1788         return undef;
1789     }
1790
1791     my $res = {};
1792     (my $byte0, my $byte1, $res->{vendor},
1793      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1794
1795     $res->{removable} = $byte1 & 128 ? 1 : 0;
1796     $res->{type} = $byte0 & 31;
1797
1798     return $res;
1799 }
1800
1801 sub path_is_scsi {
1802     my ($path) = @_;
1803
1804     my $fh = IO::File->new("+<$path") || return undef;
1805     my $res = scsi_inquiry($fh, 1);
1806     close($fh);
1807
1808     return $res;
1809 }
1810
1811 sub machine_type_is_q35 {
1812     my ($conf) = @_;
1813
1814     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1815 }
1816
1817 sub print_tabletdevice_full {
1818     my ($conf, $arch) = @_;
1819
1820     my $q35 = machine_type_is_q35($conf);
1821
1822     # we use uhci for old VMs because tablet driver was buggy in older qemu
1823     my $usbbus;
1824     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1825         $usbbus = 'ehci';
1826     } else {
1827         $usbbus = 'uhci';
1828     }
1829
1830     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1831 }
1832
1833 sub print_keyboarddevice_full {
1834     my ($conf, $arch, $machine) = @_;
1835
1836     return undef if $arch ne 'aarch64';
1837
1838     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1839 }
1840
1841 sub print_drivedevice_full {
1842     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1843
1844     my $device = '';
1845     my $maxdev = 0;
1846
1847     if ($drive->{interface} eq 'virtio') {
1848         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1849         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1850         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1851     } elsif ($drive->{interface} eq 'scsi') {
1852
1853         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1854         my $unit = $drive->{index} % $maxdev;
1855         my $devicetype = 'hd';
1856         my $path = '';
1857         if (drive_is_cdrom($drive)) {
1858             $devicetype = 'cd';
1859         } else {
1860             if ($drive->{file} =~ m|^/|) {
1861                 $path = $drive->{file};
1862                 if (my $info = path_is_scsi($path)) {
1863                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1864                         $devicetype = 'block';
1865                     } elsif ($info->{type} == 1) { # tape
1866                         $devicetype = 'generic';
1867                     }
1868                 }
1869             } else {
1870                  $path = PVE::Storage::path($storecfg, $drive->{file});
1871             }
1872
1873             if($path =~ m/^iscsi\:\/\//){
1874                 $devicetype = 'generic';
1875             }
1876         }
1877
1878         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1879             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1880         } else {
1881             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1882         }
1883
1884         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1885             $device .= ",rotation_rate=1";
1886         }
1887         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1888
1889     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1890         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1891         my $controller = int($drive->{index} / $maxdev);
1892         my $unit = $drive->{index} % $maxdev;
1893         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1894
1895         $device = "ide-$devicetype";
1896         if ($drive->{interface} eq 'ide') {
1897             $device .= ",bus=ide.$controller,unit=$unit";
1898         } else {
1899             $device .= ",bus=ahci$controller.$unit";
1900         }
1901         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1902
1903         if ($devicetype eq 'hd') {
1904             if (my $model = $drive->{model}) {
1905                 $model = URI::Escape::uri_unescape($model);
1906                 $device .= ",model=$model";
1907             }
1908             if ($drive->{ssd}) {
1909                 $device .= ",rotation_rate=1";
1910             }
1911         }
1912         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1913     } elsif ($drive->{interface} eq 'usb') {
1914         die "implement me";
1915         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1916     } else {
1917         die "unsupported interface type";
1918     }
1919
1920     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1921
1922     if (my $serial = $drive->{serial}) {
1923         $serial = URI::Escape::uri_unescape($serial);
1924         $device .= ",serial=$serial";
1925     }
1926
1927
1928     return $device;
1929 }
1930
1931 sub get_initiator_name {
1932     my $initiator;
1933
1934     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1935     while (defined(my $line = <$fh>)) {
1936         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1937         $initiator = $1;
1938         last;
1939     }
1940     $fh->close();
1941
1942     return $initiator;
1943 }
1944
1945 sub print_drive_full {
1946     my ($storecfg, $vmid, $drive) = @_;
1947
1948     my $path;
1949     my $volid = $drive->{file};
1950     my $format;
1951
1952     if (drive_is_cdrom($drive)) {
1953         $path = get_iso_path($storecfg, $vmid, $volid);
1954     } else {
1955         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1956         if ($storeid) {
1957             $path = PVE::Storage::path($storecfg, $volid);
1958             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1959             $format = qemu_img_format($scfg, $volname);
1960         } else {
1961             $path = $volid;
1962             $format = "raw";
1963         }
1964    }
1965
1966     my $opts = '';
1967     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1968     foreach my $o (@qemu_drive_options) {
1969         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1970     }
1971
1972     # snapshot only accepts on|off
1973     if (defined($drive->{snapshot})) {
1974         my $v = $drive->{snapshot} ? 'on' : 'off';
1975         $opts .= ",snapshot=$v";
1976     }
1977
1978     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1979         my ($dir, $qmpname) = @$type;
1980         if (my $v = $drive->{"mbps$dir"}) {
1981             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1982         }
1983         if (my $v = $drive->{"mbps${dir}_max"}) {
1984             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1985         }
1986         if (my $v = $drive->{"bps${dir}_max_length"}) {
1987             $opts .= ",throttling.bps$qmpname-max-length=$v";
1988         }
1989         if (my $v = $drive->{"iops${dir}"}) {
1990             $opts .= ",throttling.iops$qmpname=$v";
1991         }
1992         if (my $v = $drive->{"iops${dir}_max"}) {
1993             $opts .= ",throttling.iops$qmpname-max=$v";
1994         }
1995         if (my $v = $drive->{"iops${dir}_max_length"}) {
1996             $opts .= ",throttling.iops$qmpname-max-length=$v";
1997         }
1998     }
1999
2000     $opts .= ",format=$format" if $format && !$drive->{format};
2001
2002     my $cache_direct = 0;
2003
2004     if (my $cache = $drive->{cache}) {
2005         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2006     } elsif (!drive_is_cdrom($drive)) {
2007         $opts .= ",cache=none";
2008         $cache_direct = 1;
2009     }
2010
2011     # aio native works only with O_DIRECT
2012     if (!$drive->{aio}) {
2013         if($cache_direct) {
2014             $opts .= ",aio=native";
2015         } else {
2016             $opts .= ",aio=threads";
2017         }
2018     }
2019
2020     if (!drive_is_cdrom($drive)) {
2021         my $detectzeroes;
2022         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2023             $detectzeroes = 'off';
2024         } elsif ($drive->{discard}) {
2025             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2026         } else {
2027             # This used to be our default with discard not being specified:
2028             $detectzeroes = 'on';
2029         }
2030         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2031     }
2032
2033     my $pathinfo = $path ? "file=$path," : '';
2034
2035     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2036 }
2037
2038 sub print_netdevice_full {
2039     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2040
2041     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2042
2043     my $device = $net->{model};
2044     if ($net->{model} eq 'virtio') {
2045          $device = 'virtio-net-pci';
2046      };
2047
2048     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2049     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2050     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2051         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2052         my $vectors = $net->{queues} * 2 + 2;
2053         $tmpstr .= ",vectors=$vectors,mq=on";
2054     }
2055     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2056
2057     if ($use_old_bios_files) {
2058         my $romfile;
2059         if ($device eq 'virtio-net-pci') {
2060             $romfile = 'pxe-virtio.rom';
2061         } elsif ($device eq 'e1000') {
2062             $romfile = 'pxe-e1000.rom';
2063         } elsif ($device eq 'ne2k') {
2064             $romfile = 'pxe-ne2k_pci.rom';
2065         } elsif ($device eq 'pcnet') {
2066             $romfile = 'pxe-pcnet.rom';
2067         } elsif ($device eq 'rtl8139') {
2068             $romfile = 'pxe-rtl8139.rom';
2069         }
2070         $tmpstr .= ",romfile=$romfile" if $romfile;
2071     }
2072
2073     return $tmpstr;
2074 }
2075
2076 sub print_netdev_full {
2077     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2078
2079     my $i = '';
2080     if ($netid =~ m/^net(\d+)$/) {
2081         $i = int($1);
2082     }
2083
2084     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2085
2086     my $ifname = "tap${vmid}i$i";
2087
2088     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2089     die "interface name '$ifname' is too long (max 15 character)\n"
2090         if length($ifname) >= 16;
2091
2092     my $vhostparam = '';
2093     if (is_native($arch)) {
2094         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2095     }
2096
2097     my $vmname = $conf->{name} || "vm$vmid";
2098
2099     my $netdev = "";
2100     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2101
2102     if ($net->{bridge}) {
2103         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2104     } else {
2105         $netdev = "type=user,id=$netid,hostname=$vmname";
2106     }
2107
2108     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2109
2110     return $netdev;
2111 }
2112
2113
2114 sub print_cpu_device {
2115     my ($conf, $id) = @_;
2116
2117     my $kvm = $conf->{kvm} // 1;
2118     my $cpu = $kvm ? "kvm64" : "qemu64";
2119     if (my $cputype = $conf->{cpu}) {
2120         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2121             or die "Cannot parse cpu description: $cputype\n";
2122         $cpu = $cpuconf->{cputype};
2123     }
2124
2125     my $cores = $conf->{cores} || 1;
2126
2127     my $current_core = ($id - 1) % $cores;
2128     my $current_socket = int(($id - 1 - $current_core)/$cores);
2129
2130     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2131 }
2132
2133 my $vga_map = {
2134     'cirrus' => 'cirrus-vga',
2135     'std' => 'VGA',
2136     'vmware' => 'vmware-svga',
2137     'virtio' => 'virtio-vga',
2138 };
2139
2140 sub print_vga_device {
2141     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2142
2143     my $type = $vga_map->{$vga->{type}};
2144     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2145         $type = 'virtio-gpu';
2146     }
2147     my $vgamem_mb = $vga->{memory};
2148     if ($qxlnum) {
2149         $type = $id ? 'qxl' : 'qxl-vga';
2150     }
2151     die "no devicetype for $vga->{type}\n" if !$type;
2152
2153     my $memory = "";
2154     if ($vgamem_mb) {
2155         if ($vga->{type} eq 'virtio') {
2156             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2157             $memory = ",max_hostmem=$bytes";
2158         } elsif ($qxlnum) {
2159             # from https://www.spice-space.org/multiple-monitors.html
2160             $memory = ",vgamem_mb=$vga->{memory}";
2161             my $ram = $vgamem_mb * 4;
2162             my $vram = $vgamem_mb * 2;
2163             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2164         } else {
2165             $memory = ",vgamem_mb=$vga->{memory}";
2166         }
2167     } elsif ($qxlnum && $id) {
2168         $memory = ",ram_size=67108864,vram_size=33554432";
2169     }
2170
2171     my $q35 = machine_type_is_q35($conf);
2172     my $vgaid = "vga" . ($id // '');
2173     my $pciaddr;
2174
2175     if ($q35 && $vgaid eq 'vga') {
2176         # the first display uses pcie.0 bus on q35 machines
2177         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2178     } else {
2179         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2180     }
2181
2182     return "$type,id=${vgaid}${memory}${pciaddr}";
2183 }
2184
2185 sub drive_is_cloudinit {
2186     my ($drive) = @_;
2187     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2188 }
2189
2190 sub drive_is_cdrom {
2191     my ($drive, $exclude_cloudinit) = @_;
2192
2193     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2194
2195     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2196
2197 }
2198
2199 sub parse_number_sets {
2200     my ($set) = @_;
2201     my $res = [];
2202     foreach my $part (split(/;/, $set)) {
2203         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2204             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2205             push @$res, [ $1, $2 ];
2206         } else {
2207             die "invalid range: $part\n";
2208         }
2209     }
2210     return $res;
2211 }
2212
2213 sub parse_numa {
2214     my ($data) = @_;
2215
2216     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2217     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2218     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2219     return $res;
2220 }
2221
2222 sub parse_hostpci {
2223     my ($value) = @_;
2224
2225     return undef if !$value;
2226
2227     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2228
2229     my @idlist = split(/;/, $res->{host});
2230     delete $res->{host};
2231     foreach my $id (@idlist) {
2232         if ($id =~ m/\./) { # full id 00:00.1
2233             push @{$res->{pciid}}, {
2234                 id => $id,
2235             };
2236         } else { # partial id 00:00
2237             $res->{pciid} = PVE::SysFSTools::lspci($id);
2238         }
2239     }
2240     return $res;
2241 }
2242
2243 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2244 sub parse_net {
2245     my ($data) = @_;
2246
2247     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2248     if ($@) {
2249         warn $@;
2250         return undef;
2251     }
2252     if (!defined($res->{macaddr})) {
2253         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2254         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2255     }
2256     return $res;
2257 }
2258
2259 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2260 sub parse_ipconfig {
2261     my ($data) = @_;
2262
2263     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2264     if ($@) {
2265         warn $@;
2266         return undef;
2267     }
2268
2269     if ($res->{gw} && !$res->{ip}) {
2270         warn 'gateway specified without specifying an IP address';
2271         return undef;
2272     }
2273     if ($res->{gw6} && !$res->{ip6}) {
2274         warn 'IPv6 gateway specified without specifying an IPv6 address';
2275         return undef;
2276     }
2277     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2278         warn 'gateway specified together with DHCP';
2279         return undef;
2280     }
2281     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2282         # gw6 + auto/dhcp
2283         warn "IPv6 gateway specified together with $res->{ip6} address";
2284         return undef;
2285     }
2286
2287     if (!$res->{ip} && !$res->{ip6}) {
2288         return { ip => 'dhcp', ip6 => 'dhcp' };
2289     }
2290
2291     return $res;
2292 }
2293
2294 sub print_net {
2295     my $net = shift;
2296
2297     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2298 }
2299
2300 sub add_random_macs {
2301     my ($settings) = @_;
2302
2303     foreach my $opt (keys %$settings) {
2304         next if $opt !~ m/^net(\d+)$/;
2305         my $net = parse_net($settings->{$opt});
2306         next if !$net;
2307         $settings->{$opt} = print_net($net);
2308     }
2309 }
2310
2311 sub vm_is_volid_owner {
2312     my ($storecfg, $vmid, $volid) = @_;
2313
2314     if ($volid !~  m|^/|) {
2315         my ($path, $owner);
2316         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2317         if ($owner && ($owner == $vmid)) {
2318             return 1;
2319         }
2320     }
2321
2322     return undef;
2323 }
2324
2325 sub split_flagged_list {
2326     my $text = shift || '';
2327     $text =~ s/[,;]/ /g;
2328     $text =~ s/^\s+//;
2329     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2330 }
2331
2332 sub join_flagged_list {
2333     my ($how, $lst) = @_;
2334     join $how, map { $lst->{$_} . $_ } keys %$lst;
2335 }
2336
2337 sub vmconfig_delete_pending_option {
2338     my ($conf, $key, $force) = @_;
2339
2340     delete $conf->{pending}->{$key};
2341     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2342     $pending_delete_hash->{$key} = $force ? '!' : '';
2343     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2344 }
2345
2346 sub vmconfig_undelete_pending_option {
2347     my ($conf, $key) = @_;
2348
2349     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2350     delete $pending_delete_hash->{$key};
2351
2352     if (%$pending_delete_hash) {
2353         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2354     } else {
2355         delete $conf->{pending}->{delete};
2356     }
2357 }
2358
2359 sub vmconfig_register_unused_drive {
2360     my ($storecfg, $vmid, $conf, $drive) = @_;
2361
2362     if (drive_is_cloudinit($drive)) {
2363         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2364         warn $@ if $@;
2365     } elsif (!drive_is_cdrom($drive)) {
2366         my $volid = $drive->{file};
2367         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2368             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2369         }
2370     }
2371 }
2372
2373 sub vmconfig_cleanup_pending {
2374     my ($conf) = @_;
2375
2376     # remove pending changes when nothing changed
2377     my $changes;
2378     foreach my $opt (keys %{$conf->{pending}}) {
2379         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2380             $changes = 1;
2381             delete $conf->{pending}->{$opt};
2382         }
2383     }
2384
2385     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2386     my $pending_delete_hash = {};
2387     while (my ($opt, $force) = each %$current_delete_hash) {
2388         if (defined($conf->{$opt})) {
2389             $pending_delete_hash->{$opt} = $force;
2390         } else {
2391             $changes = 1;
2392         }
2393     }
2394
2395     if (%$pending_delete_hash) {
2396         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2397     } else {
2398         delete $conf->{pending}->{delete};
2399     }
2400
2401     return $changes;
2402 }
2403
2404 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2405 my $smbios1_fmt = {
2406     uuid => {
2407         type => 'string',
2408         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2409         format_description => 'UUID',
2410         description => "Set SMBIOS1 UUID.",
2411         optional => 1,
2412     },
2413     version => {
2414         type => 'string',
2415         pattern => '[A-Za-z0-9+\/]+={0,2}',
2416         format_description => 'Base64 encoded string',
2417         description => "Set SMBIOS1 version.",
2418         optional => 1,
2419     },
2420     serial => {
2421         type => 'string',
2422         pattern => '[A-Za-z0-9+\/]+={0,2}',
2423         format_description => 'Base64 encoded string',
2424         description => "Set SMBIOS1 serial number.",
2425         optional => 1,
2426     },
2427     manufacturer => {
2428         type => 'string',
2429         pattern => '[A-Za-z0-9+\/]+={0,2}',
2430         format_description => 'Base64 encoded string',
2431         description => "Set SMBIOS1 manufacturer.",
2432         optional => 1,
2433     },
2434     product => {
2435         type => 'string',
2436         pattern => '[A-Za-z0-9+\/]+={0,2}',
2437         format_description => 'Base64 encoded string',
2438         description => "Set SMBIOS1 product ID.",
2439         optional => 1,
2440     },
2441     sku => {
2442         type => 'string',
2443         pattern => '[A-Za-z0-9+\/]+={0,2}',
2444         format_description => 'Base64 encoded string',
2445         description => "Set SMBIOS1 SKU string.",
2446         optional => 1,
2447     },
2448     family => {
2449         type => 'string',
2450         pattern => '[A-Za-z0-9+\/]+={0,2}',
2451         format_description => 'Base64 encoded string',
2452         description => "Set SMBIOS1 family string.",
2453         optional => 1,
2454     },
2455     base64 => {
2456         type => 'boolean',
2457         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2458         optional => 1,
2459     },
2460 };
2461
2462 sub parse_smbios1 {
2463     my ($data) = @_;
2464
2465     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2466     warn $@ if $@;
2467     return $res;
2468 }
2469
2470 sub print_smbios1 {
2471     my ($smbios1) = @_;
2472     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2473 }
2474
2475 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2476
2477 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2478 sub verify_bootdisk {
2479     my ($value, $noerr) = @_;
2480
2481     return $value if is_valid_drivename($value);
2482
2483     return undef if $noerr;
2484
2485     die "invalid boot disk '$value'\n";
2486 }
2487
2488 sub parse_watchdog {
2489     my ($value) = @_;
2490
2491     return undef if !$value;
2492
2493     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2494     warn $@ if $@;
2495     return $res;
2496 }
2497
2498 sub parse_guest_agent {
2499     my ($value) = @_;
2500
2501     return {} if !defined($value->{agent});
2502
2503     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2504     warn $@ if $@;
2505
2506     # if the agent is disabled ignore the other potentially set properties
2507     return {} if !$res->{enabled};
2508     return $res;
2509 }
2510
2511 sub parse_vga {
2512     my ($value) = @_;
2513
2514     return {} if !$value;
2515     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2516     warn $@ if $@;
2517     return $res;
2518 }
2519
2520 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2521 sub verify_usb_device {
2522     my ($value, $noerr) = @_;
2523
2524     return $value if parse_usb_device($value);
2525
2526     return undef if $noerr;
2527
2528     die "unable to parse usb device\n";
2529 }
2530
2531 # add JSON properties for create and set function
2532 sub json_config_properties {
2533     my $prop = shift;
2534
2535     foreach my $opt (keys %$confdesc) {
2536         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2537         $prop->{$opt} = $confdesc->{$opt};
2538     }
2539
2540     return $prop;
2541 }
2542
2543 # return copy of $confdesc_cloudinit to generate documentation
2544 sub cloudinit_config_properties {
2545
2546     return dclone($confdesc_cloudinit);
2547 }
2548
2549 sub check_type {
2550     my ($key, $value) = @_;
2551
2552     die "unknown setting '$key'\n" if !$confdesc->{$key};
2553
2554     my $type = $confdesc->{$key}->{type};
2555
2556     if (!defined($value)) {
2557         die "got undefined value\n";
2558     }
2559
2560     if ($value =~ m/[\n\r]/) {
2561         die "property contains a line feed\n";
2562     }
2563
2564     if ($type eq 'boolean') {
2565         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2566         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2567         die "type check ('boolean') failed - got '$value'\n";
2568     } elsif ($type eq 'integer') {
2569         return int($1) if $value =~ m/^(\d+)$/;
2570         die "type check ('integer') failed - got '$value'\n";
2571     } elsif ($type eq 'number') {
2572         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2573         die "type check ('number') failed - got '$value'\n";
2574     } elsif ($type eq 'string') {
2575         if (my $fmt = $confdesc->{$key}->{format}) {
2576             PVE::JSONSchema::check_format($fmt, $value);
2577             return $value;
2578         }
2579         $value =~ s/^\"(.*)\"$/$1/;
2580         return $value;
2581     } else {
2582         die "internal error"
2583     }
2584 }
2585
2586 sub touch_config {
2587     my ($vmid) = @_;
2588
2589     my $conf = PVE::QemuConfig->config_file($vmid);
2590     utime undef, undef, $conf;
2591 }
2592
2593 sub destroy_vm {
2594     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2595
2596     my $conffile = PVE::QemuConfig->config_file($vmid);
2597
2598     my $conf = PVE::QemuConfig->load_config($vmid);
2599
2600     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2601
2602     if ($conf->{template}) {
2603         # check if any base image is still used by a linked clone
2604         foreach_drive($conf, sub {
2605                 my ($ds, $drive) = @_;
2606
2607                 return if drive_is_cdrom($drive);
2608
2609                 my $volid = $drive->{file};
2610
2611                 return if !$volid || $volid =~ m|^/|;
2612
2613                 die "base volume '$volid' is still in use by linked cloned\n"
2614                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2615
2616         });
2617     }
2618
2619     # only remove disks owned by this VM
2620     foreach_drive($conf, sub {
2621         my ($ds, $drive) = @_;
2622
2623         return if drive_is_cdrom($drive, 1);
2624
2625         my $volid = $drive->{file};
2626
2627         return if !$volid || $volid =~ m|^/|;
2628
2629         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2630         return if !$path || !$owner || ($owner != $vmid);
2631
2632         eval {
2633             PVE::Storage::vdisk_free($storecfg, $volid);
2634         };
2635         warn "Could not remove disk '$volid', check manually: $@" if $@;
2636
2637     });
2638
2639     if ($keep_empty_config) {
2640         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2641     } else {
2642         unlink $conffile;
2643     }
2644
2645     # also remove unused disk
2646     eval {
2647         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2648
2649         eval {
2650             PVE::Storage::foreach_volid($dl, sub {
2651                 my ($volid, $sid, $volname, $d) = @_;
2652                 PVE::Storage::vdisk_free($storecfg, $volid);
2653             });
2654         };
2655         warn $@ if $@;
2656
2657     };
2658     warn $@ if $@;
2659 }
2660
2661 sub parse_vm_config {
2662     my ($filename, $raw) = @_;
2663
2664     return undef if !defined($raw);
2665
2666     my $res = {
2667         digest => Digest::SHA::sha1_hex($raw),
2668         snapshots => {},
2669         pending => {},
2670     };
2671
2672     $filename =~ m|/qemu-server/(\d+)\.conf$|
2673         || die "got strange filename '$filename'";
2674
2675     my $vmid = $1;
2676
2677     my $conf = $res;
2678     my $descr;
2679     my $section = '';
2680
2681     my @lines = split(/\n/, $raw);
2682     foreach my $line (@lines) {
2683         next if $line =~ m/^\s*$/;
2684
2685         if ($line =~ m/^\[PENDING\]\s*$/i) {
2686             $section = 'pending';
2687             if (defined($descr)) {
2688                 $descr =~ s/\s+$//;
2689                 $conf->{description} = $descr;
2690             }
2691             $descr = undef;
2692             $conf = $res->{$section} = {};
2693             next;
2694
2695         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2696             $section = $1;
2697             if (defined($descr)) {
2698                 $descr =~ s/\s+$//;
2699                 $conf->{description} = $descr;
2700             }
2701             $descr = undef;
2702             $conf = $res->{snapshots}->{$section} = {};
2703             next;
2704         }
2705
2706         if ($line =~ m/^\#(.*)\s*$/) {
2707             $descr = '' if !defined($descr);
2708             $descr .= PVE::Tools::decode_text($1) . "\n";
2709             next;
2710         }
2711
2712         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2713             $descr = '' if !defined($descr);
2714             $descr .= PVE::Tools::decode_text($2);
2715         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2716             $conf->{snapstate} = $1;
2717         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2718             my $key = $1;
2719             my $value = $2;
2720             $conf->{$key} = $value;
2721         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2722             my $value = $1;
2723             if ($section eq 'pending') {
2724                 $conf->{delete} = $value; # we parse this later
2725             } else {
2726                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2727             }
2728         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2729             my $key = $1;
2730             my $value = $2;
2731             eval { $value = check_type($key, $value); };
2732             if ($@) {
2733                 warn "vm $vmid - unable to parse value of '$key' - $@";
2734             } else {
2735                 $key = 'ide2' if $key eq 'cdrom';
2736                 my $fmt = $confdesc->{$key}->{format};
2737                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2738                     my $v = parse_drive($key, $value);
2739                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2740                         $v->{file} = $volid;
2741                         $value = print_drive($vmid, $v);
2742                     } else {
2743                         warn "vm $vmid - unable to parse value of '$key'\n";
2744                         next;
2745                     }
2746                 }
2747
2748                 $conf->{$key} = $value;
2749             }
2750         }
2751     }
2752
2753     if (defined($descr)) {
2754         $descr =~ s/\s+$//;
2755         $conf->{description} = $descr;
2756     }
2757     delete $res->{snapstate}; # just to be sure
2758
2759     return $res;
2760 }
2761
2762 sub write_vm_config {
2763     my ($filename, $conf) = @_;
2764
2765     delete $conf->{snapstate}; # just to be sure
2766
2767     if ($conf->{cdrom}) {
2768         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2769         $conf->{ide2} = $conf->{cdrom};
2770         delete $conf->{cdrom};
2771     }
2772
2773     # we do not use 'smp' any longer
2774     if ($conf->{sockets}) {
2775         delete $conf->{smp};
2776     } elsif ($conf->{smp}) {
2777         $conf->{sockets} = $conf->{smp};
2778         delete $conf->{cores};
2779         delete $conf->{smp};
2780     }
2781
2782     my $used_volids = {};
2783
2784     my $cleanup_config = sub {
2785         my ($cref, $pending, $snapname) = @_;
2786
2787         foreach my $key (keys %$cref) {
2788             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2789                 $key eq 'snapstate' || $key eq 'pending';
2790             my $value = $cref->{$key};
2791             if ($key eq 'delete') {
2792                 die "propertry 'delete' is only allowed in [PENDING]\n"
2793                     if !$pending;
2794                 # fixme: check syntax?
2795                 next;
2796             }
2797             eval { $value = check_type($key, $value); };
2798             die "unable to parse value of '$key' - $@" if $@;
2799
2800             $cref->{$key} = $value;
2801
2802             if (!$snapname && is_valid_drivename($key)) {
2803                 my $drive = parse_drive($key, $value);
2804                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2805             }
2806         }
2807     };
2808
2809     &$cleanup_config($conf);
2810
2811     &$cleanup_config($conf->{pending}, 1);
2812
2813     foreach my $snapname (keys %{$conf->{snapshots}}) {
2814         die "internal error" if $snapname eq 'pending';
2815         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2816     }
2817
2818     # remove 'unusedX' settings if we re-add a volume
2819     foreach my $key (keys %$conf) {
2820         my $value = $conf->{$key};
2821         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2822             delete $conf->{$key};
2823         }
2824     }
2825
2826     my $generate_raw_config = sub {
2827         my ($conf, $pending) = @_;
2828
2829         my $raw = '';
2830
2831         # add description as comment to top of file
2832         if (defined(my $descr = $conf->{description})) {
2833             if ($descr) {
2834                 foreach my $cl (split(/\n/, $descr)) {
2835                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2836                 }
2837             } else {
2838                 $raw .= "#\n" if $pending;
2839             }
2840         }
2841
2842         foreach my $key (sort keys %$conf) {
2843             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2844             $raw .= "$key: $conf->{$key}\n";
2845         }
2846         return $raw;
2847     };
2848
2849     my $raw = &$generate_raw_config($conf);
2850
2851     if (scalar(keys %{$conf->{pending}})){
2852         $raw .= "\n[PENDING]\n";
2853         $raw .= &$generate_raw_config($conf->{pending}, 1);
2854     }
2855
2856     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2857         $raw .= "\n[$snapname]\n";
2858         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2859     }
2860
2861     return $raw;
2862 }
2863
2864 sub load_defaults {
2865
2866     my $res = {};
2867
2868     # we use static defaults from our JSON schema configuration
2869     foreach my $key (keys %$confdesc) {
2870         if (defined(my $default = $confdesc->{$key}->{default})) {
2871             $res->{$key} = $default;
2872         }
2873     }
2874
2875     return $res;
2876 }
2877
2878 sub config_list {
2879     my $vmlist = PVE::Cluster::get_vmlist();
2880     my $res = {};
2881     return $res if !$vmlist || !$vmlist->{ids};
2882     my $ids = $vmlist->{ids};
2883
2884     foreach my $vmid (keys %$ids) {
2885         my $d = $ids->{$vmid};
2886         next if !$d->{node} || $d->{node} ne $nodename;
2887         next if !$d->{type} || $d->{type} ne 'qemu';
2888         $res->{$vmid}->{exists} = 1;
2889     }
2890     return $res;
2891 }
2892
2893 # test if VM uses local resources (to prevent migration)
2894 sub check_local_resources {
2895     my ($conf, $noerr) = @_;
2896
2897     my @loc_res = ();
2898
2899     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2900     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2901
2902     push @loc_res, "ivshmem" if $conf->{ivshmem};
2903
2904     foreach my $k (keys %$conf) {
2905         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2906         # sockets are safe: they will recreated be on the target side post-migrate
2907         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2908         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2909     }
2910
2911     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2912
2913     return \@loc_res;
2914 }
2915
2916 # check if used storages are available on all nodes (use by migrate)
2917 sub check_storage_availability {
2918     my ($storecfg, $conf, $node) = @_;
2919
2920     foreach_drive($conf, sub {
2921         my ($ds, $drive) = @_;
2922
2923         my $volid = $drive->{file};
2924         return if !$volid;
2925
2926         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2927         return if !$sid;
2928
2929         # check if storage is available on both nodes
2930         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2931         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2932    });
2933 }
2934
2935 # list nodes where all VM images are available (used by has_feature API)
2936 sub shared_nodes {
2937     my ($conf, $storecfg) = @_;
2938
2939     my $nodelist = PVE::Cluster::get_nodelist();
2940     my $nodehash = { map { $_ => 1 } @$nodelist };
2941     my $nodename = PVE::INotify::nodename();
2942
2943     foreach_drive($conf, sub {
2944         my ($ds, $drive) = @_;
2945
2946         my $volid = $drive->{file};
2947         return if !$volid;
2948
2949         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2950         if ($storeid) {
2951             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2952             if ($scfg->{disable}) {
2953                 $nodehash = {};
2954             } elsif (my $avail = $scfg->{nodes}) {
2955                 foreach my $node (keys %$nodehash) {
2956                     delete $nodehash->{$node} if !$avail->{$node};
2957                 }
2958             } elsif (!$scfg->{shared}) {
2959                 foreach my $node (keys %$nodehash) {
2960                     delete $nodehash->{$node} if $node ne $nodename
2961                 }
2962             }
2963         }
2964     });
2965
2966     return $nodehash
2967 }
2968
2969 sub check_local_storage_availability {
2970     my ($conf, $storecfg) = @_;
2971
2972     my $nodelist = PVE::Cluster::get_nodelist();
2973     my $nodehash = { map { $_ => {} } @$nodelist };
2974
2975     foreach_drive($conf, sub {
2976         my ($ds, $drive) = @_;
2977
2978         my $volid = $drive->{file};
2979         return if !$volid;
2980
2981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2982         if ($storeid) {
2983             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2984
2985             if ($scfg->{disable}) {
2986                 foreach my $node (keys %$nodehash) {
2987                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2988                 }
2989             } elsif (my $avail = $scfg->{nodes}) {
2990                 foreach my $node (keys %$nodehash) {
2991                     if (!$avail->{$node}) {
2992                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2993                     }
2994                 }
2995             }
2996         }
2997     });
2998
2999     foreach my $node (values %$nodehash) {
3000         if (my $unavail = $node->{unavailable_storages}) {
3001             $node->{unavailable_storages} = [ sort keys %$unavail ];
3002         }
3003     }
3004
3005     return $nodehash
3006 }
3007
3008 sub check_cmdline {
3009     my ($pidfile, $pid) = @_;
3010
3011     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3012     if (defined($fh)) {
3013         my $line = <$fh>;
3014         $fh->close;
3015         return undef if !$line;
3016         my @param = split(/\0/, $line);
3017
3018         my $cmd = $param[0];
3019         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3020
3021         for (my $i = 0; $i < scalar (@param); $i++) {
3022             my $p = $param[$i];
3023             next if !$p;
3024             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3025                 my $p = $param[$i+1];
3026                 return 1 if $p && ($p eq $pidfile);
3027                 return undef;
3028             }
3029         }
3030     }
3031     return undef;
3032 }
3033
3034 sub check_running {
3035     my ($vmid, $nocheck, $node) = @_;
3036
3037     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3038
3039     die "unable to find configuration file for VM $vmid - no such machine\n"
3040         if !$nocheck && ! -f $filename;
3041
3042     my $pidfile = pidfile_name($vmid);
3043
3044     if (my $fd = IO::File->new("<$pidfile")) {
3045         my $st = stat($fd);
3046         my $line = <$fd>;
3047         close($fd);
3048
3049         my $mtime = $st->mtime;
3050         if ($mtime > time()) {
3051             warn "file '$filename' modified in future\n";
3052         }
3053
3054         if ($line =~ m/^(\d+)$/) {
3055             my $pid = $1;
3056             if (check_cmdline($pidfile, $pid)) {
3057                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3058                     return $pid;
3059                 }
3060             }
3061         }
3062     }
3063
3064     return undef;
3065 }
3066
3067 sub vzlist {
3068
3069     my $vzlist = config_list();
3070
3071     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3072
3073     while (defined(my $de = $fd->read)) {
3074         next if $de !~ m/^(\d+)\.pid$/;
3075         my $vmid = $1;
3076         next if !defined($vzlist->{$vmid});
3077         if (my $pid = check_running($vmid)) {
3078             $vzlist->{$vmid}->{pid} = $pid;
3079         }
3080     }
3081
3082     return $vzlist;
3083 }
3084
3085 sub disksize {
3086     my ($storecfg, $conf) = @_;
3087
3088     my $bootdisk = $conf->{bootdisk};
3089     return undef if !$bootdisk;
3090     return undef if !is_valid_drivename($bootdisk);
3091
3092     return undef if !$conf->{$bootdisk};
3093
3094     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3095     return undef if !defined($drive);
3096
3097     return undef if drive_is_cdrom($drive);
3098
3099     my $volid = $drive->{file};
3100     return undef if !$volid;
3101
3102     return $drive->{size};
3103 }
3104
3105 our $vmstatus_return_properties = {
3106     vmid => get_standard_option('pve-vmid'),
3107     status => {
3108         description => "Qemu process status.",
3109         type => 'string',
3110         enum => ['stopped', 'running'],
3111     },
3112     maxmem => {
3113         description => "Maximum memory in bytes.",
3114         type => 'integer',
3115         optional => 1,
3116         renderer => 'bytes',
3117     },
3118     maxdisk => {
3119         description => "Root disk size in bytes.",
3120         type => 'integer',
3121         optional => 1,
3122         renderer => 'bytes',
3123     },
3124     name => {
3125         description => "VM name.",
3126         type => 'string',
3127         optional => 1,
3128     },
3129     qmpstatus => {
3130         description => "Qemu QMP agent status.",
3131         type => 'string',
3132         optional => 1,
3133     },
3134     pid => {
3135         description => "PID of running qemu process.",
3136         type => 'integer',
3137         optional => 1,
3138     },
3139     uptime => {
3140         description => "Uptime.",
3141         type => 'integer',
3142         optional => 1,
3143         renderer => 'duration',
3144     },
3145     cpus => {
3146         description => "Maximum usable CPUs.",
3147         type => 'number',
3148         optional => 1,
3149     },
3150     lock => {
3151         description => "The current config lock, if any.",
3152         type => 'string',
3153         optional => 1,
3154     }
3155 };
3156
3157 my $last_proc_pid_stat;
3158
3159 # get VM status information
3160 # This must be fast and should not block ($full == false)
3161 # We only query KVM using QMP if $full == true (this can be slow)
3162 sub vmstatus {
3163     my ($opt_vmid, $full) = @_;
3164
3165     my $res = {};
3166
3167     my $storecfg = PVE::Storage::config();
3168
3169     my $list = vzlist();
3170     my $defaults = load_defaults();
3171
3172     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3173
3174     my $cpucount = $cpuinfo->{cpus} || 1;
3175
3176     foreach my $vmid (keys %$list) {
3177         next if $opt_vmid && ($vmid ne $opt_vmid);
3178
3179         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3180         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3181
3182         my $d = { vmid => $vmid };
3183         $d->{pid} = $list->{$vmid}->{pid};
3184
3185         # fixme: better status?
3186         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3187
3188         my $size = disksize($storecfg, $conf);
3189         if (defined($size)) {
3190             $d->{disk} = 0; # no info available
3191             $d->{maxdisk} = $size;
3192         } else {
3193             $d->{disk} = 0;
3194             $d->{maxdisk} = 0;
3195         }
3196
3197         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3198             * ($conf->{cores} || $defaults->{cores});
3199         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3200         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3201
3202         $d->{name} = $conf->{name} || "VM $vmid";
3203         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3204             : $defaults->{memory}*(1024*1024);
3205
3206         if ($conf->{balloon}) {
3207             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3208             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3209                 : $defaults->{shares};
3210         }
3211
3212         $d->{uptime} = 0;
3213         $d->{cpu} = 0;
3214         $d->{mem} = 0;
3215
3216         $d->{netout} = 0;
3217         $d->{netin} = 0;
3218
3219         $d->{diskread} = 0;
3220         $d->{diskwrite} = 0;
3221
3222         $d->{template} = PVE::QemuConfig->is_template($conf);
3223
3224         $d->{serial} = 1 if conf_has_serial($conf);
3225         $d->{lock} = $conf->{lock} if $conf->{lock};
3226
3227         $res->{$vmid} = $d;
3228     }
3229
3230     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3231     foreach my $dev (keys %$netdev) {
3232         next if $dev !~ m/^tap([1-9]\d*)i/;
3233         my $vmid = $1;
3234         my $d = $res->{$vmid};
3235         next if !$d;
3236
3237         $d->{netout} += $netdev->{$dev}->{receive};
3238         $d->{netin} += $netdev->{$dev}->{transmit};
3239
3240         if ($full) {
3241             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3242             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3243         }
3244
3245     }
3246
3247     my $ctime = gettimeofday;
3248
3249     foreach my $vmid (keys %$list) {
3250
3251         my $d = $res->{$vmid};
3252         my $pid = $d->{pid};
3253         next if !$pid;
3254
3255         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3256         next if !$pstat; # not running
3257
3258         my $used = $pstat->{utime} + $pstat->{stime};
3259
3260         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3261
3262         if ($pstat->{vsize}) {
3263             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3264         }
3265
3266         my $old = $last_proc_pid_stat->{$pid};
3267         if (!$old) {
3268             $last_proc_pid_stat->{$pid} = {
3269                 time => $ctime,
3270                 used => $used,
3271                 cpu => 0,
3272             };
3273             next;
3274         }
3275
3276         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3277
3278         if ($dtime > 1000) {
3279             my $dutime = $used -  $old->{used};
3280
3281             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3282             $last_proc_pid_stat->{$pid} = {
3283                 time => $ctime,
3284                 used => $used,
3285                 cpu => $d->{cpu},
3286             };
3287         } else {
3288             $d->{cpu} = $old->{cpu};
3289         }
3290     }
3291
3292     return $res if !$full;
3293
3294     my $qmpclient = PVE::QMPClient->new();
3295
3296     my $ballooncb = sub {
3297         my ($vmid, $resp) = @_;
3298
3299         my $info = $resp->{'return'};
3300         return if !$info->{max_mem};
3301
3302         my $d = $res->{$vmid};
3303
3304         # use memory assigned to VM
3305         $d->{maxmem} = $info->{max_mem};
3306         $d->{balloon} = $info->{actual};
3307
3308         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3309             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3310             $d->{freemem} = $info->{free_mem};
3311         }
3312
3313         $d->{ballooninfo} = $info;
3314     };
3315
3316     my $blockstatscb = sub {
3317         my ($vmid, $resp) = @_;
3318         my $data = $resp->{'return'} || [];
3319         my $totalrdbytes = 0;
3320         my $totalwrbytes = 0;
3321
3322         for my $blockstat (@$data) {
3323             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3324             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3325
3326             $blockstat->{device} =~ s/drive-//;
3327             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3328         }
3329         $res->{$vmid}->{diskread} = $totalrdbytes;
3330         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3331     };
3332
3333     my $statuscb = sub {
3334         my ($vmid, $resp) = @_;
3335
3336         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3337         # this fails if ballon driver is not loaded, so this must be
3338         # the last commnand (following command are aborted if this fails).
3339         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3340
3341         my $status = 'unknown';
3342         if (!defined($status = $resp->{'return'}->{status})) {
3343             warn "unable to get VM status\n";
3344             return;
3345         }
3346
3347         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3348     };
3349
3350     foreach my $vmid (keys %$list) {
3351         next if $opt_vmid && ($vmid ne $opt_vmid);
3352         next if !$res->{$vmid}->{pid}; # not running
3353         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3354     }
3355
3356     $qmpclient->queue_execute(undef, 2);
3357
3358     foreach my $vmid (keys %$list) {
3359         next if $opt_vmid && ($vmid ne $opt_vmid);
3360         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3361     }
3362
3363     return $res;
3364 }
3365
3366 sub foreach_drive {
3367     my ($conf, $func, @param) = @_;
3368
3369     foreach my $ds (valid_drive_names()) {
3370         next if !defined($conf->{$ds});
3371
3372         my $drive = parse_drive($ds, $conf->{$ds});
3373         next if !$drive;
3374
3375         &$func($ds, $drive, @param);
3376     }
3377 }
3378
3379 sub foreach_volid {
3380     my ($conf, $func, @param) = @_;
3381
3382     my $volhash = {};
3383
3384     my $test_volid = sub {
3385         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3386
3387         return if !$volid;
3388
3389         $volhash->{$volid}->{cdrom} //= 1;
3390         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3391
3392         $volhash->{$volid}->{replicate} //= 0;
3393         $volhash->{$volid}->{replicate} = 1 if $replicate;
3394
3395         $volhash->{$volid}->{shared} //= 0;
3396         $volhash->{$volid}->{shared} = 1 if $shared;
3397
3398         $volhash->{$volid}->{referenced_in_config} //= 0;
3399         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3400
3401         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3402             if defined($snapname);
3403         $volhash->{$volid}->{size} = $size if $size;
3404     };
3405
3406     foreach_drive($conf, sub {
3407         my ($ds, $drive) = @_;
3408         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3409     });
3410
3411     foreach my $snapname (keys %{$conf->{snapshots}}) {
3412         my $snap = $conf->{snapshots}->{$snapname};
3413         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3414         foreach_drive($snap, sub {
3415             my ($ds, $drive) = @_;
3416             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3417         });
3418     }
3419
3420     foreach my $volid (keys %$volhash) {
3421         &$func($volid, $volhash->{$volid}, @param);
3422     }
3423 }
3424
3425 sub conf_has_serial {
3426     my ($conf) = @_;
3427
3428     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3429         if ($conf->{"serial$i"}) {
3430             return 1;
3431         }
3432     }
3433
3434     return 0;
3435 }
3436
3437 sub conf_has_audio {
3438     my ($conf, $id) = @_;
3439
3440     $id //= 0;
3441     my $audio = $conf->{"audio$id"};
3442     return undef if !defined($audio);
3443
3444     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3445     my $audiodriver = $audioproperties->{driver} // 'spice';
3446
3447     return {
3448         dev => $audioproperties->{device},
3449         dev_id => "audiodev$id",
3450         backend => $audiodriver,
3451         backend_id => "$audiodriver-backend${id}",
3452     };
3453 }
3454
3455 sub vga_conf_has_spice {
3456     my ($vga) = @_;
3457
3458     my $vgaconf = parse_vga($vga);
3459     my $vgatype = $vgaconf->{type};
3460     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3461
3462     return $1 || 1;
3463 }
3464
3465 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3466 sub get_host_arch() {
3467     $host_arch = (POSIX::uname())[4] if !$host_arch;
3468     return $host_arch;
3469 }
3470
3471 sub is_native($) {
3472     my ($arch) = @_;
3473     return get_host_arch() eq $arch;
3474 }
3475
3476 my $default_machines = {
3477     x86_64 => 'pc',
3478     aarch64 => 'virt',
3479 };
3480
3481 sub get_basic_machine_info {
3482     my ($conf, $forcemachine) = @_;
3483
3484     my $arch = $conf->{arch} // get_host_arch();
3485     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3486     return ($arch, $machine);
3487 }
3488
3489 sub get_ovmf_files($) {
3490     my ($arch) = @_;
3491
3492     my $ovmf = $OVMF->{$arch}
3493         or die "no OVMF images known for architecture '$arch'\n";
3494
3495     return @$ovmf;
3496 }
3497
3498 my $Arch2Qemu = {
3499     aarch64 => '/usr/bin/qemu-system-aarch64',
3500     x86_64 => '/usr/bin/qemu-system-x86_64',
3501 };
3502 sub get_command_for_arch($) {
3503     my ($arch) = @_;
3504     return '/usr/bin/kvm' if is_native($arch);
3505
3506     my $cmd = $Arch2Qemu->{$arch}
3507         or die "don't know how to emulate architecture '$arch'\n";
3508     return $cmd;
3509 }
3510
3511 sub get_cpu_options {
3512     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3513
3514     my $cpuFlags = [];
3515     my $ostype = $conf->{ostype};
3516
3517     my $cpu = $kvm ? "kvm64" : "qemu64";
3518     if ($arch eq 'aarch64') {
3519         $cpu = 'cortex-a57';
3520     }
3521     my $hv_vendor_id;
3522     if (my $cputype = $conf->{cpu}) {
3523         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3524             or die "Cannot parse cpu description: $cputype\n";
3525         $cpu = $cpuconf->{cputype};
3526         $kvm_off = 1 if $cpuconf->{hidden};
3527         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3528
3529         if (defined(my $flags = $cpuconf->{flags})) {
3530             push @$cpuFlags, split(";", $flags);
3531         }
3532     }
3533
3534     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3535
3536     push @$cpuFlags , '-x2apic'
3537         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3538
3539     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3540
3541     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3542
3543     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3544
3545         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3546         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3547     }
3548
3549     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3550
3551     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3552
3553     push @$cpuFlags, 'kvm=off' if $kvm_off;
3554
3555     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3556         push @$cpuFlags, "vendor=${cpu_vendor}"
3557             if $cpu_vendor ne 'default';
3558     } elsif ($arch ne 'aarch64') {
3559         die "internal error"; # should not happen
3560     }
3561
3562     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3563
3564     return ('-cpu', $cpu);
3565 }
3566
3567 sub config_to_command {
3568     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3569
3570     my $cmd = [];
3571     my $globalFlags = [];
3572     my $machineFlags = [];
3573     my $rtcFlags = [];
3574     my $devices = [];
3575     my $pciaddr = '';
3576     my $bridges = {};
3577     my $vernum = 0; # unknown
3578     my $ostype = $conf->{ostype};
3579     my $winversion = windows_version($ostype);
3580     my $kvm = $conf->{kvm};
3581
3582     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3583     my $kvm_binary = get_command_for_arch($arch);
3584     my $kvmver = kvm_user_version($kvm_binary);
3585     $kvm //= 1 if is_native($arch);
3586
3587     if ($kvm) {
3588         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3589             if !defined kvm_version();
3590     }
3591
3592     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3593         $vernum = $1*1000000+$2*1000;
3594     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3595         $vernum = $1*1000000+$2*1000+$3;
3596     }
3597
3598     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3599
3600     my $have_ovz = -f '/proc/vz/vestat';
3601
3602     my $q35 = machine_type_is_q35($conf);
3603     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3604     my $use_old_bios_files = undef;
3605     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3606
3607     my $cpuunits = defined($conf->{cpuunits}) ?
3608             $conf->{cpuunits} : $defaults->{cpuunits};
3609
3610     push @$cmd, $kvm_binary;
3611
3612     push @$cmd, '-id', $vmid;
3613
3614     my $vmname = $conf->{name} || "vm$vmid";
3615
3616     push @$cmd, '-name', $vmname;
3617
3618     my $use_virtio = 0;
3619
3620     my $qmpsocket = qmp_socket($vmid);
3621     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3622     push @$cmd, '-mon', "chardev=qmp,mode=control";
3623
3624     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3625         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3626         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3627     }
3628
3629     push @$cmd, '-pidfile' , pidfile_name($vmid);
3630
3631     push @$cmd, '-daemonize';
3632
3633     if ($conf->{smbios1}) {
3634         my $smbios_conf = parse_smbios1($conf->{smbios1});
3635         if ($smbios_conf->{base64}) {
3636             # Do not pass base64 flag to qemu
3637             delete $smbios_conf->{base64};
3638             my $smbios_string = "";
3639             foreach my $key (keys %$smbios_conf) {
3640                 my $value;
3641                 if ($key eq "uuid") {
3642                     $value = $smbios_conf->{uuid}
3643                 } else {
3644                     $value = decode_base64($smbios_conf->{$key});
3645                 }
3646                 # qemu accepts any binary data, only commas need escaping by double comma
3647                 $value =~ s/,/,,/g;
3648                 $smbios_string .= "," . $key . "=" . $value if $value;
3649             }
3650             push @$cmd, '-smbios', "type=1" . $smbios_string;
3651         } else {
3652             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3653         }
3654     }
3655
3656     if ($conf->{vmgenid}) {
3657         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3658     }
3659
3660     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3661     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3662         die "uefi base image not found\n" if ! -f $ovmf_code;
3663
3664         my $path;
3665         my $format;
3666         if (my $efidisk = $conf->{efidisk0}) {
3667             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3668             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3669             $format = $d->{format};
3670             if ($storeid) {
3671                 $path = PVE::Storage::path($storecfg, $d->{file});
3672                 if (!defined($format)) {
3673                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3674                     $format = qemu_img_format($scfg, $volname);
3675                 }
3676             } else {
3677                 $path = $d->{file};
3678                 die "efidisk format must be specified\n"
3679                     if !defined($format);
3680             }
3681         } else {
3682             warn "no efidisk configured! Using temporary efivars disk.\n";
3683             $path = "/tmp/$vmid-ovmf.fd";
3684             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3685             $format = 'raw';
3686         }
3687
3688         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3689         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3690     }
3691
3692     # load q35 config
3693     if ($q35) {
3694         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3695         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3696             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3697         } else {
3698             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3699         }
3700     }
3701
3702     # add usb controllers
3703     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3704     push @$devices, @usbcontrollers if @usbcontrollers;
3705     my $vga = parse_vga($conf->{vga});
3706
3707     my $qxlnum = vga_conf_has_spice($conf->{vga});
3708     $vga->{type} = 'qxl' if $qxlnum;
3709
3710     if (!$vga->{type}) {
3711         if ($arch eq 'aarch64') {
3712             $vga->{type} = 'virtio';
3713         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3714             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3715         } else {
3716             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3717         }
3718     }
3719
3720     # enable absolute mouse coordinates (needed by vnc)
3721     my $tablet;
3722     if (defined($conf->{tablet})) {
3723         $tablet = $conf->{tablet};
3724     } else {
3725         $tablet = $defaults->{tablet};
3726         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3727         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3728     }
3729
3730     if ($tablet) {
3731         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3732         my $kbd = print_keyboarddevice_full($conf, $arch);
3733         push @$devices, '-device', $kbd if defined($kbd);
3734     }
3735
3736     my $kvm_off = 0;
3737     my $gpu_passthrough;
3738
3739     # host pci devices
3740     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3741         my $d = parse_hostpci($conf->{"hostpci$i"});
3742         next if !$d;
3743
3744         my $pcie = $d->{pcie};
3745         if ($pcie) {
3746             die "q35 machine model is not enabled" if !$q35;
3747             # win7 wants to have the pcie devices directly on the pcie bus
3748             # instead of in the root port
3749             if ($winversion == 7) {
3750                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3751             } else {
3752                 $pciaddr = print_pcie_addr("hostpci$i");
3753             }
3754         } else {
3755             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3756         }
3757
3758         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3759         my $romfile = $d->{romfile};
3760
3761         my $xvga = '';
3762         if ($d->{'x-vga'}) {
3763             $xvga = ',x-vga=on';
3764             $kvm_off = 1;
3765             $vga->{type} = 'none' if !defined($conf->{vga});
3766             $gpu_passthrough = 1;
3767
3768             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3769                 $xvga = "";
3770             }
3771         }
3772         my $pcidevices = $d->{pciid};
3773         my $multifunction = 1 if @$pcidevices > 1;
3774         my $sysfspath;
3775         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3776             my $id = $pcidevices->[0]->{id};
3777             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3778             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3779         } elsif ($d->{mdev}) {
3780             warn "ignoring mediated device with multifunction device\n";
3781         }
3782
3783         my $j=0;
3784         foreach my $pcidevice (@$pcidevices) {
3785
3786             my $id = "hostpci$i";
3787             $id .= ".$j" if $multifunction;
3788             my $addr = $pciaddr;
3789             $addr .= ".$j" if $multifunction;
3790             my $devicestr = "vfio-pci";
3791             if ($sysfspath) {
3792                 $devicestr .= ",sysfsdev=$sysfspath";
3793             } else {
3794                 $devicestr .= ",host=$pcidevice->{id}";
3795             }
3796             $devicestr .= ",id=$id$addr";
3797
3798             if($j == 0){
3799                 $devicestr .= "$rombar$xvga";
3800                 $devicestr .= ",multifunction=on" if $multifunction;
3801                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3802             }
3803
3804             push @$devices, '-device', $devicestr;
3805             $j++;
3806         }
3807     }
3808
3809     # usb devices
3810     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3811     push @$devices, @usbdevices if @usbdevices;
3812     # serial devices
3813     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3814         if (my $path = $conf->{"serial$i"}) {
3815             if ($path eq 'socket') {
3816                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3817                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3818                 # On aarch64, serial0 is the UART device. Qemu only allows
3819                 # connecting UART devices via the '-serial' command line, as
3820                 # the device has a fixed slot on the hardware...
3821                 if ($arch eq 'aarch64' && $i == 0) {
3822                     push @$devices, '-serial', "chardev:serial$i";
3823                 } else {
3824                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3825                 }
3826             } else {
3827                 die "no such serial device\n" if ! -c $path;
3828                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3829                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3830             }
3831         }
3832     }
3833
3834     # parallel devices
3835     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3836         if (my $path = $conf->{"parallel$i"}) {
3837             die "no such parallel device\n" if ! -c $path;
3838             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3839             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3840             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3841         }
3842     }
3843
3844     if (my $audio = conf_has_audio($conf)) {
3845
3846         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3847
3848         my $id = $audio->{dev_id};
3849         if ($audio->{dev} eq 'AC97') {
3850             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3851         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3852             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3853             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3854             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3855         } else {
3856             die "unkown audio device '$audio->{dev}', implement me!";
3857         }
3858
3859         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3860     }
3861
3862     my $sockets = 1;
3863     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3864     $sockets = $conf->{sockets} if  $conf->{sockets};
3865
3866     my $cores = $conf->{cores} || 1;
3867
3868     my $maxcpus = $sockets * $cores;
3869
3870     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3871
3872     my $allowed_vcpus = $cpuinfo->{cpus};
3873
3874     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3875         if ($allowed_vcpus < $maxcpus);
3876
3877     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3878
3879         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3880         for (my $i = 2; $i <= $vcpus; $i++)  {
3881             my $cpustr = print_cpu_device($conf,$i);
3882             push @$cmd, '-device', $cpustr;
3883         }
3884
3885     } else {
3886
3887         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3888     }
3889     push @$cmd, '-nodefaults';
3890
3891     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3892
3893     my $bootindex_hash = {};
3894     my $i = 1;
3895     foreach my $o (split(//, $bootorder)) {
3896         $bootindex_hash->{$o} = $i*100;
3897         $i++;
3898     }
3899
3900     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3901
3902     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3903
3904     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3905
3906     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3907         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3908         my $socket = vnc_socket($vmid);
3909         push @$cmd,  '-vnc', "unix:$socket,password";
3910     } else {
3911         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3912         push @$cmd, '-nographic';
3913     }
3914
3915     # time drift fix
3916     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3917
3918     my $useLocaltime = $conf->{localtime};
3919
3920     if ($winversion >= 5) { # windows
3921         $useLocaltime = 1 if !defined($conf->{localtime});
3922
3923         # use time drift fix when acpi is enabled
3924         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3925             $tdf = 1 if !defined($conf->{tdf});
3926         }
3927     }
3928
3929     if ($winversion >= 6) {
3930         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3931         push @$cmd, '-no-hpet';
3932     }
3933
3934     push @$rtcFlags, 'driftfix=slew' if $tdf;
3935
3936     if (!$kvm) {
3937         push @$machineFlags, 'accel=tcg';
3938     }
3939
3940     if ($machine_type) {
3941         push @$machineFlags, "type=${machine_type}";
3942     }
3943
3944     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3945         push @$rtcFlags, "base=$conf->{startdate}";
3946     } elsif ($useLocaltime) {
3947         push @$rtcFlags, 'base=localtime';
3948     }
3949
3950     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3951
3952     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3953
3954     push @$cmd, '-S' if $conf->{freeze};
3955
3956     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3957
3958     if (parse_guest_agent($conf)->{enabled}) {
3959         my $qgasocket = qmp_socket($vmid, 1);
3960         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3961         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3962         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3963         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3964     }
3965
3966     my $spice_port;
3967
3968     if ($qxlnum) {
3969         if ($qxlnum > 1) {
3970             if ($winversion){
3971                 for(my $i = 1; $i < $qxlnum; $i++){
3972                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3973                 }
3974             } else {
3975                 # assume other OS works like Linux
3976                 my ($ram, $vram) = ("134217728", "67108864");
3977                 if ($vga->{memory}) {
3978                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3979                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3980                 }
3981                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3982                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3983             }
3984         }
3985
3986         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3987
3988         my $nodename = PVE::INotify::nodename();
3989         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3990         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3991         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3992         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3993         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3994
3995         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3996
3997         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3998         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3999         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4000     }
4001
4002     # enable balloon by default, unless explicitly disabled
4003     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4004         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4005         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4006     }
4007
4008     if ($conf->{watchdog}) {
4009         my $wdopts = parse_watchdog($conf->{watchdog});
4010         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4011         my $watchdog = $wdopts->{model} || 'i6300esb';
4012         push @$devices, '-device', "$watchdog$pciaddr";
4013         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4014     }
4015
4016     my $vollist = [];
4017     my $scsicontroller = {};
4018     my $ahcicontroller = {};
4019     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4020
4021     # Add iscsi initiator name if available
4022     if (my $initiator = get_initiator_name()) {
4023         push @$devices, '-iscsi', "initiator-name=$initiator";
4024     }
4025
4026     foreach_drive($conf, sub {
4027         my ($ds, $drive) = @_;
4028
4029         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4030             push @$vollist, $drive->{file};
4031         }
4032
4033         # ignore efidisk here, already added in bios/fw handling code above
4034         return if $drive->{interface} eq 'efidisk';
4035
4036         $use_virtio = 1 if $ds =~ m/^virtio/;
4037
4038         if (drive_is_cdrom ($drive)) {
4039             if ($bootindex_hash->{d}) {
4040                 $drive->{bootindex} = $bootindex_hash->{d};
4041                 $bootindex_hash->{d} += 1;
4042             }
4043         } else {
4044             if ($bootindex_hash->{c}) {
4045                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4046                 $bootindex_hash->{c} += 1;
4047             }
4048         }
4049
4050         if($drive->{interface} eq 'virtio'){
4051            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4052         }
4053
4054         if ($drive->{interface} eq 'scsi') {
4055
4056             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4057
4058             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4059             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4060
4061             my $iothread = '';
4062             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4063                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4064                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4065             } elsif ($drive->{iothread}) {
4066                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4067             }
4068
4069             my $queues = '';
4070             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4071                 $queues = ",num_queues=$drive->{queues}";
4072             }
4073
4074             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4075             $scsicontroller->{$controller}=1;
4076         }
4077
4078         if ($drive->{interface} eq 'sata') {
4079            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4080            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4081            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4082            $ahcicontroller->{$controller}=1;
4083         }
4084
4085         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4086         push @$devices, '-drive',$drive_cmd;
4087         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4088     });
4089
4090     for (my $i = 0; $i < $MAX_NETS; $i++) {
4091          next if !$conf->{"net$i"};
4092          my $d = parse_net($conf->{"net$i"});
4093          next if !$d;
4094
4095          $use_virtio = 1 if $d->{model} eq 'virtio';
4096
4097          if ($bootindex_hash->{n}) {
4098             $d->{bootindex} = $bootindex_hash->{n};
4099             $bootindex_hash->{n} += 1;
4100          }
4101
4102          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4103          push @$devices, '-netdev', $netdevfull;
4104
4105          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4106          push @$devices, '-device', $netdevicefull;
4107     }
4108
4109     if ($conf->{ivshmem}) {
4110         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4111
4112         my $bus;
4113         if ($q35) {
4114             $bus = print_pcie_addr("ivshmem");
4115         } else {
4116             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4117         }
4118
4119         my $ivshmem_name = $ivshmem->{name} // $vmid;
4120         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4121
4122         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4123         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4124     }
4125
4126     if (!$q35) {
4127         # add pci bridges
4128         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4129            $bridges->{1} = 1;
4130            $bridges->{2} = 1;
4131         }
4132
4133         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4134
4135         while (my ($k, $v) = each %$bridges) {
4136             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4137             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4138         }
4139     }
4140
4141     push @$cmd, @$devices;
4142     push @$cmd, '-rtc', join(',', @$rtcFlags)
4143         if scalar(@$rtcFlags);
4144     push @$cmd, '-machine', join(',', @$machineFlags)
4145         if scalar(@$machineFlags);
4146     push @$cmd, '-global', join(',', @$globalFlags)
4147         if scalar(@$globalFlags);
4148
4149     if (my $vmstate = $conf->{vmstate}) {
4150         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4151         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4152         push @$cmd, '-loadstate', $statepath;
4153     }
4154
4155     # add custom args
4156     if ($conf->{args}) {
4157         my $aa = PVE::Tools::split_args($conf->{args});
4158         push @$cmd, @$aa;
4159     }
4160
4161     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4162 }
4163
4164 sub vnc_socket {
4165     my ($vmid) = @_;
4166     return "${var_run_tmpdir}/$vmid.vnc";
4167 }
4168
4169 sub spice_port {
4170     my ($vmid) = @_;
4171
4172     my $res = vm_mon_cmd($vmid, 'query-spice');
4173
4174     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4175 }
4176
4177 sub qmp_socket {
4178     my ($vmid, $qga, $name) = @_;
4179     my $sockettype = $qga ? 'qga' : 'qmp';
4180     my $ext = $name ? '-'.$name : '';
4181     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4182 }
4183
4184 sub pidfile_name {
4185     my ($vmid) = @_;
4186     return "${var_run_tmpdir}/$vmid.pid";
4187 }
4188
4189 sub vm_devices_list {
4190     my ($vmid) = @_;
4191
4192     my $res = vm_mon_cmd($vmid, 'query-pci');
4193     my $devices_to_check = [];
4194     my $devices = {};
4195     foreach my $pcibus (@$res) {
4196         push @$devices_to_check, @{$pcibus->{devices}},
4197     }
4198
4199     while (@$devices_to_check) {
4200         my $to_check = [];
4201         for my $d (@$devices_to_check) {
4202             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4203             next if !$d->{'pci_bridge'};
4204
4205             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4206             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4207         }
4208         $devices_to_check = $to_check;
4209     }
4210
4211     my $resblock = vm_mon_cmd($vmid, 'query-block');
4212     foreach my $block (@$resblock) {
4213         if($block->{device} =~ m/^drive-(\S+)/){
4214                 $devices->{$1} = 1;
4215         }
4216     }
4217
4218     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4219     foreach my $mice (@$resmice) {
4220         if ($mice->{name} eq 'QEMU HID Tablet') {
4221             $devices->{tablet} = 1;
4222             last;
4223         }
4224     }
4225
4226     # for usb devices there is no query-usb
4227     # but we can iterate over the entries in
4228     # qom-list path=/machine/peripheral
4229     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4230     foreach my $per (@$resperipheral) {
4231         if ($per->{name} =~ m/^usb\d+$/) {
4232             $devices->{$per->{name}} = 1;
4233         }
4234     }
4235
4236     return $devices;
4237 }
4238
4239 sub vm_deviceplug {
4240     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4241
4242     my $q35 = machine_type_is_q35($conf);
4243
4244     my $devices_list = vm_devices_list($vmid);
4245     return 1 if defined($devices_list->{$deviceid});
4246
4247     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4248
4249     if ($deviceid eq 'tablet') {
4250
4251         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4252
4253     } elsif ($deviceid eq 'keyboard') {
4254
4255         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4256
4257     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4258
4259         die "usb hotplug currently not reliable\n";
4260         # since we can't reliably hot unplug all added usb devices
4261         # and usb passthrough disables live migration
4262         # we disable usb hotplugging for now
4263         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4264
4265     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4266
4267         qemu_iothread_add($vmid, $deviceid, $device);
4268
4269         qemu_driveadd($storecfg, $vmid, $device);
4270         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4271
4272         qemu_deviceadd($vmid, $devicefull);
4273         eval { qemu_deviceaddverify($vmid, $deviceid); };
4274         if (my $err = $@) {
4275             eval { qemu_drivedel($vmid, $deviceid); };
4276             warn $@ if $@;
4277             die $err;
4278         }
4279
4280     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4281
4282
4283         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4284         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4285         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4286
4287         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4288
4289         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4290             qemu_iothread_add($vmid, $deviceid, $device);
4291             $devicefull .= ",iothread=iothread-$deviceid";
4292         }
4293
4294         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4295             $devicefull .= ",num_queues=$device->{queues}";
4296         }
4297
4298         qemu_deviceadd($vmid, $devicefull);
4299         qemu_deviceaddverify($vmid, $deviceid);
4300
4301     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4302
4303         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4304         qemu_driveadd($storecfg, $vmid, $device);
4305
4306         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4307         eval { qemu_deviceadd($vmid, $devicefull); };
4308         if (my $err = $@) {
4309             eval { qemu_drivedel($vmid, $deviceid); };
4310             warn $@ if $@;
4311             die $err;
4312         }
4313
4314     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4315
4316         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4317
4318         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4319         my $use_old_bios_files = undef;
4320         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4321
4322         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4323         qemu_deviceadd($vmid, $netdevicefull);
4324         eval {
4325             qemu_deviceaddverify($vmid, $deviceid);
4326             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4327         };
4328         if (my $err = $@) {
4329             eval { qemu_netdevdel($vmid, $deviceid); };
4330             warn $@ if $@;
4331             die $err;
4332         }
4333
4334     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4335
4336         my $bridgeid = $2;
4337         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4338         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4339
4340         qemu_deviceadd($vmid, $devicefull);
4341         qemu_deviceaddverify($vmid, $deviceid);
4342
4343     } else {
4344         die "can't hotplug device '$deviceid'\n";
4345     }
4346
4347     return 1;
4348 }
4349
4350 # fixme: this should raise exceptions on error!
4351 sub vm_deviceunplug {
4352     my ($vmid, $conf, $deviceid) = @_;
4353
4354     my $devices_list = vm_devices_list($vmid);
4355     return 1 if !defined($devices_list->{$deviceid});
4356
4357     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4358
4359     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4360
4361         qemu_devicedel($vmid, $deviceid);
4362
4363     } elsif ($deviceid =~ m/^usb\d+$/) {
4364
4365         die "usb hotplug currently not reliable\n";
4366         # when unplugging usb devices this way,
4367         # there may be remaining usb controllers/hubs
4368         # so we disable it for now
4369         qemu_devicedel($vmid, $deviceid);
4370         qemu_devicedelverify($vmid, $deviceid);
4371
4372     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4373
4374         qemu_devicedel($vmid, $deviceid);
4375         qemu_devicedelverify($vmid, $deviceid);
4376         qemu_drivedel($vmid, $deviceid);
4377         qemu_iothread_del($conf, $vmid, $deviceid);
4378
4379     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4380
4381         qemu_devicedel($vmid, $deviceid);
4382         qemu_devicedelverify($vmid, $deviceid);
4383         qemu_iothread_del($conf, $vmid, $deviceid);
4384
4385     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4386
4387         qemu_devicedel($vmid, $deviceid);
4388         qemu_drivedel($vmid, $deviceid);
4389         qemu_deletescsihw($conf, $vmid, $deviceid);
4390
4391     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4392
4393         qemu_devicedel($vmid, $deviceid);
4394         qemu_devicedelverify($vmid, $deviceid);
4395         qemu_netdevdel($vmid, $deviceid);
4396
4397     } else {
4398         die "can't unplug device '$deviceid'\n";
4399     }
4400
4401     return 1;
4402 }
4403
4404 sub qemu_deviceadd {
4405     my ($vmid, $devicefull) = @_;
4406
4407     $devicefull = "driver=".$devicefull;
4408     my %options =  split(/[=,]/, $devicefull);
4409
4410     vm_mon_cmd($vmid, "device_add" , %options);
4411 }
4412
4413 sub qemu_devicedel {
4414     my ($vmid, $deviceid) = @_;
4415
4416     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4417 }
4418
4419 sub qemu_iothread_add {
4420     my($vmid, $deviceid, $device) = @_;
4421
4422     if ($device->{iothread}) {
4423         my $iothreads = vm_iothreads_list($vmid);
4424         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4425     }
4426 }
4427
4428 sub qemu_iothread_del {
4429     my($conf, $vmid, $deviceid) = @_;
4430
4431     my $confid = $deviceid;
4432     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4433         $confid = 'scsi' . $1;
4434     }
4435     my $device = parse_drive($confid, $conf->{$confid});
4436     if ($device->{iothread}) {
4437         my $iothreads = vm_iothreads_list($vmid);
4438         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4439     }
4440 }
4441
4442 sub qemu_objectadd {
4443     my($vmid, $objectid, $qomtype) = @_;
4444
4445     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4446
4447     return 1;
4448 }
4449
4450 sub qemu_objectdel {
4451     my($vmid, $objectid) = @_;
4452
4453     vm_mon_cmd($vmid, "object-del", id => $objectid);
4454
4455     return 1;
4456 }
4457
4458 sub qemu_driveadd {
4459     my ($storecfg, $vmid, $device) = @_;
4460
4461     my $drive = print_drive_full($storecfg, $vmid, $device);
4462     $drive =~ s/\\/\\\\/g;
4463     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4464
4465     # If the command succeeds qemu prints: "OK"
4466     return 1 if $ret =~ m/OK/s;
4467
4468     die "adding drive failed: $ret\n";
4469 }
4470
4471 sub qemu_drivedel {
4472     my($vmid, $deviceid) = @_;
4473
4474     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4475     $ret =~ s/^\s+//;
4476
4477     return 1 if $ret eq "";
4478
4479     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4480     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4481
4482     die "deleting drive $deviceid failed : $ret\n";
4483 }
4484
4485 sub qemu_deviceaddverify {
4486     my ($vmid, $deviceid) = @_;
4487
4488     for (my $i = 0; $i <= 5; $i++) {
4489          my $devices_list = vm_devices_list($vmid);
4490          return 1 if defined($devices_list->{$deviceid});
4491          sleep 1;
4492     }
4493
4494     die "error on hotplug device '$deviceid'\n";
4495 }
4496
4497
4498 sub qemu_devicedelverify {
4499     my ($vmid, $deviceid) = @_;
4500
4501     # need to verify that the device is correctly removed as device_del
4502     # is async and empty return is not reliable
4503
4504     for (my $i = 0; $i <= 5; $i++) {
4505          my $devices_list = vm_devices_list($vmid);
4506          return 1 if !defined($devices_list->{$deviceid});
4507          sleep 1;
4508     }
4509
4510     die "error on hot-unplugging device '$deviceid'\n";
4511 }
4512
4513 sub qemu_findorcreatescsihw {
4514     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4515
4516     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4517
4518     my $scsihwid="$controller_prefix$controller";
4519     my $devices_list = vm_devices_list($vmid);
4520
4521     if(!defined($devices_list->{$scsihwid})) {
4522         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4523     }
4524
4525     return 1;
4526 }
4527
4528 sub qemu_deletescsihw {
4529     my ($conf, $vmid, $opt) = @_;
4530
4531     my $device = parse_drive($opt, $conf->{$opt});
4532
4533     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4534         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4535         return 1;
4536     }
4537
4538     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4539
4540     my $devices_list = vm_devices_list($vmid);
4541     foreach my $opt (keys %{$devices_list}) {
4542         if (PVE::QemuServer::is_valid_drivename($opt)) {
4543             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4544             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4545                 return 1;
4546             }
4547         }
4548     }
4549
4550     my $scsihwid="scsihw$controller";
4551
4552     vm_deviceunplug($vmid, $conf, $scsihwid);
4553
4554     return 1;
4555 }
4556
4557 sub qemu_add_pci_bridge {
4558     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4559
4560     my $bridges = {};
4561
4562     my $bridgeid;
4563
4564     print_pci_addr($device, $bridges, $arch, $machine_type);
4565
4566     while (my ($k, $v) = each %$bridges) {
4567         $bridgeid = $k;
4568     }
4569     return 1 if !defined($bridgeid) || $bridgeid < 1;
4570
4571     my $bridge = "pci.$bridgeid";
4572     my $devices_list = vm_devices_list($vmid);
4573
4574     if (!defined($devices_list->{$bridge})) {
4575         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4576     }
4577
4578     return 1;
4579 }
4580
4581 sub qemu_set_link_status {
4582     my ($vmid, $device, $up) = @_;
4583
4584     vm_mon_cmd($vmid, "set_link", name => $device,
4585                up => $up ? JSON::true : JSON::false);
4586 }
4587
4588 sub qemu_netdevadd {
4589     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4590
4591     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4592     my %options =  split(/[=,]/, $netdev);
4593
4594     vm_mon_cmd($vmid, "netdev_add",  %options);
4595     return 1;
4596 }
4597
4598 sub qemu_netdevdel {
4599     my ($vmid, $deviceid) = @_;
4600
4601     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4602 }
4603
4604 sub qemu_usb_hotplug {
4605     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4606
4607     return if !$device;
4608
4609     # remove the old one first
4610     vm_deviceunplug($vmid, $conf, $deviceid);
4611
4612     # check if xhci controller is necessary and available
4613     if ($device->{usb3}) {
4614
4615         my $devicelist = vm_devices_list($vmid);
4616
4617         if (!$devicelist->{xhci}) {
4618             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4619             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4620         }
4621     }
4622     my $d = parse_usb_device($device->{host});
4623     $d->{usb3} = $device->{usb3};
4624
4625     # add the new one
4626     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4627 }
4628
4629 sub qemu_cpu_hotplug {
4630     my ($vmid, $conf, $vcpus) = @_;
4631
4632     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4633
4634     my $sockets = 1;
4635     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4636     $sockets = $conf->{sockets} if  $conf->{sockets};
4637     my $cores = $conf->{cores} || 1;
4638     my $maxcpus = $sockets * $cores;
4639
4640     $vcpus = $maxcpus if !$vcpus;
4641
4642     die "you can't add more vcpus than maxcpus\n"
4643         if $vcpus > $maxcpus;
4644
4645     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4646
4647     if ($vcpus < $currentvcpus) {
4648
4649         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4650
4651             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4652                 qemu_devicedel($vmid, "cpu$i");
4653                 my $retry = 0;
4654                 my $currentrunningvcpus = undef;
4655                 while (1) {
4656                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4657                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4658                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4659                     $retry++;
4660                     sleep 1;
4661                 }
4662                 #update conf after each succesfull cpu unplug
4663                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4664                 PVE::QemuConfig->write_config($vmid, $conf);
4665             }
4666         } else {
4667             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4668         }
4669
4670         return;
4671     }
4672
4673     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4674     die "vcpus in running vm does not match its configuration\n"
4675         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4676
4677     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4678
4679         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4680             my $cpustr = print_cpu_device($conf, $i);
4681             qemu_deviceadd($vmid, $cpustr);
4682
4683             my $retry = 0;
4684             my $currentrunningvcpus = undef;
4685             while (1) {
4686                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4687                 last if scalar(@{$currentrunningvcpus}) == $i;
4688                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4689                 sleep 1;
4690                 $retry++;
4691             }
4692             #update conf after each succesfull cpu hotplug
4693             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4694             PVE::QemuConfig->write_config($vmid, $conf);
4695         }
4696     } else {
4697
4698         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4699             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4700         }
4701     }
4702 }
4703
4704 sub qemu_block_set_io_throttle {
4705     my ($vmid, $deviceid,
4706         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4707         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4708         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4709         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4710
4711     return if !check_running($vmid) ;
4712
4713     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4714         bps => int($bps),
4715         bps_rd => int($bps_rd),
4716         bps_wr => int($bps_wr),
4717         iops => int($iops),
4718         iops_rd => int($iops_rd),
4719         iops_wr => int($iops_wr),
4720         bps_max => int($bps_max),
4721         bps_rd_max => int($bps_rd_max),
4722         bps_wr_max => int($bps_wr_max),
4723         iops_max => int($iops_max),
4724         iops_rd_max => int($iops_rd_max),
4725         iops_wr_max => int($iops_wr_max),
4726         bps_max_length => int($bps_max_length),
4727         bps_rd_max_length => int($bps_rd_max_length),
4728         bps_wr_max_length => int($bps_wr_max_length),
4729         iops_max_length => int($iops_max_length),
4730         iops_rd_max_length => int($iops_rd_max_length),
4731         iops_wr_max_length => int($iops_wr_max_length),
4732     );
4733
4734 }
4735
4736 # old code, only used to shutdown old VM after update
4737 sub __read_avail {
4738     my ($fh, $timeout) = @_;
4739
4740     my $sel = new IO::Select;
4741     $sel->add($fh);
4742
4743     my $res = '';
4744     my $buf;
4745
4746     my @ready;
4747     while (scalar (@ready = $sel->can_read($timeout))) {
4748         my $count;
4749         if ($count = $fh->sysread($buf, 8192)) {
4750             if ($buf =~ /^(.*)\(qemu\) $/s) {
4751                 $res .= $1;
4752                 last;
4753             } else {
4754                 $res .= $buf;
4755             }
4756         } else {
4757             if (!defined($count)) {
4758                 die "$!\n";
4759             }
4760             last;
4761         }
4762     }
4763
4764     die "monitor read timeout\n" if !scalar(@ready);
4765
4766     return $res;
4767 }
4768
4769 sub qemu_block_resize {
4770     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4771
4772     my $running = check_running($vmid);
4773
4774     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4775
4776     return if !$running;
4777
4778     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4779
4780 }
4781
4782 sub qemu_volume_snapshot {
4783     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4784
4785     my $running = check_running($vmid);
4786
4787     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4788         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4789     } else {
4790         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4791     }
4792 }
4793
4794 sub qemu_volume_snapshot_delete {
4795     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4796
4797     my $running = check_running($vmid);
4798
4799     if($running) {
4800
4801         $running = undef;
4802         my $conf = PVE::QemuConfig->load_config($vmid);
4803         foreach_drive($conf, sub {
4804             my ($ds, $drive) = @_;
4805             $running = 1 if $drive->{file} eq $volid;
4806         });
4807     }
4808
4809     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4810         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4811     } else {
4812         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4813     }
4814 }
4815
4816 sub set_migration_caps {
4817     my ($vmid) = @_;
4818
4819     my $cap_ref = [];
4820
4821     my $enabled_cap = {
4822         "auto-converge" => 1,
4823         "xbzrle" => 1,
4824         "x-rdma-pin-all" => 0,
4825         "zero-blocks" => 0,
4826         "compress" => 0
4827     };
4828
4829     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4830
4831     for my $supported_capability (@$supported_capabilities) {
4832         push @$cap_ref, {
4833             capability => $supported_capability->{capability},
4834             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4835         };
4836     }
4837
4838     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4839 }
4840
4841 my $fast_plug_option = {
4842     'lock' => 1,
4843     'name' => 1,
4844     'onboot' => 1,
4845     'shares' => 1,
4846     'startup' => 1,
4847     'description' => 1,
4848     'protection' => 1,
4849     'vmstatestorage' => 1,
4850     'hookscript' => 1,
4851 };
4852
4853 # hotplug changes in [PENDING]
4854 # $selection hash can be used to only apply specified options, for
4855 # example: { cores => 1 } (only apply changed 'cores')
4856 # $errors ref is used to return error messages
4857 sub vmconfig_hotplug_pending {
4858     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4859
4860     my $defaults = load_defaults();
4861     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4862
4863     # commit values which do not have any impact on running VM first
4864     # Note: those option cannot raise errors, we we do not care about
4865     # $selection and always apply them.
4866
4867     my $add_error = sub {
4868         my ($opt, $msg) = @_;
4869         $errors->{$opt} = "hotplug problem - $msg";
4870     };
4871
4872     my $changes = 0;
4873     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4874         if ($fast_plug_option->{$opt}) {
4875             $conf->{$opt} = $conf->{pending}->{$opt};
4876             delete $conf->{pending}->{$opt};
4877             $changes = 1;
4878         }
4879     }
4880
4881     if ($changes) {
4882         PVE::QemuConfig->write_config($vmid, $conf);
4883         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4884     }
4885
4886     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4887
4888     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4889     while (my ($opt, $force) = each %$pending_delete_hash) {
4890         next if $selection && !$selection->{$opt};
4891         eval {
4892             if ($opt eq 'hotplug') {
4893                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4894             } elsif ($opt eq 'tablet') {
4895                 die "skip\n" if !$hotplug_features->{usb};
4896                 if ($defaults->{tablet}) {
4897                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4898                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4899                         if $arch eq 'aarch64';
4900                 } else {
4901                     vm_deviceunplug($vmid, $conf, 'tablet');
4902                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4903                 }
4904             } elsif ($opt =~ m/^usb\d+/) {
4905                 die "skip\n";
4906                 # since we cannot reliably hot unplug usb devices
4907                 # we are disabling it
4908                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4909                 vm_deviceunplug($vmid, $conf, $opt);
4910             } elsif ($opt eq 'vcpus') {
4911                 die "skip\n" if !$hotplug_features->{cpu};
4912                 qemu_cpu_hotplug($vmid, $conf, undef);
4913             } elsif ($opt eq 'balloon') {
4914                 # enable balloon device is not hotpluggable
4915                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4916                 # here we reset the ballooning value to memory
4917                 my $balloon = $conf->{memory} || $defaults->{memory};
4918                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4919             } elsif ($fast_plug_option->{$opt}) {
4920                 # do nothing
4921             } elsif ($opt =~ m/^net(\d+)$/) {
4922                 die "skip\n" if !$hotplug_features->{network};
4923                 vm_deviceunplug($vmid, $conf, $opt);
4924             } elsif (is_valid_drivename($opt)) {
4925                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4926                 vm_deviceunplug($vmid, $conf, $opt);
4927                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4928             } elsif ($opt =~ m/^memory$/) {
4929                 die "skip\n" if !$hotplug_features->{memory};
4930                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4931             } elsif ($opt eq 'cpuunits') {
4932                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4933             } elsif ($opt eq 'cpulimit') {
4934                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4935             } else {
4936                 die "skip\n";
4937             }
4938         };
4939         if (my $err = $@) {
4940             &$add_error($opt, $err) if $err ne "skip\n";
4941         } else {
4942             # save new config if hotplug was successful
4943             delete $conf->{$opt};
4944             vmconfig_undelete_pending_option($conf, $opt);
4945             PVE::QemuConfig->write_config($vmid, $conf);
4946             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4947         }
4948     }
4949
4950     my $apply_pending_cloudinit;
4951     $apply_pending_cloudinit = sub {
4952         my ($key, $value) = @_;
4953         $apply_pending_cloudinit = sub {}; # once is enough
4954
4955         my @cloudinit_opts = keys %$confdesc_cloudinit;
4956         foreach my $opt (keys %{$conf->{pending}}) {
4957             next if !grep { $_ eq $opt } @cloudinit_opts;
4958             $conf->{$opt} = delete $conf->{pending}->{$opt};
4959         }
4960
4961         my $new_conf = { %$conf };
4962         $new_conf->{$key} = $value;
4963         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4964     };
4965
4966     foreach my $opt (keys %{$conf->{pending}}) {
4967         next if $selection && !$selection->{$opt};
4968         my $value = $conf->{pending}->{$opt};
4969         eval {
4970             if ($opt eq 'hotplug') {
4971                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4972             } elsif ($opt eq 'tablet') {
4973                 die "skip\n" if !$hotplug_features->{usb};
4974                 if ($value == 1) {
4975                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4976                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4977                         if $arch eq 'aarch64';
4978                 } elsif ($value == 0) {
4979                     vm_deviceunplug($vmid, $conf, 'tablet');
4980                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4981                 }
4982             } elsif ($opt =~ m/^usb\d+$/) {
4983                 die "skip\n";
4984                 # since we cannot reliably hot unplug usb devices
4985                 # we are disabling it
4986                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4987                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4988                 die "skip\n" if !$d;
4989                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4990             } elsif ($opt eq 'vcpus') {
4991                 die "skip\n" if !$hotplug_features->{cpu};
4992                 qemu_cpu_hotplug($vmid, $conf, $value);
4993             } elsif ($opt eq 'balloon') {
4994                 # enable/disable balloning device is not hotpluggable
4995                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4996                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4997                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4998
4999                 # allow manual ballooning if shares is set to zero
5000                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5001                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5002                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5003                 }
5004             } elsif ($opt =~ m/^net(\d+)$/) {
5005                 # some changes can be done without hotplug
5006                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5007                                     $vmid, $opt, $value, $arch, $machine_type);
5008             } elsif (is_valid_drivename($opt)) {
5009                 # some changes can be done without hotplug
5010                 my $drive = parse_drive($opt, $value);
5011                 if (drive_is_cloudinit($drive)) {
5012                     &$apply_pending_cloudinit($opt, $value);
5013                 }
5014                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5015                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5016             } elsif ($opt =~ m/^memory$/) { #dimms
5017                 die "skip\n" if !$hotplug_features->{memory};
5018                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5019             } elsif ($opt eq 'cpuunits') {
5020                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5021             } elsif ($opt eq 'cpulimit') {
5022                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5023                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5024             } else {
5025                 die "skip\n";  # skip non-hot-pluggable options
5026             }
5027         };
5028         if (my $err = $@) {
5029             &$add_error($opt, $err) if $err ne "skip\n";
5030         } else {
5031             # save new config if hotplug was successful
5032             $conf->{$opt} = $value;
5033             delete $conf->{pending}->{$opt};
5034             PVE::QemuConfig->write_config($vmid, $conf);
5035             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5036         }
5037     }
5038 }
5039
5040 sub try_deallocate_drive {
5041     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5042
5043     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5044         my $volid = $drive->{file};
5045         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5046             my $sid = PVE::Storage::parse_volume_id($volid);
5047             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5048
5049             # check if the disk is really unused
5050             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5051                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5052             PVE::Storage::vdisk_free($storecfg, $volid);
5053             return 1;
5054         } else {
5055             # If vm is not owner of this disk remove from config
5056             return 1;
5057         }
5058     }
5059
5060     return undef;
5061 }
5062
5063 sub vmconfig_delete_or_detach_drive {
5064     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5065
5066     my $drive = parse_drive($opt, $conf->{$opt});
5067
5068     my $rpcenv = PVE::RPCEnvironment::get();
5069     my $authuser = $rpcenv->get_user();
5070
5071     if ($force) {
5072         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5073         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5074     } else {
5075         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5076     }
5077 }
5078
5079 sub vmconfig_apply_pending {
5080     my ($vmid, $conf, $storecfg) = @_;
5081
5082     # cold plug
5083
5084     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5085     while (my ($opt, $force) = each %$pending_delete_hash) {
5086         die "internal error" if $opt =~ m/^unused/;
5087         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5088         if (!defined($conf->{$opt})) {
5089             vmconfig_undelete_pending_option($conf, $opt);
5090             PVE::QemuConfig->write_config($vmid, $conf);
5091         } elsif (is_valid_drivename($opt)) {
5092             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5093             vmconfig_undelete_pending_option($conf, $opt);
5094             delete $conf->{$opt};
5095             PVE::QemuConfig->write_config($vmid, $conf);
5096         } else {
5097             vmconfig_undelete_pending_option($conf, $opt);
5098             delete $conf->{$opt};
5099             PVE::QemuConfig->write_config($vmid, $conf);
5100         }
5101     }
5102
5103     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5104
5105     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5106         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5107
5108         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5109             # skip if nothing changed
5110         } elsif (is_valid_drivename($opt)) {
5111             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5112                 if defined($conf->{$opt});
5113             $conf->{$opt} = $conf->{pending}->{$opt};
5114         } else {
5115             $conf->{$opt} = $conf->{pending}->{$opt};
5116         }
5117
5118         delete $conf->{pending}->{$opt};
5119         PVE::QemuConfig->write_config($vmid, $conf);
5120     }
5121 }
5122
5123 my $safe_num_ne = sub {
5124     my ($a, $b) = @_;
5125
5126     return 0 if !defined($a) && !defined($b);
5127     return 1 if !defined($a);
5128     return 1 if !defined($b);
5129
5130     return $a != $b;
5131 };
5132
5133 my $safe_string_ne = sub {
5134     my ($a, $b) = @_;
5135
5136     return 0 if !defined($a) && !defined($b);
5137     return 1 if !defined($a);
5138     return 1 if !defined($b);
5139
5140     return $a ne $b;
5141 };
5142
5143 sub vmconfig_update_net {
5144     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5145
5146     my $newnet = parse_net($value);
5147
5148     if ($conf->{$opt}) {
5149         my $oldnet = parse_net($conf->{$opt});
5150
5151         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5152             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5153             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5154             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5155
5156             # for non online change, we try to hot-unplug
5157             die "skip\n" if !$hotplug;
5158             vm_deviceunplug($vmid, $conf, $opt);
5159         } else {
5160
5161             die "internal error" if $opt !~ m/net(\d+)/;
5162             my $iface = "tap${vmid}i$1";
5163
5164             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5165                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5166                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5167                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5168                 PVE::Network::tap_unplug($iface);
5169                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5170             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5171                 # Rate can be applied on its own but any change above needs to
5172                 # include the rate in tap_plug since OVS resets everything.
5173                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5174             }
5175
5176             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5177                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5178             }
5179
5180             return 1;
5181         }
5182     }
5183
5184     if ($hotplug) {
5185         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5186     } else {
5187         die "skip\n";
5188     }
5189 }
5190
5191 sub vmconfig_update_disk {
5192     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5193
5194     # fixme: do we need force?
5195
5196     my $drive = parse_drive($opt, $value);
5197
5198     if ($conf->{$opt}) {
5199
5200         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5201
5202             my $media = $drive->{media} || 'disk';
5203             my $oldmedia = $old_drive->{media} || 'disk';
5204             die "unable to change media type\n" if $media ne $oldmedia;
5205
5206             if (!drive_is_cdrom($old_drive)) {
5207
5208                 if ($drive->{file} ne $old_drive->{file}) {
5209
5210                     die "skip\n" if !$hotplug;
5211
5212                     # unplug and register as unused
5213                     vm_deviceunplug($vmid, $conf, $opt);
5214                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5215
5216                 } else {
5217                     # update existing disk
5218
5219                     # skip non hotpluggable value
5220                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5221                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5222                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5223                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5224                         die "skip\n";
5225                     }
5226
5227                     # apply throttle
5228                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5229                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5230                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5231                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5232                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5233                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5234                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5235                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5236                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5237                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5238                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5239                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5240                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5241                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5242                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5243                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5244                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5245                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5246
5247                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5248                                                    ($drive->{mbps} || 0)*1024*1024,
5249                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5250                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5251                                                    $drive->{iops} || 0,
5252                                                    $drive->{iops_rd} || 0,
5253                                                    $drive->{iops_wr} || 0,
5254                                                    ($drive->{mbps_max} || 0)*1024*1024,
5255                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5256                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5257                                                    $drive->{iops_max} || 0,
5258                                                    $drive->{iops_rd_max} || 0,
5259                                                    $drive->{iops_wr_max} || 0,
5260                                                    $drive->{bps_max_length} || 1,
5261                                                    $drive->{bps_rd_max_length} || 1,
5262                                                    $drive->{bps_wr_max_length} || 1,
5263                                                    $drive->{iops_max_length} || 1,
5264                                                    $drive->{iops_rd_max_length} || 1,
5265                                                    $drive->{iops_wr_max_length} || 1);
5266
5267                     }
5268
5269                     return 1;
5270                 }
5271
5272             } else { # cdrom
5273
5274                 if ($drive->{file} eq 'none') {
5275                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5276                     if (drive_is_cloudinit($old_drive)) {
5277                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5278                     }
5279                 } else {
5280                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5281                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5282                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5283                 }
5284
5285                 return 1;
5286             }
5287         }
5288     }
5289
5290     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5291     # hotplug new disks
5292     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5293     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5294 }
5295
5296 sub vm_start {
5297     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5298         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5299
5300     PVE::QemuConfig->lock_config($vmid, sub {
5301         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5302
5303         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5304
5305         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5306
5307         PVE::QemuConfig->check_lock($conf)
5308             if !($skiplock || $is_suspended);
5309
5310         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5311
5312         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5313             vmconfig_apply_pending($vmid, $conf, $storecfg);
5314             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5315         }
5316
5317         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5318
5319         my $defaults = load_defaults();
5320
5321         # set environment variable useful inside network script
5322         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5323
5324         my $local_volumes = {};
5325
5326         if ($targetstorage) {
5327             foreach_drive($conf, sub {
5328                 my ($ds, $drive) = @_;
5329
5330                 return if drive_is_cdrom($drive);
5331
5332                 my $volid = $drive->{file};
5333
5334                 return if !$volid;
5335
5336                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5337
5338                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5339                 return if $scfg->{shared};
5340                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5341             });
5342
5343             my $format = undef;
5344
5345             foreach my $opt (sort keys %$local_volumes) {
5346
5347                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5348                 my $drive = parse_drive($opt, $conf->{$opt});
5349
5350                 #if remote storage is specified, use default format
5351                 if ($targetstorage && $targetstorage ne "1") {
5352                     $storeid = $targetstorage;
5353                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5354                     $format = $defFormat;
5355                 } else {
5356                     #else we use same format than original
5357                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5358                     $format = qemu_img_format($scfg, $volid);
5359                 }
5360
5361                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5362                 my $newdrive = $drive;
5363                 $newdrive->{format} = $format;
5364                 $newdrive->{file} = $newvolid;
5365                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5366                 $local_volumes->{$opt} = $drivestr;
5367                 #pass drive to conf for command line
5368                 $conf->{$opt} = $drivestr;
5369             }
5370         }
5371
5372         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5373
5374         if ($is_suspended) {
5375             # enforce machine type on suspended vm to ensure HW compatibility
5376             $forcemachine = $conf->{runningmachine};
5377             print "Resuming suspended VM\n";
5378         }
5379
5380         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5381
5382         my $migrate_port = 0;
5383         my $migrate_uri;
5384         if ($statefile) {
5385             if ($statefile eq 'tcp') {
5386                 my $localip = "localhost";
5387                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5388                 my $nodename = PVE::INotify::nodename();
5389
5390                 if (!defined($migration_type)) {
5391                     if (defined($datacenterconf->{migration}->{type})) {
5392                         $migration_type = $datacenterconf->{migration}->{type};
5393                     } else {
5394                         $migration_type = 'secure';
5395                     }
5396                 }
5397
5398                 if ($migration_type eq 'insecure') {
5399                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5400                     if ($migrate_network_addr) {
5401                         $localip = $migrate_network_addr;
5402                     } else {
5403                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5404                     }
5405
5406                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5407                 }
5408
5409                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5410                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5411                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5412                 push @$cmd, '-incoming', $migrate_uri;
5413                 push @$cmd, '-S';
5414
5415             } elsif ($statefile eq 'unix') {
5416                 # should be default for secure migrations as a ssh TCP forward
5417                 # tunnel is not deterministic reliable ready and fails regurarly
5418                 # to set up in time, so use UNIX socket forwards
5419                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5420                 unlink $socket_addr;
5421
5422                 $migrate_uri = "unix:$socket_addr";
5423
5424                 push @$cmd, '-incoming', $migrate_uri;
5425                 push @$cmd, '-S';
5426
5427             } else {
5428                 push @$cmd, '-loadstate', $statefile;
5429             }
5430         } elsif ($paused) {
5431             push @$cmd, '-S';
5432         }
5433
5434         # host pci devices
5435         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5436           my $d = parse_hostpci($conf->{"hostpci$i"});
5437           next if !$d;
5438           my $pcidevices = $d->{pciid};
5439           foreach my $pcidevice (@$pcidevices) {
5440                 my $pciid = $pcidevice->{id};
5441
5442                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5443                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5444                 die "no pci device info for device '$pciid'\n" if !$info;
5445
5446                 if ($d->{mdev}) {
5447                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5448                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5449                 } else {
5450                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5451                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5452                     die "can't reset pci device '$pciid'\n"
5453                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5454                 }
5455           }
5456         }
5457
5458         PVE::Storage::activate_volumes($storecfg, $vollist);
5459
5460         eval {
5461             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5462                 outfunc => sub {}, errfunc => sub {});
5463         };
5464         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5465         # timeout should be more than enough here...
5466         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5467
5468         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5469                                                   : $defaults->{cpuunits};
5470
5471         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5472         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5473
5474         my %properties = (
5475             Slice => 'qemu.slice',
5476             KillMode => 'none',
5477             CPUShares => $cpuunits
5478         );
5479
5480         if (my $cpulimit = $conf->{cpulimit}) {
5481             $properties{CPUQuota} = int($cpulimit * 100);
5482         }
5483         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5484
5485         my $run_qemu = sub {
5486             PVE::Tools::run_fork sub {
5487                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5488                 run_command($cmd, %run_params);
5489             };
5490         };
5491
5492         if ($conf->{hugepages}) {
5493
5494             my $code = sub {
5495                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5496                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5497
5498                 PVE::QemuServer::Memory::hugepages_mount();
5499                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5500
5501                 eval { $run_qemu->() };
5502                 if (my $err = $@) {
5503                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5504                     die $err;
5505                 }
5506
5507                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5508             };
5509             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5510
5511         } else {
5512             eval { $run_qemu->() };
5513         }
5514
5515         if (my $err = $@) {
5516             # deactivate volumes if start fails
5517             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5518             die "start failed: $err";
5519         }
5520
5521         print "migration listens on $migrate_uri\n" if $migrate_uri;
5522
5523         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5524             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5525             warn $@ if $@;
5526         }
5527
5528         #start nbd server for storage migration
5529         if ($targetstorage) {
5530             my $nodename = PVE::INotify::nodename();
5531             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5532             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5533             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5534             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5535
5536             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5537
5538             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5539
5540             foreach my $opt (sort keys %$local_volumes) {
5541                 my $volid = $local_volumes->{$opt};
5542                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5543                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5544                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5545             }
5546         }
5547
5548         if ($migratedfrom) {
5549             eval {
5550                 set_migration_caps($vmid);
5551             };
5552             warn $@ if $@;
5553
5554             if ($spice_port) {
5555                 print "spice listens on port $spice_port\n";
5556                 if ($spice_ticket) {
5557                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5558                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5559                 }
5560             }
5561
5562         } else {
5563             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5564                 if !$statefile && $conf->{balloon};
5565
5566             foreach my $opt (keys %$conf) {
5567                 next if $opt !~  m/^net\d+$/;
5568                 my $nicconf = parse_net($conf->{$opt});
5569                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5570             }
5571         }
5572
5573         vm_mon_cmd_nocheck($vmid, 'qom-set',
5574                     path => "machine/peripheral/balloon0",
5575                     property => "guest-stats-polling-interval",
5576                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5577
5578         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5579             print "Resumed VM, removing state\n";
5580             delete $conf->@{qw(lock vmstate runningmachine)};
5581             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5582             PVE::Storage::vdisk_free($storecfg, $vmstate);
5583             PVE::QemuConfig->write_config($vmid, $conf);
5584         }
5585
5586         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5587     });
5588 }
5589
5590 sub vm_mon_cmd {
5591     my ($vmid, $execute, %params) = @_;
5592
5593     my $cmd = { execute => $execute, arguments => \%params };
5594     vm_qmp_command($vmid, $cmd);
5595 }
5596
5597 sub vm_mon_cmd_nocheck {
5598     my ($vmid, $execute, %params) = @_;
5599
5600     my $cmd = { execute => $execute, arguments => \%params };
5601     vm_qmp_command($vmid, $cmd, 1);
5602 }
5603
5604 sub vm_qmp_command {
5605     my ($vmid, $cmd, $nocheck) = @_;
5606
5607     my $res;
5608
5609     my $timeout;
5610     if ($cmd->{arguments}) {
5611         $timeout = delete $cmd->{arguments}->{timeout};
5612     }
5613
5614     eval {
5615         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5616         my $sname = qmp_socket($vmid);
5617         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5618             my $qmpclient = PVE::QMPClient->new();
5619
5620             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5621         } else {
5622             die "unable to open monitor socket\n";
5623         }
5624     };
5625     if (my $err = $@) {
5626         syslog("err", "VM $vmid qmp command failed - $err");
5627         die $err;
5628     }
5629
5630     return $res;
5631 }
5632
5633 sub vm_human_monitor_command {
5634     my ($vmid, $cmdline) = @_;
5635
5636     my $cmd = {
5637         execute => 'human-monitor-command',
5638         arguments => { 'command-line' => $cmdline},
5639     };
5640
5641     return vm_qmp_command($vmid, $cmd);
5642 }
5643
5644 sub vm_commandline {
5645     my ($storecfg, $vmid, $snapname) = @_;
5646
5647     my $conf = PVE::QemuConfig->load_config($vmid);
5648
5649     if ($snapname) {
5650         my $snapshot = $conf->{snapshots}->{$snapname};
5651         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5652
5653         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5654
5655         $conf = $snapshot;
5656     }
5657
5658     my $defaults = load_defaults();
5659
5660     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5661
5662     return PVE::Tools::cmd2string($cmd);
5663 }
5664
5665 sub vm_reset {
5666     my ($vmid, $skiplock) = @_;
5667
5668     PVE::QemuConfig->lock_config($vmid, sub {
5669
5670         my $conf = PVE::QemuConfig->load_config($vmid);
5671
5672         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5673
5674         vm_mon_cmd($vmid, "system_reset");
5675     });
5676 }
5677
5678 sub get_vm_volumes {
5679     my ($conf) = @_;
5680
5681     my $vollist = [];
5682     foreach_volid($conf, sub {
5683         my ($volid, $attr) = @_;
5684
5685         return if $volid =~ m|^/|;
5686
5687         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5688         return if !$sid;
5689
5690         push @$vollist, $volid;
5691     });
5692
5693     return $vollist;
5694 }
5695
5696 sub vm_stop_cleanup {
5697     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5698
5699     eval {
5700
5701         if (!$keepActive) {
5702             my $vollist = get_vm_volumes($conf);
5703             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5704         }
5705
5706         foreach my $ext (qw(mon qmp pid vnc qga)) {
5707             unlink "/var/run/qemu-server/${vmid}.$ext";
5708         }
5709
5710         if ($conf->{ivshmem}) {
5711             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5712             # just delete it for now, VMs which have this already open do not
5713             # are affected, but new VMs will get a separated one. If this
5714             # becomes an issue we either add some sort of ref-counting or just
5715             # add a "don't delete on stop" flag to the ivshmem format.
5716             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5717         }
5718
5719         foreach my $key (keys %$conf) {
5720             next if $key !~ m/^hostpci(\d+)$/;
5721             my $hostpciindex = $1;
5722             my $d = parse_hostpci($conf->{$key});
5723             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5724
5725             foreach my $pci (@{$d->{pciid}}) {
5726                 my $pciid = $pci->{id};
5727                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5728             }
5729         }
5730
5731         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5732     };
5733     warn $@ if $@; # avoid errors - just warn
5734 }
5735
5736 # Note: use $nockeck to skip tests if VM configuration file exists.
5737 # We need that when migration VMs to other nodes (files already moved)
5738 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5739 sub vm_stop {
5740     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5741
5742     $force = 1 if !defined($force) && !$shutdown;
5743
5744     if ($migratedfrom){
5745         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5746         kill 15, $pid if $pid;
5747         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5748         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5749         return;
5750     }
5751
5752     PVE::QemuConfig->lock_config($vmid, sub {
5753
5754         my $pid = check_running($vmid, $nocheck);
5755         return if !$pid;
5756
5757         my $conf;
5758         if (!$nocheck) {
5759             $conf = PVE::QemuConfig->load_config($vmid);
5760             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5761             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5762                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5763                 $timeout = $opts->{down} if $opts->{down};
5764             }
5765             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5766         }
5767
5768         eval {
5769             if ($shutdown) {
5770                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5771                     vm_qmp_command($vmid, {
5772                         execute => "guest-shutdown",
5773                         arguments => { timeout => $timeout }
5774                     }, $nocheck);
5775                 } else {
5776                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5777                 }
5778             } else {
5779                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5780             }
5781         };
5782         my $err = $@;
5783
5784         if (!$err) {
5785             $timeout = 60 if !defined($timeout);
5786
5787             my $count = 0;
5788             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5789                 $count++;
5790                 sleep 1;
5791             }
5792
5793             if ($count >= $timeout) {
5794                 if ($force) {
5795                     warn "VM still running - terminating now with SIGTERM\n";
5796                     kill 15, $pid;
5797                 } else {
5798                     die "VM quit/powerdown failed - got timeout\n";
5799                 }
5800             } else {
5801                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5802                 return;
5803             }
5804         } else {
5805             if ($force) {
5806                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5807                 kill 15, $pid;
5808             } else {
5809                 die "VM quit/powerdown failed\n";
5810             }
5811         }
5812
5813         # wait again
5814         $timeout = 10;
5815
5816         my $count = 0;
5817         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5818             $count++;
5819             sleep 1;
5820         }
5821
5822         if ($count >= $timeout) {
5823             warn "VM still running - terminating now with SIGKILL\n";
5824             kill 9, $pid;
5825             sleep 1;
5826         }
5827
5828         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5829    });
5830 }
5831
5832 sub vm_suspend {
5833     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5834
5835     my $conf;
5836     my $path;
5837     my $storecfg;
5838     my $vmstate;
5839
5840     PVE::QemuConfig->lock_config($vmid, sub {
5841
5842         $conf = PVE::QemuConfig->load_config($vmid);
5843
5844         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5845         PVE::QemuConfig->check_lock($conf)
5846             if !($skiplock || $is_backing_up);
5847
5848         die "cannot suspend to disk during backup\n"
5849             if $is_backing_up && $includestate;
5850
5851         if ($includestate) {
5852             $conf->{lock} = 'suspending';
5853             my $date = strftime("%Y-%m-%d", localtime(time()));
5854             $storecfg = PVE::Storage::config();
5855             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5856             $path = PVE::Storage::path($storecfg, $vmstate);
5857             PVE::QemuConfig->write_config($vmid, $conf);
5858         } else {
5859             vm_mon_cmd($vmid, "stop");
5860         }
5861     });
5862
5863     if ($includestate) {
5864         # save vm state
5865         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5866
5867         eval {
5868             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5869             for(;;) {
5870                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5871                 if (!$state->{status}) {
5872                     die "savevm not active\n";
5873                 } elsif ($state->{status} eq 'active') {
5874                     sleep(1);
5875                     next;
5876                 } elsif ($state->{status} eq 'completed') {
5877                     print "State saved, quitting\n";
5878                     last;
5879                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5880                     die "query-savevm failed with error '$state->{error}'\n"
5881                 } else {
5882                     die "query-savevm returned status '$state->{status}'\n";
5883                 }
5884             }
5885         };
5886         my $err = $@;
5887
5888         PVE::QemuConfig->lock_config($vmid, sub {
5889             $conf = PVE::QemuConfig->load_config($vmid);
5890             if ($err) {
5891                 # cleanup, but leave suspending lock, to indicate something went wrong
5892                 eval {
5893                     vm_mon_cmd($vmid, "savevm-end");
5894                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5895                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5896                     delete $conf->@{qw(vmstate runningmachine)};
5897                     PVE::QemuConfig->write_config($vmid, $conf);
5898                 };
5899                 warn $@ if $@;
5900                 die $err;
5901             }
5902
5903             die "lock changed unexpectedly\n"
5904                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5905
5906             vm_qmp_command($vmid, { execute => "quit" });
5907             $conf->{lock} = 'suspended';
5908             PVE::QemuConfig->write_config($vmid, $conf);
5909         });
5910     }
5911 }
5912
5913 sub vm_resume {
5914     my ($vmid, $skiplock, $nocheck) = @_;
5915
5916     PVE::QemuConfig->lock_config($vmid, sub {
5917         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5918         my $res = $vm_mon_cmd->($vmid, 'query-status');
5919         my $resume_cmd = 'cont';
5920
5921         if ($res->{status} && $res->{status} eq 'suspended') {
5922             $resume_cmd = 'system_wakeup';
5923         }
5924
5925         if (!$nocheck) {
5926
5927             my $conf = PVE::QemuConfig->load_config($vmid);
5928
5929             PVE::QemuConfig->check_lock($conf)
5930                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5931         }
5932
5933         $vm_mon_cmd->($vmid, $resume_cmd);
5934     });
5935 }
5936
5937 sub vm_sendkey {
5938     my ($vmid, $skiplock, $key) = @_;
5939
5940     PVE::QemuConfig->lock_config($vmid, sub {
5941
5942         my $conf = PVE::QemuConfig->load_config($vmid);
5943
5944         # there is no qmp command, so we use the human monitor command
5945         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5946         die $res if $res ne '';
5947     });
5948 }
5949
5950 sub vm_destroy {
5951     my ($storecfg, $vmid, $skiplock) = @_;
5952
5953     PVE::QemuConfig->lock_config($vmid, sub {
5954
5955         my $conf = PVE::QemuConfig->load_config($vmid);
5956
5957         if (!check_running($vmid)) {
5958             destroy_vm($storecfg, $vmid, undef, $skiplock);
5959         } else {
5960             die "VM $vmid is running - destroy failed\n";
5961         }
5962     });
5963 }
5964
5965 # vzdump restore implementaion
5966
5967 sub tar_archive_read_firstfile {
5968     my $archive = shift;
5969
5970     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5971
5972     # try to detect archive type first
5973     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5974         die "unable to open file '$archive'\n";
5975     my $firstfile = <$fh>;
5976     kill 15, $pid;
5977     close $fh;
5978
5979     die "ERROR: archive contaions no data\n" if !$firstfile;
5980     chomp $firstfile;
5981
5982     return $firstfile;
5983 }
5984
5985 sub tar_restore_cleanup {
5986     my ($storecfg, $statfile) = @_;
5987
5988     print STDERR "starting cleanup\n";
5989
5990     if (my $fd = IO::File->new($statfile, "r")) {
5991         while (defined(my $line = <$fd>)) {
5992             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5993                 my $volid = $2;
5994                 eval {
5995                     if ($volid =~ m|^/|) {
5996                         unlink $volid || die 'unlink failed\n';
5997                     } else {
5998                         PVE::Storage::vdisk_free($storecfg, $volid);
5999                     }
6000                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6001                 };
6002                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6003             } else {
6004                 print STDERR "unable to parse line in statfile - $line";
6005             }
6006         }
6007         $fd->close();
6008     }
6009 }
6010
6011 sub restore_archive {
6012     my ($archive, $vmid, $user, $opts) = @_;
6013
6014     my $format = $opts->{format};
6015     my $comp;
6016
6017     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6018         $format = 'tar' if !$format;
6019         $comp = 'gzip';
6020     } elsif ($archive =~ m/\.tar$/) {
6021         $format = 'tar' if !$format;
6022     } elsif ($archive =~ m/.tar.lzo$/) {
6023         $format = 'tar' if !$format;
6024         $comp = 'lzop';
6025     } elsif ($archive =~ m/\.vma$/) {
6026         $format = 'vma' if !$format;
6027     } elsif ($archive =~ m/\.vma\.gz$/) {
6028         $format = 'vma' if !$format;
6029         $comp = 'gzip';
6030     } elsif ($archive =~ m/\.vma\.lzo$/) {
6031         $format = 'vma' if !$format;
6032         $comp = 'lzop';
6033     } else {
6034         $format = 'vma' if !$format; # default
6035     }
6036
6037     # try to detect archive format
6038     if ($format eq 'tar') {
6039         return restore_tar_archive($archive, $vmid, $user, $opts);
6040     } else {
6041         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6042     }
6043 }
6044
6045 sub restore_update_config_line {
6046     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6047
6048     return if $line =~ m/^\#qmdump\#/;
6049     return if $line =~ m/^\#vzdump\#/;
6050     return if $line =~ m/^lock:/;
6051     return if $line =~ m/^unused\d+:/;
6052     return if $line =~ m/^parent:/;
6053
6054     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6055     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6056         # try to convert old 1.X settings
6057         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6058         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6059             my ($model, $macaddr) = split(/\=/, $devconfig);
6060             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6061             my $net = {
6062                 model => $model,
6063                 bridge => "vmbr$ind",
6064                 macaddr => $macaddr,
6065             };
6066             my $netstr = print_net($net);
6067
6068             print $outfd "net$cookie->{netcount}: $netstr\n";
6069             $cookie->{netcount}++;
6070         }
6071     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6072         my ($id, $netstr) = ($1, $2);
6073         my $net = parse_net($netstr);
6074         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6075         $netstr = print_net($net);
6076         print $outfd "$id: $netstr\n";
6077     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6078         my $virtdev = $1;
6079         my $value = $3;
6080         my $di = parse_drive($virtdev, $value);
6081         if (defined($di->{backup}) && !$di->{backup}) {
6082             print $outfd "#$line";
6083         } elsif ($map->{$virtdev}) {
6084             delete $di->{format}; # format can change on restore
6085             $di->{file} = $map->{$virtdev};
6086             $value = print_drive($vmid, $di);
6087             print $outfd "$virtdev: $value\n";
6088         } else {
6089             print $outfd $line;
6090         }
6091     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6092         my $vmgenid = $1;
6093         if ($vmgenid ne '0') {
6094             # always generate a new vmgenid if there was a valid one setup
6095             $vmgenid = generate_uuid();
6096         }
6097         print $outfd "vmgenid: $vmgenid\n";
6098     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6099         my ($uuid, $uuid_str);
6100         UUID::generate($uuid);
6101         UUID::unparse($uuid, $uuid_str);
6102         my $smbios1 = parse_smbios1($2);
6103         $smbios1->{uuid} = $uuid_str;
6104         print $outfd $1.print_smbios1($smbios1)."\n";
6105     } else {
6106         print $outfd $line;
6107     }
6108 }
6109
6110 sub scan_volids {
6111     my ($cfg, $vmid) = @_;
6112
6113     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6114
6115     my $volid_hash = {};
6116     foreach my $storeid (keys %$info) {
6117         foreach my $item (@{$info->{$storeid}}) {
6118             next if !($item->{volid} && $item->{size});
6119             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6120             $volid_hash->{$item->{volid}} = $item;
6121         }
6122     }
6123
6124     return $volid_hash;
6125 }
6126
6127 sub is_volume_in_use {
6128     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6129
6130     my $path = PVE::Storage::path($storecfg, $volid);
6131
6132     my $scan_config = sub {
6133         my ($cref, $snapname) = @_;
6134
6135         foreach my $key (keys %$cref) {
6136             my $value = $cref->{$key};
6137             if (is_valid_drivename($key)) {
6138                 next if $skip_drive && $key eq $skip_drive;
6139                 my $drive = parse_drive($key, $value);
6140                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6141                 return 1 if $volid eq $drive->{file};
6142                 if ($drive->{file} =~ m!^/!) {
6143                     return 1 if $drive->{file} eq $path;
6144                 } else {
6145                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6146                     next if !$storeid;
6147                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6148                     next if !$scfg;
6149                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6150                 }
6151             }
6152         }
6153
6154         return 0;
6155     };
6156
6157     return 1 if &$scan_config($conf);
6158
6159     undef $skip_drive;
6160
6161     foreach my $snapname (keys %{$conf->{snapshots}}) {
6162         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6163     }
6164
6165     return 0;
6166 }
6167
6168 sub update_disksize {
6169     my ($vmid, $conf, $volid_hash) = @_;
6170
6171     my $changes;
6172     my $prefix = "VM $vmid:";
6173
6174     # used and unused disks
6175     my $referenced = {};
6176
6177     # Note: it is allowed to define multiple storages with same path (alias), so
6178     # we need to check both 'volid' and real 'path' (two different volid can point
6179     # to the same path).
6180
6181     my $referencedpath = {};
6182
6183     # update size info
6184     foreach my $opt (keys %$conf) {
6185         if (is_valid_drivename($opt)) {
6186             my $drive = parse_drive($opt, $conf->{$opt});
6187             my $volid = $drive->{file};
6188             next if !$volid;
6189
6190             $referenced->{$volid} = 1;
6191             if ($volid_hash->{$volid} &&
6192                 (my $path = $volid_hash->{$volid}->{path})) {
6193                 $referencedpath->{$path} = 1;
6194             }
6195
6196             next if drive_is_cdrom($drive);
6197             next if !$volid_hash->{$volid};
6198
6199             $drive->{size} = $volid_hash->{$volid}->{size};
6200             my $new = print_drive($vmid, $drive);
6201             if ($new ne $conf->{$opt}) {
6202                 $changes = 1;
6203                 $conf->{$opt} = $new;
6204                 print "$prefix update disk '$opt' information.\n";
6205             }
6206         }
6207     }
6208
6209     # remove 'unusedX' entry if volume is used
6210     foreach my $opt (keys %$conf) {
6211         next if $opt !~ m/^unused\d+$/;
6212         my $volid = $conf->{$opt};
6213         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6214         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6215             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6216             $changes = 1;
6217             delete $conf->{$opt};
6218         }
6219
6220         $referenced->{$volid} = 1;
6221         $referencedpath->{$path} = 1 if $path;
6222     }
6223
6224     foreach my $volid (sort keys %$volid_hash) {
6225         next if $volid =~ m/vm-$vmid-state-/;
6226         next if $referenced->{$volid};
6227         my $path = $volid_hash->{$volid}->{path};
6228         next if !$path; # just to be sure
6229         next if $referencedpath->{$path};
6230         $changes = 1;
6231         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6232         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6233         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6234     }
6235
6236     return $changes;
6237 }
6238
6239 sub rescan {
6240     my ($vmid, $nolock, $dryrun) = @_;
6241
6242     my $cfg = PVE::Storage::config();
6243
6244     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6245     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6246     foreach my $stor (keys %{$cfg->{ids}}) {
6247         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6248     }
6249
6250     print "rescan volumes...\n";
6251     my $volid_hash = scan_volids($cfg, $vmid);
6252
6253     my $updatefn =  sub {
6254         my ($vmid) = @_;
6255
6256         my $conf = PVE::QemuConfig->load_config($vmid);
6257
6258         PVE::QemuConfig->check_lock($conf);
6259
6260         my $vm_volids = {};
6261         foreach my $volid (keys %$volid_hash) {
6262             my $info = $volid_hash->{$volid};
6263             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6264         }
6265
6266         my $changes = update_disksize($vmid, $conf, $vm_volids);
6267
6268         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6269     };
6270
6271     if (defined($vmid)) {
6272         if ($nolock) {
6273             &$updatefn($vmid);
6274         } else {
6275             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6276         }
6277     } else {
6278         my $vmlist = config_list();
6279         foreach my $vmid (keys %$vmlist) {
6280             if ($nolock) {
6281                 &$updatefn($vmid);
6282             } else {
6283                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6284             }
6285         }
6286     }
6287 }
6288
6289 sub restore_vma_archive {
6290     my ($archive, $vmid, $user, $opts, $comp) = @_;
6291
6292     my $readfrom = $archive;
6293
6294     my $cfg = PVE::Storage::config();
6295     my $commands = [];
6296     my $bwlimit = $opts->{bwlimit};
6297
6298     my $dbg_cmdstring = '';
6299     my $add_pipe = sub {
6300         my ($cmd) = @_;
6301         push @$commands, $cmd;
6302         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6303         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6304         $readfrom = '-';
6305     };
6306
6307     my $input = undef;
6308     if ($archive eq '-') {
6309         $input = '<&STDIN';
6310     } else {
6311         # If we use a backup from a PVE defined storage we also consider that
6312         # storage's rate limit:
6313         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6314         if (defined($volid)) {
6315             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6316             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6317             if ($readlimit) {
6318                 print STDERR "applying read rate limit: $readlimit\n";
6319                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6320                 $add_pipe->($cstream);
6321             }
6322         }
6323     }
6324
6325     if ($comp) {
6326         my $cmd;
6327         if ($comp eq 'gzip') {
6328             $cmd = ['zcat', $readfrom];
6329         } elsif ($comp eq 'lzop') {
6330             $cmd = ['lzop', '-d', '-c', $readfrom];
6331         } else {
6332             die "unknown compression method '$comp'\n";
6333         }
6334         $add_pipe->($cmd);
6335     }
6336
6337     my $tmpdir = "/var/tmp/vzdumptmp$$";
6338     rmtree $tmpdir;
6339
6340     # disable interrupts (always do cleanups)
6341     local $SIG{INT} =
6342         local $SIG{TERM} =
6343         local $SIG{QUIT} =
6344         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6345
6346     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6347     POSIX::mkfifo($mapfifo, 0600);
6348     my $fifofh;
6349
6350     my $openfifo = sub {
6351         open($fifofh, '>', $mapfifo) || die $!;
6352     };
6353
6354     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6355
6356     my $oldtimeout;
6357     my $timeout = 5;
6358
6359     my $devinfo = {};
6360
6361     my $rpcenv = PVE::RPCEnvironment::get();
6362
6363     my $conffile = PVE::QemuConfig->config_file($vmid);
6364     my $tmpfn = "$conffile.$$.tmp";
6365
6366     # Note: $oldconf is undef if VM does not exists
6367     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6368     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6369
6370     my %storage_limits;
6371
6372     my $print_devmap = sub {
6373         my $virtdev_hash = {};
6374
6375         my $cfgfn = "$tmpdir/qemu-server.conf";
6376
6377         # we can read the config - that is already extracted
6378         my $fh = IO::File->new($cfgfn, "r") ||
6379             "unable to read qemu-server.conf - $!\n";
6380
6381         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6382         if (-f $fwcfgfn) {
6383             my $pve_firewall_dir = '/etc/pve/firewall';
6384             mkdir $pve_firewall_dir; # make sure the dir exists
6385             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6386         }
6387
6388         while (defined(my $line = <$fh>)) {
6389             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6390                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6391                 die "archive does not contain data for drive '$virtdev'\n"
6392                     if !$devinfo->{$devname};
6393                 if (defined($opts->{storage})) {
6394                     $storeid = $opts->{storage} || 'local';
6395                 } elsif (!$storeid) {
6396                     $storeid = 'local';
6397                 }
6398                 $format = 'raw' if !$format;
6399                 $devinfo->{$devname}->{devname} = $devname;
6400                 $devinfo->{$devname}->{virtdev} = $virtdev;
6401                 $devinfo->{$devname}->{format} = $format;
6402                 $devinfo->{$devname}->{storeid} = $storeid;
6403
6404                 # check permission on storage
6405                 my $pool = $opts->{pool}; # todo: do we need that?
6406                 if ($user ne 'root@pam') {
6407                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6408                 }
6409
6410                 $storage_limits{$storeid} = $bwlimit;
6411
6412                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6413             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6414                 my $virtdev = $1;
6415                 my $drive = parse_drive($virtdev, $2);
6416                 if (drive_is_cloudinit($drive)) {
6417                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6418                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6419                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6420
6421                     my $d = {
6422                         format => $format,
6423                         storeid => $opts->{storage} // $storeid,
6424                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6425                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6426                         name => "vm-$vmid-cloudinit",
6427                         is_cloudinit => 1,
6428                     };
6429                     $virtdev_hash->{$virtdev} = $d;
6430                 }
6431             }
6432         }
6433
6434         foreach my $key (keys %storage_limits) {
6435             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6436             next if !$limit;
6437             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6438             $storage_limits{$key} = $limit * 1024;
6439         }
6440
6441         foreach my $devname (keys %$devinfo) {
6442             die "found no device mapping information for device '$devname'\n"
6443                 if !$devinfo->{$devname}->{virtdev};
6444         }
6445
6446         # create empty/temp config
6447         if ($oldconf) {
6448             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6449             foreach_drive($oldconf, sub {
6450                 my ($ds, $drive) = @_;
6451
6452                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6453
6454                 my $volid = $drive->{file};
6455                 return if !$volid || $volid =~ m|^/|;
6456
6457                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6458                 return if !$path || !$owner || ($owner != $vmid);
6459
6460                 # Note: only delete disk we want to restore
6461                 # other volumes will become unused
6462                 if ($virtdev_hash->{$ds}) {
6463                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6464                     if (my $err = $@) {
6465                         warn $err;
6466                     }
6467                 }
6468             });
6469
6470             # delete vmstate files, after the restore we have no snapshots anymore
6471             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6472                 my $snap = $oldconf->{snapshots}->{$snapname};
6473                 if ($snap->{vmstate}) {
6474                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6475                     if (my $err = $@) {
6476                         warn $err;
6477                     }
6478                 }
6479             }
6480         }
6481
6482         my $map = {};
6483         foreach my $virtdev (sort keys %$virtdev_hash) {
6484             my $d = $virtdev_hash->{$virtdev};
6485             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6486             my $storeid = $d->{storeid};
6487             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6488
6489             my $map_opts = '';
6490             if (my $limit = $storage_limits{$storeid}) {
6491                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6492             }
6493
6494             # test if requested format is supported
6495             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6496             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6497             $d->{format} = $defFormat if !$supported;
6498
6499             my $name;
6500             if ($d->{is_cloudinit}) {
6501                 $name = $d->{name};
6502                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6503             }
6504
6505             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6506             print STDERR "new volume ID is '$volid'\n";
6507             $d->{volid} = $volid;
6508
6509             PVE::Storage::activate_volumes($cfg, [$volid]);
6510
6511             my $write_zeros = 1;
6512             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6513                 $write_zeros = 0;
6514             }
6515
6516             if (!$d->{is_cloudinit}) {
6517                 my $path = PVE::Storage::path($cfg, $volid);
6518
6519                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6520
6521                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6522             }
6523             $map->{$virtdev} = $volid;
6524         }
6525
6526         $fh->seek(0, 0) || die "seek failed - $!\n";
6527
6528         my $outfd = new IO::File ($tmpfn, "w") ||
6529             die "unable to write config for VM $vmid\n";
6530
6531         my $cookie = { netcount => 0 };
6532         while (defined(my $line = <$fh>)) {
6533             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6534         }
6535
6536         $fh->close();
6537         $outfd->close();
6538     };
6539
6540     eval {
6541         # enable interrupts
6542         local $SIG{INT} =
6543             local $SIG{TERM} =
6544             local $SIG{QUIT} =
6545             local $SIG{HUP} =
6546             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6547         local $SIG{ALRM} = sub { die "got timeout\n"; };
6548
6549         $oldtimeout = alarm($timeout);
6550
6551         my $parser = sub {
6552             my $line = shift;
6553
6554             print "$line\n";
6555
6556             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6557                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6558                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6559             } elsif ($line =~ m/^CTIME: /) {
6560                 # we correctly received the vma config, so we can disable
6561                 # the timeout now for disk allocation (set to 10 minutes, so
6562                 # that we always timeout if something goes wrong)
6563                 alarm(600);
6564                 &$print_devmap();
6565                 print $fifofh "done\n";
6566                 my $tmp = $oldtimeout || 0;
6567                 $oldtimeout = undef;
6568                 alarm($tmp);
6569                 close($fifofh);
6570             }
6571         };
6572
6573         print "restore vma archive: $dbg_cmdstring\n";
6574         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6575     };
6576     my $err = $@;
6577
6578     alarm($oldtimeout) if $oldtimeout;
6579
6580     my $vollist = [];
6581     foreach my $devname (keys %$devinfo) {
6582         my $volid = $devinfo->{$devname}->{volid};
6583         push @$vollist, $volid if $volid;
6584     }
6585
6586     PVE::Storage::deactivate_volumes($cfg, $vollist);
6587
6588     unlink $mapfifo;
6589
6590     if ($err) {
6591         rmtree $tmpdir;
6592         unlink $tmpfn;
6593
6594         foreach my $devname (keys %$devinfo) {
6595             my $volid = $devinfo->{$devname}->{volid};
6596             next if !$volid;
6597             eval {
6598                 if ($volid =~ m|^/|) {
6599                     unlink $volid || die 'unlink failed\n';
6600                 } else {
6601                     PVE::Storage::vdisk_free($cfg, $volid);
6602                 }
6603                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6604             };
6605             print STDERR "unable to cleanup '$volid' - $@" if $@;
6606         }
6607         die $err;
6608     }
6609
6610     rmtree $tmpdir;
6611
6612     rename($tmpfn, $conffile) ||
6613         die "unable to commit configuration file '$conffile'\n";
6614
6615     PVE::Cluster::cfs_update(); # make sure we read new file
6616
6617     eval { rescan($vmid, 1); };
6618     warn $@ if $@;
6619 }
6620
6621 sub restore_tar_archive {
6622     my ($archive, $vmid, $user, $opts) = @_;
6623
6624     if ($archive ne '-') {
6625         my $firstfile = tar_archive_read_firstfile($archive);
6626         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6627             if $firstfile ne 'qemu-server.conf';
6628     }
6629
6630     my $storecfg = PVE::Storage::config();
6631
6632     # destroy existing data - keep empty config
6633     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6634     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6635
6636     my $tocmd = "/usr/lib/qemu-server/qmextract";
6637
6638     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6639     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6640     $tocmd .= ' --prealloc' if $opts->{prealloc};
6641     $tocmd .= ' --info' if $opts->{info};
6642
6643     # tar option "xf" does not autodetect compression when read from STDIN,
6644     # so we pipe to zcat
6645     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6646         PVE::Tools::shellquote("--to-command=$tocmd");
6647
6648     my $tmpdir = "/var/tmp/vzdumptmp$$";
6649     mkpath $tmpdir;
6650
6651     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6652     local $ENV{VZDUMP_VMID} = $vmid;
6653     local $ENV{VZDUMP_USER} = $user;
6654
6655     my $conffile = PVE::QemuConfig->config_file($vmid);
6656     my $tmpfn = "$conffile.$$.tmp";
6657
6658     # disable interrupts (always do cleanups)
6659     local $SIG{INT} =
6660         local $SIG{TERM} =
6661         local $SIG{QUIT} =
6662         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6663
6664     eval {
6665         # enable interrupts
6666         local $SIG{INT} =
6667             local $SIG{TERM} =
6668             local $SIG{QUIT} =
6669             local $SIG{HUP} =
6670             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6671
6672         if ($archive eq '-') {
6673             print "extracting archive from STDIN\n";
6674             run_command($cmd, input => "<&STDIN");
6675         } else {
6676             print "extracting archive '$archive'\n";
6677             run_command($cmd);
6678         }
6679
6680         return if $opts->{info};
6681
6682         # read new mapping
6683         my $map = {};
6684         my $statfile = "$tmpdir/qmrestore.stat";
6685         if (my $fd = IO::File->new($statfile, "r")) {
6686             while (defined (my $line = <$fd>)) {
6687                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6688                     $map->{$1} = $2 if $1;
6689                 } else {
6690                     print STDERR "unable to parse line in statfile - $line\n";
6691                 }
6692             }
6693             $fd->close();
6694         }
6695
6696         my $confsrc = "$tmpdir/qemu-server.conf";
6697
6698         my $srcfd = new IO::File($confsrc, "r") ||
6699             die "unable to open file '$confsrc'\n";
6700
6701         my $outfd = new IO::File ($tmpfn, "w") ||
6702             die "unable to write config for VM $vmid\n";
6703
6704         my $cookie = { netcount => 0 };
6705         while (defined (my $line = <$srcfd>)) {
6706             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6707         }
6708
6709         $srcfd->close();
6710         $outfd->close();
6711     };
6712     my $err = $@;
6713
6714     if ($err) {
6715
6716         unlink $tmpfn;
6717
6718         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6719
6720         die $err;
6721     }
6722
6723     rmtree $tmpdir;
6724
6725     rename $tmpfn, $conffile ||
6726         die "unable to commit configuration file '$conffile'\n";
6727
6728     PVE::Cluster::cfs_update(); # make sure we read new file
6729
6730     eval { rescan($vmid, 1); };
6731     warn $@ if $@;
6732 };
6733
6734 sub foreach_storage_used_by_vm {
6735     my ($conf, $func) = @_;
6736
6737     my $sidhash = {};
6738
6739     foreach_drive($conf, sub {
6740         my ($ds, $drive) = @_;
6741         return if drive_is_cdrom($drive);
6742
6743         my $volid = $drive->{file};
6744
6745         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6746         $sidhash->{$sid} = $sid if $sid;
6747     });
6748
6749     foreach my $sid (sort keys %$sidhash) {
6750         &$func($sid);
6751     }
6752 }
6753
6754 sub do_snapshots_with_qemu {
6755     my ($storecfg, $volid) = @_;
6756
6757     my $storage_name = PVE::Storage::parse_volume_id($volid);
6758     my $scfg = $storecfg->{ids}->{$storage_name};
6759
6760     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6761         return 1;
6762     }
6763
6764     if ($volid =~ m/\.(qcow2|qed)$/){
6765         return 1;
6766     }
6767
6768     return undef;
6769 }
6770
6771 sub qga_check_running {
6772     my ($vmid, $nowarn) = @_;
6773
6774     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6775     if ($@) {
6776         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6777         return 0;
6778     }
6779     return 1;
6780 }
6781
6782 sub template_create {
6783     my ($vmid, $conf, $disk) = @_;
6784
6785     my $storecfg = PVE::Storage::config();
6786
6787     foreach_drive($conf, sub {
6788         my ($ds, $drive) = @_;
6789
6790         return if drive_is_cdrom($drive);
6791         return if $disk && $ds ne $disk;
6792
6793         my $volid = $drive->{file};
6794         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6795
6796         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6797         $drive->{file} = $voliddst;
6798         $conf->{$ds} = print_drive($vmid, $drive);
6799         PVE::QemuConfig->write_config($vmid, $conf);
6800     });
6801 }
6802
6803 sub convert_iscsi_path {
6804     my ($path) = @_;
6805
6806     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6807         my $portal = $1;
6808         my $target = $2;
6809         my $lun = $3;
6810
6811         my $initiator_name = get_initiator_name();
6812
6813         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6814                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6815     }
6816
6817     die "cannot convert iscsi path '$path', unkown format\n";
6818 }
6819
6820 sub qemu_img_convert {
6821     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6822
6823     my $storecfg = PVE::Storage::config();
6824     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6825     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6826
6827     if ($src_storeid && $dst_storeid) {
6828
6829         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6830
6831         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6832         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6833
6834         my $src_format = qemu_img_format($src_scfg, $src_volname);
6835         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6836
6837         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6838         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6839
6840         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6841         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6842
6843         my $cmd = [];
6844         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6845         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6846         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6847         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6848
6849         if ($src_is_iscsi) {
6850             push @$cmd, '--image-opts';
6851             $src_path = convert_iscsi_path($src_path);
6852         } else {
6853             push @$cmd, '-f', $src_format;
6854         }
6855
6856         if ($dst_is_iscsi) {
6857             push @$cmd, '--target-image-opts';
6858             $dst_path = convert_iscsi_path($dst_path);
6859         } else {
6860             push @$cmd, '-O', $dst_format;
6861         }
6862
6863         push @$cmd, $src_path;
6864
6865         if (!$dst_is_iscsi && $is_zero_initialized) {
6866             push @$cmd, "zeroinit:$dst_path";
6867         } else {
6868             push @$cmd, $dst_path;
6869         }
6870
6871         my $parser = sub {
6872             my $line = shift;
6873             if($line =~ m/\((\S+)\/100\%\)/){
6874                 my $percent = $1;
6875                 my $transferred = int($size * $percent / 100);
6876                 my $remaining = $size - $transferred;
6877
6878                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6879             }
6880
6881         };
6882
6883         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6884         my $err = $@;
6885         die "copy failed: $err" if $err;
6886     }
6887 }
6888
6889 sub qemu_img_format {
6890     my ($scfg, $volname) = @_;
6891
6892     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6893         return $1;
6894     } else {
6895         return "raw";
6896     }
6897 }
6898
6899 sub qemu_drive_mirror {
6900     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6901
6902     $jobs = {} if !$jobs;
6903
6904     my $qemu_target;
6905     my $format;
6906     $jobs->{"drive-$drive"} = {};
6907
6908     if ($dst_volid =~ /^nbd:/) {
6909         $qemu_target = $dst_volid;
6910         $format = "nbd";
6911     } else {
6912         my $storecfg = PVE::Storage::config();
6913         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6914
6915         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6916
6917         $format = qemu_img_format($dst_scfg, $dst_volname);
6918
6919         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6920
6921         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6922     }
6923
6924     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6925     $opts->{format} = $format if $format;
6926
6927     if (defined($bwlimit)) {
6928         $opts->{speed} = $bwlimit * 1024;
6929         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6930     } else {
6931         print "drive mirror is starting for drive-$drive\n";
6932     }
6933
6934     # if a job already runs for this device we get an error, catch it for cleanup
6935     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6936     if (my $err = $@) {
6937         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6938         warn "$@\n" if $@;
6939         die "mirroring error: $err\n";
6940     }
6941
6942     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6943 }
6944
6945 sub qemu_drive_mirror_monitor {
6946     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6947
6948     eval {
6949         my $err_complete = 0;
6950
6951         while (1) {
6952             die "storage migration timed out\n" if $err_complete > 300;
6953
6954             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6955
6956             my $running_mirror_jobs = {};
6957             foreach my $stat (@$stats) {
6958                 next if $stat->{type} ne 'mirror';
6959                 $running_mirror_jobs->{$stat->{device}} = $stat;
6960             }
6961
6962             my $readycounter = 0;
6963
6964             foreach my $job (keys %$jobs) {
6965
6966                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6967                     print "$job : finished\n";
6968                     delete $jobs->{$job};
6969                     next;
6970                 }
6971
6972                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6973
6974                 my $busy = $running_mirror_jobs->{$job}->{busy};
6975                 my $ready = $running_mirror_jobs->{$job}->{ready};
6976                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6977                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6978                     my $remaining = $total - $transferred;
6979                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6980
6981                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6982                 }
6983
6984                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6985             }
6986
6987             last if scalar(keys %$jobs) == 0;
6988
6989             if ($readycounter == scalar(keys %$jobs)) {
6990                 print "all mirroring jobs are ready \n";
6991                 last if $skipcomplete; #do the complete later
6992
6993                 if ($vmiddst && $vmiddst != $vmid) {
6994                     my $agent_running = $qga && qga_check_running($vmid);
6995                     if ($agent_running) {
6996                         print "freeze filesystem\n";
6997                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6998                     } else {
6999                         print "suspend vm\n";
7000                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7001                     }
7002
7003                     # if we clone a disk for a new target vm, we don't switch the disk
7004                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7005
7006                     if ($agent_running) {
7007                         print "unfreeze filesystem\n";
7008                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7009                     } else {
7010                         print "resume vm\n";
7011                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7012                     }
7013
7014                     last;
7015                 } else {
7016
7017                     foreach my $job (keys %$jobs) {
7018                         # try to switch the disk if source and destination are on the same guest
7019                         print "$job: Completing block job...\n";
7020
7021                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7022                         if ($@ =~ m/cannot be completed/) {
7023                             print "$job: Block job cannot be completed, try again.\n";
7024                             $err_complete++;
7025                         }else {
7026                             print "$job: Completed successfully.\n";
7027                             $jobs->{$job}->{complete} = 1;
7028                         }
7029                     }
7030                 }
7031             }
7032             sleep 1;
7033         }
7034     };
7035     my $err = $@;
7036
7037     if ($err) {
7038         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7039         die "mirroring error: $err";
7040     }
7041
7042 }
7043
7044 sub qemu_blockjobs_cancel {
7045     my ($vmid, $jobs) = @_;
7046
7047     foreach my $job (keys %$jobs) {
7048         print "$job: Cancelling block job\n";
7049         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7050         $jobs->{$job}->{cancel} = 1;
7051     }
7052
7053     while (1) {
7054         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7055
7056         my $running_jobs = {};
7057         foreach my $stat (@$stats) {
7058             $running_jobs->{$stat->{device}} = $stat;
7059         }
7060
7061         foreach my $job (keys %$jobs) {
7062
7063             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7064                 print "$job: Done.\n";
7065                 delete $jobs->{$job};
7066             }
7067         }
7068
7069         last if scalar(keys %$jobs) == 0;
7070
7071         sleep 1;
7072     }
7073 }
7074
7075 sub clone_disk {
7076     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7077         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7078
7079     my $newvolid;
7080
7081     if (!$full) {
7082         print "create linked clone of drive $drivename ($drive->{file})\n";
7083         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7084         push @$newvollist, $newvolid;
7085     } else {
7086
7087         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7088         $storeid = $storage if $storage;
7089
7090         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7091         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7092
7093         print "create full clone of drive $drivename ($drive->{file})\n";
7094         my $name = undef;
7095         if (drive_is_cloudinit($drive)) {
7096             $name = "vm-$newvmid-cloudinit";
7097             $snapname = undef;
7098             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7099             if ($dst_format ne 'raw') {
7100                 $name .= ".$dst_format";
7101             }
7102         }
7103         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7104         push @$newvollist, $newvolid;
7105
7106         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7107
7108         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7109         if (!$running || $snapname) {
7110             # TODO: handle bwlimits
7111             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7112         } else {
7113
7114             my $kvmver = get_running_qemu_version ($vmid);
7115             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7116                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7117                     if $drive->{iothread};
7118             }
7119
7120             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7121         }
7122     }
7123
7124     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7125
7126     my $disk = $drive;
7127     $disk->{format} = undef;
7128     $disk->{file} = $newvolid;
7129     $disk->{size} = $size;
7130
7131     return $disk;
7132 }
7133
7134 # this only works if VM is running
7135 sub get_current_qemu_machine {
7136     my ($vmid) = @_;
7137
7138     my $cmd = { execute => 'query-machines', arguments => {} };
7139     my $res = vm_qmp_command($vmid, $cmd);
7140
7141     my ($current, $default);
7142     foreach my $e (@$res) {
7143         $default = $e->{name} if $e->{'is-default'};
7144         $current = $e->{name} if $e->{'is-current'};
7145     }
7146
7147     # fallback to the default machine if current is not supported by qemu
7148     return $current || $default || 'pc';
7149 }
7150
7151 sub get_running_qemu_version {
7152     my ($vmid) = @_;
7153     my $cmd = { execute => 'query-version', arguments => {} };
7154     my $res = vm_qmp_command($vmid, $cmd);
7155     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7156 }
7157
7158 sub qemu_machine_feature_enabled {
7159     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7160
7161     my $current_major;
7162     my $current_minor;
7163
7164     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7165
7166         $current_major = $3;
7167         $current_minor = $4;
7168
7169     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7170
7171         $current_major = $1;
7172         $current_minor = $2;
7173     }
7174
7175     return 1 if $current_major > $version_major ||
7176                 ($current_major == $version_major &&
7177                  $current_minor >= $version_minor);
7178 }
7179
7180 sub qemu_machine_pxe {
7181     my ($vmid, $conf, $machine) = @_;
7182
7183     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7184
7185     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7186         $machine .= '.pxe';
7187     }
7188
7189     return $machine;
7190 }
7191
7192 sub qemu_use_old_bios_files {
7193     my ($machine_type) = @_;
7194
7195     return if !$machine_type;
7196
7197     my $use_old_bios_files = undef;
7198
7199     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7200         $machine_type = $1;
7201         $use_old_bios_files = 1;
7202     } else {
7203         my $kvmver = kvm_user_version();
7204         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7205         # load new efi bios files on migration. So this hack is required to allow
7206         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7207         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7208         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7209     }
7210
7211     return ($use_old_bios_files, $machine_type);
7212 }
7213
7214 sub create_efidisk($$$$$) {
7215     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7216
7217     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7218     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7219
7220     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7221     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7222     PVE::Storage::activate_volumes($storecfg, [$volid]);
7223
7224     my $path = PVE::Storage::path($storecfg, $volid);
7225     eval {
7226         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7227     };
7228     die "Copying EFI vars image failed: $@" if $@;
7229
7230     return ($volid, $vars_size);
7231 }
7232
7233 sub vm_iothreads_list {
7234     my ($vmid) = @_;
7235
7236     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7237
7238     my $iothreads = {};
7239     foreach my $iothread (@$res) {
7240         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7241     }
7242
7243     return $iothreads;
7244 }
7245
7246 sub scsihw_infos {
7247     my ($conf, $drive) = @_;
7248
7249     my $maxdev = 0;
7250
7251     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7252         $maxdev = 7;
7253     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7254         $maxdev = 1;
7255     } else {
7256         $maxdev = 256;
7257     }
7258
7259     my $controller = int($drive->{index} / $maxdev);
7260     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7261
7262     return ($maxdev, $controller, $controller_prefix);
7263 }
7264
7265 sub add_hyperv_enlightenments {
7266     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7267
7268     return if $winversion < 6;
7269     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7270
7271     if ($gpu_passthrough || defined($hv_vendor_id)) {
7272         $hv_vendor_id //= 'proxmox';
7273         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7274     }
7275
7276     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7277         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7278         push @$cpuFlags , 'hv_vapic';
7279         push @$cpuFlags , 'hv_time';
7280     } else {
7281         push @$cpuFlags , 'hv_spinlocks=0xffff';
7282     }
7283
7284     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7285         push @$cpuFlags , 'hv_reset';
7286         push @$cpuFlags , 'hv_vpindex';
7287         push @$cpuFlags , 'hv_runtime';
7288     }
7289
7290     if ($winversion >= 7) {
7291         push @$cpuFlags , 'hv_relaxed';
7292
7293         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7294             push @$cpuFlags , 'hv_synic';
7295             push @$cpuFlags , 'hv_stimer';
7296         }
7297
7298         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7299             push @$cpuFlags , 'hv_ipi';
7300         }
7301     }
7302 }
7303
7304 sub windows_version {
7305     my ($ostype) = @_;
7306
7307     return 0 if !$ostype;
7308
7309     my $winversion = 0;
7310
7311     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7312         $winversion = 5;
7313     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7314         $winversion = 6;
7315     } elsif ($ostype =~ m/^win(\d+)$/) {
7316         $winversion = $1;
7317     }
7318
7319     return $winversion;
7320 }
7321
7322 sub resolve_dst_disk_format {
7323         my ($storecfg, $storeid, $src_volname, $format) = @_;
7324         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7325
7326         if (!$format) {
7327             # if no target format is specified, use the source disk format as hint
7328             if ($src_volname) {
7329                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7330                 $format = qemu_img_format($scfg, $src_volname);
7331             } else {
7332                 return $defFormat;
7333             }
7334         }
7335
7336         # test if requested format is supported - else use default
7337         my $supported = grep { $_ eq $format } @$validFormats;
7338         $format = $defFormat if !$supported;
7339         return $format;
7340 }
7341
7342 sub resolve_first_disk {
7343     my $conf = shift;
7344     my @disks = PVE::QemuServer::valid_drive_names();
7345     my $firstdisk;
7346     foreach my $ds (reverse @disks) {
7347         next if !$conf->{$ds};
7348         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7349         next if PVE::QemuServer::drive_is_cdrom($disk);
7350         $firstdisk = $ds;
7351     }
7352     return $firstdisk;
7353 }
7354
7355 sub generate_uuid {
7356     my ($uuid, $uuid_str);
7357     UUID::generate($uuid);
7358     UUID::unparse($uuid, $uuid_str);
7359     return $uuid_str;
7360 }
7361
7362 sub generate_smbios1_uuid {
7363     return "uuid=".generate_uuid();
7364 }
7365
7366 sub nbd_stop {
7367     my ($vmid) = @_;
7368
7369     vm_mon_cmd($vmid, 'nbd-server-stop');
7370 }
7371
7372 # bash completion helper
7373
7374 sub complete_backup_archives {
7375     my ($cmdname, $pname, $cvalue) = @_;
7376
7377     my $cfg = PVE::Storage::config();
7378
7379     my $storeid;
7380
7381     if ($cvalue =~ m/^([^:]+):/) {
7382         $storeid = $1;
7383     }
7384
7385     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7386
7387     my $res = [];
7388     foreach my $id (keys %$data) {
7389         foreach my $item (@{$data->{$id}}) {
7390             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7391             push @$res, $item->{volid} if defined($item->{volid});
7392         }
7393     }
7394
7395     return $res;
7396 }
7397
7398 my $complete_vmid_full = sub {
7399     my ($running) = @_;
7400
7401     my $idlist = vmstatus();
7402
7403     my $res = [];
7404
7405     foreach my $id (keys %$idlist) {
7406         my $d = $idlist->{$id};
7407         if (defined($running)) {
7408             next if $d->{template};
7409             next if $running && $d->{status} ne 'running';
7410             next if !$running && $d->{status} eq 'running';
7411         }
7412         push @$res, $id;
7413
7414     }
7415     return $res;
7416 };
7417
7418 sub complete_vmid {
7419     return &$complete_vmid_full();
7420 }
7421
7422 sub complete_vmid_stopped {
7423     return &$complete_vmid_full(0);
7424 }
7425
7426 sub complete_vmid_running {
7427     return &$complete_vmid_full(1);
7428 }
7429
7430 sub complete_storage {
7431
7432     my $cfg = PVE::Storage::config();
7433     my $ids = $cfg->{ids};
7434
7435     my $res = [];
7436     foreach my $sid (keys %$ids) {
7437         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7438         next if !$ids->{$sid}->{content}->{images};
7439         push @$res, $sid;
7440     }
7441
7442     return $res;
7443 }
7444
7445 1;