PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($vmid, $drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         my $devs = PVE::SysFSTools::lspci($id);
2250         if (!scalar(@$devs)) {
2251             die "no pci device found for '$id'\n";
2252         }
2253         push @{$res->{pciid}}, @$devs;
2254     }
2255     return $res;
2256 }
2257
2258 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2259 sub parse_net {
2260     my ($data) = @_;
2261
2262     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2263     if ($@) {
2264         warn $@;
2265         return undef;
2266     }
2267     if (!defined($res->{macaddr})) {
2268         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2269         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2270     }
2271     return $res;
2272 }
2273
2274 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2275 sub parse_ipconfig {
2276     my ($data) = @_;
2277
2278     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2279     if ($@) {
2280         warn $@;
2281         return undef;
2282     }
2283
2284     if ($res->{gw} && !$res->{ip}) {
2285         warn 'gateway specified without specifying an IP address';
2286         return undef;
2287     }
2288     if ($res->{gw6} && !$res->{ip6}) {
2289         warn 'IPv6 gateway specified without specifying an IPv6 address';
2290         return undef;
2291     }
2292     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2293         warn 'gateway specified together with DHCP';
2294         return undef;
2295     }
2296     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2297         # gw6 + auto/dhcp
2298         warn "IPv6 gateway specified together with $res->{ip6} address";
2299         return undef;
2300     }
2301
2302     if (!$res->{ip} && !$res->{ip6}) {
2303         return { ip => 'dhcp', ip6 => 'dhcp' };
2304     }
2305
2306     return $res;
2307 }
2308
2309 sub print_net {
2310     my $net = shift;
2311
2312     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2313 }
2314
2315 sub add_random_macs {
2316     my ($settings) = @_;
2317
2318     foreach my $opt (keys %$settings) {
2319         next if $opt !~ m/^net(\d+)$/;
2320         my $net = parse_net($settings->{$opt});
2321         next if !$net;
2322         $settings->{$opt} = print_net($net);
2323     }
2324 }
2325
2326 sub vm_is_volid_owner {
2327     my ($storecfg, $vmid, $volid) = @_;
2328
2329     if ($volid !~  m|^/|) {
2330         my ($path, $owner);
2331         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2332         if ($owner && ($owner == $vmid)) {
2333             return 1;
2334         }
2335     }
2336
2337     return undef;
2338 }
2339
2340 sub vmconfig_register_unused_drive {
2341     my ($storecfg, $vmid, $conf, $drive) = @_;
2342
2343     if (drive_is_cloudinit($drive)) {
2344         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2345         warn $@ if $@;
2346     } elsif (!drive_is_cdrom($drive)) {
2347         my $volid = $drive->{file};
2348         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2349             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2350         }
2351     }
2352 }
2353
2354 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2355 my $smbios1_fmt = {
2356     uuid => {
2357         type => 'string',
2358         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2359         format_description => 'UUID',
2360         description => "Set SMBIOS1 UUID.",
2361         optional => 1,
2362     },
2363     version => {
2364         type => 'string',
2365         pattern => '[A-Za-z0-9+\/]+={0,2}',
2366         format_description => 'Base64 encoded string',
2367         description => "Set SMBIOS1 version.",
2368         optional => 1,
2369     },
2370     serial => {
2371         type => 'string',
2372         pattern => '[A-Za-z0-9+\/]+={0,2}',
2373         format_description => 'Base64 encoded string',
2374         description => "Set SMBIOS1 serial number.",
2375         optional => 1,
2376     },
2377     manufacturer => {
2378         type => 'string',
2379         pattern => '[A-Za-z0-9+\/]+={0,2}',
2380         format_description => 'Base64 encoded string',
2381         description => "Set SMBIOS1 manufacturer.",
2382         optional => 1,
2383     },
2384     product => {
2385         type => 'string',
2386         pattern => '[A-Za-z0-9+\/]+={0,2}',
2387         format_description => 'Base64 encoded string',
2388         description => "Set SMBIOS1 product ID.",
2389         optional => 1,
2390     },
2391     sku => {
2392         type => 'string',
2393         pattern => '[A-Za-z0-9+\/]+={0,2}',
2394         format_description => 'Base64 encoded string',
2395         description => "Set SMBIOS1 SKU string.",
2396         optional => 1,
2397     },
2398     family => {
2399         type => 'string',
2400         pattern => '[A-Za-z0-9+\/]+={0,2}',
2401         format_description => 'Base64 encoded string',
2402         description => "Set SMBIOS1 family string.",
2403         optional => 1,
2404     },
2405     base64 => {
2406         type => 'boolean',
2407         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2408         optional => 1,
2409     },
2410 };
2411
2412 sub parse_smbios1 {
2413     my ($data) = @_;
2414
2415     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2416     warn $@ if $@;
2417     return $res;
2418 }
2419
2420 sub print_smbios1 {
2421     my ($smbios1) = @_;
2422     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2423 }
2424
2425 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2426
2427 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2428 sub verify_bootdisk {
2429     my ($value, $noerr) = @_;
2430
2431     return $value if is_valid_drivename($value);
2432
2433     return undef if $noerr;
2434
2435     die "invalid boot disk '$value'\n";
2436 }
2437
2438 sub parse_watchdog {
2439     my ($value) = @_;
2440
2441     return undef if !$value;
2442
2443     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2444     warn $@ if $@;
2445     return $res;
2446 }
2447
2448 sub parse_guest_agent {
2449     my ($value) = @_;
2450
2451     return {} if !defined($value->{agent});
2452
2453     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2454     warn $@ if $@;
2455
2456     # if the agent is disabled ignore the other potentially set properties
2457     return {} if !$res->{enabled};
2458     return $res;
2459 }
2460
2461 sub parse_vga {
2462     my ($value) = @_;
2463
2464     return {} if !$value;
2465     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2466     warn $@ if $@;
2467     return $res;
2468 }
2469
2470 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2471 sub verify_usb_device {
2472     my ($value, $noerr) = @_;
2473
2474     return $value if parse_usb_device($value);
2475
2476     return undef if $noerr;
2477
2478     die "unable to parse usb device\n";
2479 }
2480
2481 # add JSON properties for create and set function
2482 sub json_config_properties {
2483     my $prop = shift;
2484
2485     foreach my $opt (keys %$confdesc) {
2486         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2487         $prop->{$opt} = $confdesc->{$opt};
2488     }
2489
2490     return $prop;
2491 }
2492
2493 # return copy of $confdesc_cloudinit to generate documentation
2494 sub cloudinit_config_properties {
2495
2496     return dclone($confdesc_cloudinit);
2497 }
2498
2499 sub check_type {
2500     my ($key, $value) = @_;
2501
2502     die "unknown setting '$key'\n" if !$confdesc->{$key};
2503
2504     my $type = $confdesc->{$key}->{type};
2505
2506     if (!defined($value)) {
2507         die "got undefined value\n";
2508     }
2509
2510     if ($value =~ m/[\n\r]/) {
2511         die "property contains a line feed\n";
2512     }
2513
2514     if ($type eq 'boolean') {
2515         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2516         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2517         die "type check ('boolean') failed - got '$value'\n";
2518     } elsif ($type eq 'integer') {
2519         return int($1) if $value =~ m/^(\d+)$/;
2520         die "type check ('integer') failed - got '$value'\n";
2521     } elsif ($type eq 'number') {
2522         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2523         die "type check ('number') failed - got '$value'\n";
2524     } elsif ($type eq 'string') {
2525         if (my $fmt = $confdesc->{$key}->{format}) {
2526             PVE::JSONSchema::check_format($fmt, $value);
2527             return $value;
2528         }
2529         $value =~ s/^\"(.*)\"$/$1/;
2530         return $value;
2531     } else {
2532         die "internal error"
2533     }
2534 }
2535
2536 sub destroy_vm {
2537     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2538
2539     my $conf = PVE::QemuConfig->load_config($vmid);
2540
2541     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2542
2543     if ($conf->{template}) {
2544         # check if any base image is still used by a linked clone
2545         foreach_drive($conf, sub {
2546                 my ($ds, $drive) = @_;
2547                 return if drive_is_cdrom($drive);
2548
2549                 my $volid = $drive->{file};
2550                 return if !$volid || $volid =~ m|^/|;
2551
2552                 die "base volume '$volid' is still in use by linked cloned\n"
2553                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2554
2555         });
2556     }
2557
2558     # only remove disks owned by this VM
2559     foreach_drive($conf, sub {
2560         my ($ds, $drive) = @_;
2561         return if drive_is_cdrom($drive, 1);
2562
2563         my $volid = $drive->{file};
2564         return if !$volid || $volid =~ m|^/|;
2565
2566         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2567         return if !$path || !$owner || ($owner != $vmid);
2568
2569         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2570         warn "Could not remove disk '$volid', check manually: $@" if $@;
2571     });
2572
2573     # also remove unused disk
2574     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2575     PVE::Storage::foreach_volid($vmdisks, sub {
2576         my ($volid, $sid, $volname, $d) = @_;
2577         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2578         warn $@ if $@;
2579     });
2580
2581     if (defined $replacement_conf) {
2582         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2583     } else {
2584         PVE::QemuConfig->destroy_config($vmid);
2585     }
2586 }
2587
2588 sub parse_vm_config {
2589     my ($filename, $raw) = @_;
2590
2591     return undef if !defined($raw);
2592
2593     my $res = {
2594         digest => Digest::SHA::sha1_hex($raw),
2595         snapshots => {},
2596         pending => {},
2597     };
2598
2599     $filename =~ m|/qemu-server/(\d+)\.conf$|
2600         || die "got strange filename '$filename'";
2601
2602     my $vmid = $1;
2603
2604     my $conf = $res;
2605     my $descr;
2606     my $section = '';
2607
2608     my @lines = split(/\n/, $raw);
2609     foreach my $line (@lines) {
2610         next if $line =~ m/^\s*$/;
2611
2612         if ($line =~ m/^\[PENDING\]\s*$/i) {
2613             $section = 'pending';
2614             if (defined($descr)) {
2615                 $descr =~ s/\s+$//;
2616                 $conf->{description} = $descr;
2617             }
2618             $descr = undef;
2619             $conf = $res->{$section} = {};
2620             next;
2621
2622         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2623             $section = $1;
2624             if (defined($descr)) {
2625                 $descr =~ s/\s+$//;
2626                 $conf->{description} = $descr;
2627             }
2628             $descr = undef;
2629             $conf = $res->{snapshots}->{$section} = {};
2630             next;
2631         }
2632
2633         if ($line =~ m/^\#(.*)\s*$/) {
2634             $descr = '' if !defined($descr);
2635             $descr .= PVE::Tools::decode_text($1) . "\n";
2636             next;
2637         }
2638
2639         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2640             $descr = '' if !defined($descr);
2641             $descr .= PVE::Tools::decode_text($2);
2642         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2643             $conf->{snapstate} = $1;
2644         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2645             my $key = $1;
2646             my $value = $2;
2647             $conf->{$key} = $value;
2648         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2649             my $value = $1;
2650             if ($section eq 'pending') {
2651                 $conf->{delete} = $value; # we parse this later
2652             } else {
2653                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2654             }
2655         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2656             my $key = $1;
2657             my $value = $2;
2658             eval { $value = check_type($key, $value); };
2659             if ($@) {
2660                 warn "vm $vmid - unable to parse value of '$key' - $@";
2661             } else {
2662                 $key = 'ide2' if $key eq 'cdrom';
2663                 my $fmt = $confdesc->{$key}->{format};
2664                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2665                     my $v = parse_drive($key, $value);
2666                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2667                         $v->{file} = $volid;
2668                         $value = print_drive($vmid, $v);
2669                     } else {
2670                         warn "vm $vmid - unable to parse value of '$key'\n";
2671                         next;
2672                     }
2673                 }
2674
2675                 $conf->{$key} = $value;
2676             }
2677         }
2678     }
2679
2680     if (defined($descr)) {
2681         $descr =~ s/\s+$//;
2682         $conf->{description} = $descr;
2683     }
2684     delete $res->{snapstate}; # just to be sure
2685
2686     return $res;
2687 }
2688
2689 sub write_vm_config {
2690     my ($filename, $conf) = @_;
2691
2692     delete $conf->{snapstate}; # just to be sure
2693
2694     if ($conf->{cdrom}) {
2695         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2696         $conf->{ide2} = $conf->{cdrom};
2697         delete $conf->{cdrom};
2698     }
2699
2700     # we do not use 'smp' any longer
2701     if ($conf->{sockets}) {
2702         delete $conf->{smp};
2703     } elsif ($conf->{smp}) {
2704         $conf->{sockets} = $conf->{smp};
2705         delete $conf->{cores};
2706         delete $conf->{smp};
2707     }
2708
2709     my $used_volids = {};
2710
2711     my $cleanup_config = sub {
2712         my ($cref, $pending, $snapname) = @_;
2713
2714         foreach my $key (keys %$cref) {
2715             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2716                 $key eq 'snapstate' || $key eq 'pending';
2717             my $value = $cref->{$key};
2718             if ($key eq 'delete') {
2719                 die "propertry 'delete' is only allowed in [PENDING]\n"
2720                     if !$pending;
2721                 # fixme: check syntax?
2722                 next;
2723             }
2724             eval { $value = check_type($key, $value); };
2725             die "unable to parse value of '$key' - $@" if $@;
2726
2727             $cref->{$key} = $value;
2728
2729             if (!$snapname && is_valid_drivename($key)) {
2730                 my $drive = parse_drive($key, $value);
2731                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2732             }
2733         }
2734     };
2735
2736     &$cleanup_config($conf);
2737
2738     &$cleanup_config($conf->{pending}, 1);
2739
2740     foreach my $snapname (keys %{$conf->{snapshots}}) {
2741         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2742         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2743     }
2744
2745     # remove 'unusedX' settings if we re-add a volume
2746     foreach my $key (keys %$conf) {
2747         my $value = $conf->{$key};
2748         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2749             delete $conf->{$key};
2750         }
2751     }
2752
2753     my $generate_raw_config = sub {
2754         my ($conf, $pending) = @_;
2755
2756         my $raw = '';
2757
2758         # add description as comment to top of file
2759         if (defined(my $descr = $conf->{description})) {
2760             if ($descr) {
2761                 foreach my $cl (split(/\n/, $descr)) {
2762                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2763                 }
2764             } else {
2765                 $raw .= "#\n" if $pending;
2766             }
2767         }
2768
2769         foreach my $key (sort keys %$conf) {
2770             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2771             $raw .= "$key: $conf->{$key}\n";
2772         }
2773         return $raw;
2774     };
2775
2776     my $raw = &$generate_raw_config($conf);
2777
2778     if (scalar(keys %{$conf->{pending}})){
2779         $raw .= "\n[PENDING]\n";
2780         $raw .= &$generate_raw_config($conf->{pending}, 1);
2781     }
2782
2783     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2784         $raw .= "\n[$snapname]\n";
2785         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2786     }
2787
2788     return $raw;
2789 }
2790
2791 sub load_defaults {
2792
2793     my $res = {};
2794
2795     # we use static defaults from our JSON schema configuration
2796     foreach my $key (keys %$confdesc) {
2797         if (defined(my $default = $confdesc->{$key}->{default})) {
2798             $res->{$key} = $default;
2799         }
2800     }
2801
2802     return $res;
2803 }
2804
2805 sub config_list {
2806     my $vmlist = PVE::Cluster::get_vmlist();
2807     my $res = {};
2808     return $res if !$vmlist || !$vmlist->{ids};
2809     my $ids = $vmlist->{ids};
2810
2811     foreach my $vmid (keys %$ids) {
2812         my $d = $ids->{$vmid};
2813         next if !$d->{node} || $d->{node} ne $nodename;
2814         next if !$d->{type} || $d->{type} ne 'qemu';
2815         $res->{$vmid}->{exists} = 1;
2816     }
2817     return $res;
2818 }
2819
2820 # test if VM uses local resources (to prevent migration)
2821 sub check_local_resources {
2822     my ($conf, $noerr) = @_;
2823
2824     my @loc_res = ();
2825
2826     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2827     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2828
2829     push @loc_res, "ivshmem" if $conf->{ivshmem};
2830
2831     foreach my $k (keys %$conf) {
2832         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2833         # sockets are safe: they will recreated be on the target side post-migrate
2834         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2835         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2836     }
2837
2838     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2839
2840     return \@loc_res;
2841 }
2842
2843 # check if used storages are available on all nodes (use by migrate)
2844 sub check_storage_availability {
2845     my ($storecfg, $conf, $node) = @_;
2846
2847     foreach_drive($conf, sub {
2848         my ($ds, $drive) = @_;
2849
2850         my $volid = $drive->{file};
2851         return if !$volid;
2852
2853         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2854         return if !$sid;
2855
2856         # check if storage is available on both nodes
2857         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2858         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2859    });
2860 }
2861
2862 # list nodes where all VM images are available (used by has_feature API)
2863 sub shared_nodes {
2864     my ($conf, $storecfg) = @_;
2865
2866     my $nodelist = PVE::Cluster::get_nodelist();
2867     my $nodehash = { map { $_ => 1 } @$nodelist };
2868     my $nodename = PVE::INotify::nodename();
2869
2870     foreach_drive($conf, sub {
2871         my ($ds, $drive) = @_;
2872
2873         my $volid = $drive->{file};
2874         return if !$volid;
2875
2876         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2877         if ($storeid) {
2878             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2879             if ($scfg->{disable}) {
2880                 $nodehash = {};
2881             } elsif (my $avail = $scfg->{nodes}) {
2882                 foreach my $node (keys %$nodehash) {
2883                     delete $nodehash->{$node} if !$avail->{$node};
2884                 }
2885             } elsif (!$scfg->{shared}) {
2886                 foreach my $node (keys %$nodehash) {
2887                     delete $nodehash->{$node} if $node ne $nodename
2888                 }
2889             }
2890         }
2891     });
2892
2893     return $nodehash
2894 }
2895
2896 sub check_local_storage_availability {
2897     my ($conf, $storecfg) = @_;
2898
2899     my $nodelist = PVE::Cluster::get_nodelist();
2900     my $nodehash = { map { $_ => {} } @$nodelist };
2901
2902     foreach_drive($conf, sub {
2903         my ($ds, $drive) = @_;
2904
2905         my $volid = $drive->{file};
2906         return if !$volid;
2907
2908         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2909         if ($storeid) {
2910             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2911
2912             if ($scfg->{disable}) {
2913                 foreach my $node (keys %$nodehash) {
2914                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2915                 }
2916             } elsif (my $avail = $scfg->{nodes}) {
2917                 foreach my $node (keys %$nodehash) {
2918                     if (!$avail->{$node}) {
2919                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2920                     }
2921                 }
2922             }
2923         }
2924     });
2925
2926     foreach my $node (values %$nodehash) {
2927         if (my $unavail = $node->{unavailable_storages}) {
2928             $node->{unavailable_storages} = [ sort keys %$unavail ];
2929         }
2930     }
2931
2932     return $nodehash
2933 }
2934
2935 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2936 sub check_running {
2937     my ($vmid, $nocheck, $node) = @_;
2938
2939     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2940     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2941 }
2942
2943 sub vzlist {
2944
2945     my $vzlist = config_list();
2946
2947     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2948
2949     while (defined(my $de = $fd->read)) {
2950         next if $de !~ m/^(\d+)\.pid$/;
2951         my $vmid = $1;
2952         next if !defined($vzlist->{$vmid});
2953         if (my $pid = check_running($vmid)) {
2954             $vzlist->{$vmid}->{pid} = $pid;
2955         }
2956     }
2957
2958     return $vzlist;
2959 }
2960
2961 sub disksize {
2962     my ($storecfg, $conf) = @_;
2963
2964     my $bootdisk = $conf->{bootdisk};
2965     return undef if !$bootdisk;
2966     return undef if !is_valid_drivename($bootdisk);
2967
2968     return undef if !$conf->{$bootdisk};
2969
2970     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2971     return undef if !defined($drive);
2972
2973     return undef if drive_is_cdrom($drive);
2974
2975     my $volid = $drive->{file};
2976     return undef if !$volid;
2977
2978     return $drive->{size};
2979 }
2980
2981 our $vmstatus_return_properties = {
2982     vmid => get_standard_option('pve-vmid'),
2983     status => {
2984         description => "Qemu process status.",
2985         type => 'string',
2986         enum => ['stopped', 'running'],
2987     },
2988     maxmem => {
2989         description => "Maximum memory in bytes.",
2990         type => 'integer',
2991         optional => 1,
2992         renderer => 'bytes',
2993     },
2994     maxdisk => {
2995         description => "Root disk size in bytes.",
2996         type => 'integer',
2997         optional => 1,
2998         renderer => 'bytes',
2999     },
3000     name => {
3001         description => "VM name.",
3002         type => 'string',
3003         optional => 1,
3004     },
3005     qmpstatus => {
3006         description => "Qemu QMP agent status.",
3007         type => 'string',
3008         optional => 1,
3009     },
3010     pid => {
3011         description => "PID of running qemu process.",
3012         type => 'integer',
3013         optional => 1,
3014     },
3015     uptime => {
3016         description => "Uptime.",
3017         type => 'integer',
3018         optional => 1,
3019         renderer => 'duration',
3020     },
3021     cpus => {
3022         description => "Maximum usable CPUs.",
3023         type => 'number',
3024         optional => 1,
3025     },
3026     lock => {
3027         description => "The current config lock, if any.",
3028         type => 'string',
3029         optional => 1,
3030     },
3031     tags => {
3032         description  => "The current configured tags, if any",
3033         type => 'string',
3034         optional => 1,
3035     },
3036 };
3037
3038 my $last_proc_pid_stat;
3039
3040 # get VM status information
3041 # This must be fast and should not block ($full == false)
3042 # We only query KVM using QMP if $full == true (this can be slow)
3043 sub vmstatus {
3044     my ($opt_vmid, $full) = @_;
3045
3046     my $res = {};
3047
3048     my $storecfg = PVE::Storage::config();
3049
3050     my $list = vzlist();
3051     my $defaults = load_defaults();
3052
3053     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3054
3055     my $cpucount = $cpuinfo->{cpus} || 1;
3056
3057     foreach my $vmid (keys %$list) {
3058         next if $opt_vmid && ($vmid ne $opt_vmid);
3059
3060         my $conf = PVE::QemuConfig->load_config($vmid);
3061
3062         my $d = { vmid => $vmid };
3063         $d->{pid} = $list->{$vmid}->{pid};
3064
3065         # fixme: better status?
3066         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3067
3068         my $size = disksize($storecfg, $conf);
3069         if (defined($size)) {
3070             $d->{disk} = 0; # no info available
3071             $d->{maxdisk} = $size;
3072         } else {
3073             $d->{disk} = 0;
3074             $d->{maxdisk} = 0;
3075         }
3076
3077         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3078             * ($conf->{cores} || $defaults->{cores});
3079         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3080         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3081
3082         $d->{name} = $conf->{name} || "VM $vmid";
3083         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3084             : $defaults->{memory}*(1024*1024);
3085
3086         if ($conf->{balloon}) {
3087             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3088             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3089                 : $defaults->{shares};
3090         }
3091
3092         $d->{uptime} = 0;
3093         $d->{cpu} = 0;
3094         $d->{mem} = 0;
3095
3096         $d->{netout} = 0;
3097         $d->{netin} = 0;
3098
3099         $d->{diskread} = 0;
3100         $d->{diskwrite} = 0;
3101
3102         $d->{template} = PVE::QemuConfig->is_template($conf);
3103
3104         $d->{serial} = 1 if conf_has_serial($conf);
3105         $d->{lock} = $conf->{lock} if $conf->{lock};
3106         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3107
3108         $res->{$vmid} = $d;
3109     }
3110
3111     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3112     foreach my $dev (keys %$netdev) {
3113         next if $dev !~ m/^tap([1-9]\d*)i/;
3114         my $vmid = $1;
3115         my $d = $res->{$vmid};
3116         next if !$d;
3117
3118         $d->{netout} += $netdev->{$dev}->{receive};
3119         $d->{netin} += $netdev->{$dev}->{transmit};
3120
3121         if ($full) {
3122             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3123             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3124         }
3125
3126     }
3127
3128     my $ctime = gettimeofday;
3129
3130     foreach my $vmid (keys %$list) {
3131
3132         my $d = $res->{$vmid};
3133         my $pid = $d->{pid};
3134         next if !$pid;
3135
3136         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3137         next if !$pstat; # not running
3138
3139         my $used = $pstat->{utime} + $pstat->{stime};
3140
3141         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3142
3143         if ($pstat->{vsize}) {
3144             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3145         }
3146
3147         my $old = $last_proc_pid_stat->{$pid};
3148         if (!$old) {
3149             $last_proc_pid_stat->{$pid} = {
3150                 time => $ctime,
3151                 used => $used,
3152                 cpu => 0,
3153             };
3154             next;
3155         }
3156
3157         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3158
3159         if ($dtime > 1000) {
3160             my $dutime = $used -  $old->{used};
3161
3162             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3163             $last_proc_pid_stat->{$pid} = {
3164                 time => $ctime,
3165                 used => $used,
3166                 cpu => $d->{cpu},
3167             };
3168         } else {
3169             $d->{cpu} = $old->{cpu};
3170         }
3171     }
3172
3173     return $res if !$full;
3174
3175     my $qmpclient = PVE::QMPClient->new();
3176
3177     my $ballooncb = sub {
3178         my ($vmid, $resp) = @_;
3179
3180         my $info = $resp->{'return'};
3181         return if !$info->{max_mem};
3182
3183         my $d = $res->{$vmid};
3184
3185         # use memory assigned to VM
3186         $d->{maxmem} = $info->{max_mem};
3187         $d->{balloon} = $info->{actual};
3188
3189         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3190             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3191             $d->{freemem} = $info->{free_mem};
3192         }
3193
3194         $d->{ballooninfo} = $info;
3195     };
3196
3197     my $blockstatscb = sub {
3198         my ($vmid, $resp) = @_;
3199         my $data = $resp->{'return'} || [];
3200         my $totalrdbytes = 0;
3201         my $totalwrbytes = 0;
3202
3203         for my $blockstat (@$data) {
3204             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3205             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3206
3207             $blockstat->{device} =~ s/drive-//;
3208             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3209         }
3210         $res->{$vmid}->{diskread} = $totalrdbytes;
3211         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3212     };
3213
3214     my $statuscb = sub {
3215         my ($vmid, $resp) = @_;
3216
3217         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3218         # this fails if ballon driver is not loaded, so this must be
3219         # the last commnand (following command are aborted if this fails).
3220         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3221
3222         my $status = 'unknown';
3223         if (!defined($status = $resp->{'return'}->{status})) {
3224             warn "unable to get VM status\n";
3225             return;
3226         }
3227
3228         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3229     };
3230
3231     foreach my $vmid (keys %$list) {
3232         next if $opt_vmid && ($vmid ne $opt_vmid);
3233         next if !$res->{$vmid}->{pid}; # not running
3234         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3235     }
3236
3237     $qmpclient->queue_execute(undef, 2);
3238
3239     foreach my $vmid (keys %$list) {
3240         next if $opt_vmid && ($vmid ne $opt_vmid);
3241         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3242     }
3243
3244     return $res;
3245 }
3246
3247 sub foreach_drive {
3248     my ($conf, $func, @param) = @_;
3249
3250     foreach my $ds (valid_drive_names()) {
3251         next if !defined($conf->{$ds});
3252
3253         my $drive = parse_drive($ds, $conf->{$ds});
3254         next if !$drive;
3255
3256         &$func($ds, $drive, @param);
3257     }
3258 }
3259
3260 sub foreach_volid {
3261     my ($conf, $func, @param) = @_;
3262
3263     my $volhash = {};
3264
3265     my $test_volid = sub {
3266         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3267
3268         return if !$volid;
3269
3270         $volhash->{$volid}->{cdrom} //= 1;
3271         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3272
3273         $volhash->{$volid}->{replicate} //= 0;
3274         $volhash->{$volid}->{replicate} = 1 if $replicate;
3275
3276         $volhash->{$volid}->{shared} //= 0;
3277         $volhash->{$volid}->{shared} = 1 if $shared;
3278
3279         $volhash->{$volid}->{referenced_in_config} //= 0;
3280         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3281
3282         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3283             if defined($snapname);
3284         $volhash->{$volid}->{size} = $size if $size;
3285     };
3286
3287     foreach_drive($conf, sub {
3288         my ($ds, $drive) = @_;
3289         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3290     });
3291
3292     foreach my $snapname (keys %{$conf->{snapshots}}) {
3293         my $snap = $conf->{snapshots}->{$snapname};
3294         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3295         foreach_drive($snap, sub {
3296             my ($ds, $drive) = @_;
3297             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3298         });
3299     }
3300
3301     foreach my $volid (keys %$volhash) {
3302         &$func($volid, $volhash->{$volid}, @param);
3303     }
3304 }
3305
3306 sub conf_has_serial {
3307     my ($conf) = @_;
3308
3309     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3310         if ($conf->{"serial$i"}) {
3311             return 1;
3312         }
3313     }
3314
3315     return 0;
3316 }
3317
3318 sub conf_has_audio {
3319     my ($conf, $id) = @_;
3320
3321     $id //= 0;
3322     my $audio = $conf->{"audio$id"};
3323     return undef if !defined($audio);
3324
3325     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3326     my $audiodriver = $audioproperties->{driver} // 'spice';
3327
3328     return {
3329         dev => $audioproperties->{device},
3330         dev_id => "audiodev$id",
3331         backend => $audiodriver,
3332         backend_id => "$audiodriver-backend${id}",
3333     };
3334 }
3335
3336 sub vga_conf_has_spice {
3337     my ($vga) = @_;
3338
3339     my $vgaconf = parse_vga($vga);
3340     my $vgatype = $vgaconf->{type};
3341     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3342
3343     return $1 || 1;
3344 }
3345
3346 sub is_native($) {
3347     my ($arch) = @_;
3348     return get_host_arch() eq $arch;
3349 }
3350
3351 sub get_vm_arch {
3352     my ($conf) = @_;
3353     return $conf->{arch} // get_host_arch();
3354 }
3355
3356 my $default_machines = {
3357     x86_64 => 'pc',
3358     aarch64 => 'virt',
3359 };
3360
3361 sub get_vm_machine {
3362     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3363
3364     my $machine = $forcemachine || $conf->{machine};
3365
3366     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3367         $arch //= 'x86_64';
3368         $machine ||= $default_machines->{$arch};
3369         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3370     }
3371
3372     return $machine;
3373 }
3374
3375 sub get_ovmf_files($) {
3376     my ($arch) = @_;
3377
3378     my $ovmf = $OVMF->{$arch}
3379         or die "no OVMF images known for architecture '$arch'\n";
3380
3381     return @$ovmf;
3382 }
3383
3384 my $Arch2Qemu = {
3385     aarch64 => '/usr/bin/qemu-system-aarch64',
3386     x86_64 => '/usr/bin/qemu-system-x86_64',
3387 };
3388 sub get_command_for_arch($) {
3389     my ($arch) = @_;
3390     return '/usr/bin/kvm' if is_native($arch);
3391
3392     my $cmd = $Arch2Qemu->{$arch}
3393         or die "don't know how to emulate architecture '$arch'\n";
3394     return $cmd;
3395 }
3396
3397 sub get_cpu_options {
3398     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3399
3400     my $cpuFlags = [];
3401     my $ostype = $conf->{ostype};
3402
3403     my $cpu = $kvm ? "kvm64" : "qemu64";
3404     if ($arch eq 'aarch64') {
3405         $cpu = 'cortex-a57';
3406     }
3407     my $hv_vendor_id;
3408     if (my $cputype = $conf->{cpu}) {
3409         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3410             or die "Cannot parse cpu description: $cputype\n";
3411         $cpu = $cpuconf->{cputype};
3412         $kvm_off = 1 if $cpuconf->{hidden};
3413         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3414
3415         if (defined(my $flags = $cpuconf->{flags})) {
3416             push @$cpuFlags, split(";", $flags);
3417         }
3418     }
3419
3420     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3421
3422     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3423
3424     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3425
3426     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3427
3428     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3429
3430         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3431         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3432     }
3433
3434     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3435
3436     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3437
3438     push @$cpuFlags, 'kvm=off' if $kvm_off;
3439
3440     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3441         push @$cpuFlags, "vendor=${cpu_vendor}"
3442             if $cpu_vendor ne 'default';
3443     } elsif ($arch ne 'aarch64') {
3444         die "internal error"; # should not happen
3445     }
3446
3447     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3448
3449     return ('-cpu', $cpu);
3450 }
3451
3452 sub config_to_command {
3453     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3454
3455     my $cmd = [];
3456     my $globalFlags = [];
3457     my $machineFlags = [];
3458     my $rtcFlags = [];
3459     my $devices = [];
3460     my $pciaddr = '';
3461     my $bridges = {};
3462     my $vernum = 0; # unknown
3463     my $ostype = $conf->{ostype};
3464     my $winversion = windows_version($ostype);
3465     my $kvm = $conf->{kvm};
3466
3467     my $arch = get_vm_arch($conf);
3468     my $kvm_binary = get_command_for_arch($arch);
3469     my $kvmver = kvm_user_version($kvm_binary);
3470
3471     my $add_pve_version = min_version($kvmver, 4, 1);
3472
3473     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3474     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3475     $kvm //= 1 if is_native($arch);
3476
3477     if ($kvm) {
3478         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3479             if !defined kvm_version();
3480     }
3481
3482     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3483         $vernum = $1*1000000+$2*1000;
3484     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000+$3;
3486     }
3487
3488     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3489
3490     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3491     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3492     my $use_old_bios_files = undef;
3493     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3494
3495     my $cpuunits = defined($conf->{cpuunits}) ?
3496             $conf->{cpuunits} : $defaults->{cpuunits};
3497
3498     push @$cmd, $kvm_binary;
3499
3500     push @$cmd, '-id', $vmid;
3501
3502     my $vmname = $conf->{name} || "vm$vmid";
3503
3504     push @$cmd, '-name', $vmname;
3505
3506     my $use_virtio = 0;
3507
3508     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3509     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3510     push @$cmd, '-mon', "chardev=qmp,mode=control";
3511
3512     if (min_version($machine_version, 2, 12)) {
3513         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3514         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3515     }
3516
3517     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3518
3519     push @$cmd, '-daemonize';
3520
3521     if ($conf->{smbios1}) {
3522         my $smbios_conf = parse_smbios1($conf->{smbios1});
3523         if ($smbios_conf->{base64}) {
3524             # Do not pass base64 flag to qemu
3525             delete $smbios_conf->{base64};
3526             my $smbios_string = "";
3527             foreach my $key (keys %$smbios_conf) {
3528                 my $value;
3529                 if ($key eq "uuid") {
3530                     $value = $smbios_conf->{uuid}
3531                 } else {
3532                     $value = decode_base64($smbios_conf->{$key});
3533                 }
3534                 # qemu accepts any binary data, only commas need escaping by double comma
3535                 $value =~ s/,/,,/g;
3536                 $smbios_string .= "," . $key . "=" . $value if $value;
3537             }
3538             push @$cmd, '-smbios', "type=1" . $smbios_string;
3539         } else {
3540             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3541         }
3542     }
3543
3544     if ($conf->{vmgenid}) {
3545         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3546     }
3547
3548     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3549     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3550         die "uefi base image not found\n" if ! -f $ovmf_code;
3551
3552         my $path;
3553         my $format;
3554         if (my $efidisk = $conf->{efidisk0}) {
3555             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3556             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3557             $format = $d->{format};
3558             if ($storeid) {
3559                 $path = PVE::Storage::path($storecfg, $d->{file});
3560                 if (!defined($format)) {
3561                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3562                     $format = qemu_img_format($scfg, $volname);
3563                 }
3564             } else {
3565                 $path = $d->{file};
3566                 die "efidisk format must be specified\n"
3567                     if !defined($format);
3568             }
3569         } else {
3570             warn "no efidisk configured! Using temporary efivars disk.\n";
3571             $path = "/tmp/$vmid-ovmf.fd";
3572             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3573             $format = 'raw';
3574         }
3575
3576         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3577         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3578     }
3579
3580     # load q35 config
3581     if ($q35) {
3582         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3583         if (min_version($machine_version, 4, 0)) {
3584             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3585         } else {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3587         }
3588     }
3589
3590     # add usb controllers
3591     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3592     push @$devices, @usbcontrollers if @usbcontrollers;
3593     my $vga = parse_vga($conf->{vga});
3594
3595     my $qxlnum = vga_conf_has_spice($conf->{vga});
3596     $vga->{type} = 'qxl' if $qxlnum;
3597
3598     if (!$vga->{type}) {
3599         if ($arch eq 'aarch64') {
3600             $vga->{type} = 'virtio';
3601         } elsif (min_version($machine_version, 2, 9)) {
3602             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3603         } else {
3604             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3605         }
3606     }
3607
3608     # enable absolute mouse coordinates (needed by vnc)
3609     my $tablet;
3610     if (defined($conf->{tablet})) {
3611         $tablet = $conf->{tablet};
3612     } else {
3613         $tablet = $defaults->{tablet};
3614         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3615         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3616     }
3617
3618     if ($tablet) {
3619         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3620         my $kbd = print_keyboarddevice_full($conf, $arch);
3621         push @$devices, '-device', $kbd if defined($kbd);
3622     }
3623
3624     my $kvm_off = 0;
3625     my $gpu_passthrough;
3626
3627     # host pci devices
3628     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3629         my $id = "hostpci$i";
3630         my $d = parse_hostpci($conf->{$id});
3631         next if !$d;
3632
3633         if (my $pcie = $d->{pcie}) {
3634             die "q35 machine model is not enabled" if !$q35;
3635             # win7 wants to have the pcie devices directly on the pcie bus
3636             # instead of in the root port
3637             if ($winversion == 7) {
3638                 $pciaddr = print_pcie_addr("${id}bus0");
3639             } else {
3640                 # add more root ports if needed, 4 are present by default
3641                 # by pve-q35 cfgs, rest added here on demand.
3642                 if ($i > 3) {
3643                     push @$devices, '-device', print_pcie_root_port($i);
3644                 }
3645                 $pciaddr = print_pcie_addr($id);
3646             }
3647         } else {
3648             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3649         }
3650
3651         my $xvga = '';
3652         if ($d->{'x-vga'}) {
3653             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3654             $kvm_off = 1;
3655             $vga->{type} = 'none' if !defined($conf->{vga});
3656             $gpu_passthrough = 1;
3657         }
3658
3659         my $pcidevices = $d->{pciid};
3660         my $multifunction = 1 if @$pcidevices > 1;
3661
3662         my $sysfspath;
3663         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3664             my $pci_id = $pcidevices->[0]->{id};
3665             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3666             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3667         } elsif ($d->{mdev}) {
3668             warn "ignoring mediated device '$id' with multifunction device\n";
3669         }
3670
3671         my $j=0;
3672         foreach my $pcidevice (@$pcidevices) {
3673             my $devicestr = "vfio-pci";
3674
3675             if ($sysfspath) {
3676                 $devicestr .= ",sysfsdev=$sysfspath";
3677             } else {
3678                 $devicestr .= ",host=$pcidevice->{id}";
3679             }
3680
3681             my $mf_addr = $multifunction ? ".$j" : '';
3682             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3683
3684             if ($j == 0) {
3685                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3686                 $devicestr .= "$xvga";
3687                 $devicestr .= ",multifunction=on" if $multifunction;
3688                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3689             }
3690
3691             push @$devices, '-device', $devicestr;
3692             $j++;
3693         }
3694     }
3695
3696     # usb devices
3697     my $usb_dev_features = {};
3698     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3699
3700     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3701     push @$devices, @usbdevices if @usbdevices;
3702     # serial devices
3703     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3704         if (my $path = $conf->{"serial$i"}) {
3705             if ($path eq 'socket') {
3706                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3708                 # On aarch64, serial0 is the UART device. Qemu only allows
3709                 # connecting UART devices via the '-serial' command line, as
3710                 # the device has a fixed slot on the hardware...
3711                 if ($arch eq 'aarch64' && $i == 0) {
3712                     push @$devices, '-serial', "chardev:serial$i";
3713                 } else {
3714                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3715                 }
3716             } else {
3717                 die "no such serial device\n" if ! -c $path;
3718                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3719                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3720             }
3721         }
3722     }
3723
3724     # parallel devices
3725     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3726         if (my $path = $conf->{"parallel$i"}) {
3727             die "no such parallel device\n" if ! -c $path;
3728             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3729             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3730             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3731         }
3732     }
3733
3734     if (my $audio = conf_has_audio($conf)) {
3735
3736         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3737
3738         my $id = $audio->{dev_id};
3739         if ($audio->{dev} eq 'AC97') {
3740             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3741         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3742             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3743             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3744             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3745         } else {
3746             die "unkown audio device '$audio->{dev}', implement me!";
3747         }
3748
3749         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3750     }
3751
3752     my $sockets = 1;
3753     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3754     $sockets = $conf->{sockets} if  $conf->{sockets};
3755
3756     my $cores = $conf->{cores} || 1;
3757
3758     my $maxcpus = $sockets * $cores;
3759
3760     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3761
3762     my $allowed_vcpus = $cpuinfo->{cpus};
3763
3764     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3765         if ($allowed_vcpus < $maxcpus);
3766
3767     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3768
3769         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3770         for (my $i = 2; $i <= $vcpus; $i++)  {
3771             my $cpustr = print_cpu_device($conf,$i);
3772             push @$cmd, '-device', $cpustr;
3773         }
3774
3775     } else {
3776
3777         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3778     }
3779     push @$cmd, '-nodefaults';
3780
3781     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3782
3783     my $bootindex_hash = {};
3784     my $i = 1;
3785     foreach my $o (split(//, $bootorder)) {
3786         $bootindex_hash->{$o} = $i*100;
3787         $i++;
3788     }
3789
3790     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3791
3792     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3793
3794     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3795
3796     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3797         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3798         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3799         push @$cmd,  '-vnc', "unix:$socket,password";
3800     } else {
3801         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3802         push @$cmd, '-nographic';
3803     }
3804
3805     # time drift fix
3806     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3807
3808     my $useLocaltime = $conf->{localtime};
3809
3810     if ($winversion >= 5) { # windows
3811         $useLocaltime = 1 if !defined($conf->{localtime});
3812
3813         # use time drift fix when acpi is enabled
3814         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3815             $tdf = 1 if !defined($conf->{tdf});
3816         }
3817     }
3818
3819     if ($winversion >= 6) {
3820         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3821         push @$cmd, '-no-hpet';
3822     }
3823
3824     push @$rtcFlags, 'driftfix=slew' if $tdf;
3825
3826     if (!$kvm) {
3827         push @$machineFlags, 'accel=tcg';
3828     }
3829
3830     if ($machine_type) {
3831         push @$machineFlags, "type=${machine_type}";
3832     }
3833
3834     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3835         push @$rtcFlags, "base=$conf->{startdate}";
3836     } elsif ($useLocaltime) {
3837         push @$rtcFlags, 'base=localtime';
3838     }
3839
3840     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3841
3842     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3843
3844     push @$cmd, '-S' if $conf->{freeze};
3845
3846     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3847
3848     my $guest_agent = parse_guest_agent($conf);
3849
3850     if ($guest_agent->{enabled}) {
3851         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3852         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3853
3854         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3855             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3856             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3857             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3858         } elsif ($guest_agent->{type} eq 'isa') {
3859             push @$devices, '-device', "isa-serial,chardev=qga0";
3860         }
3861     }
3862
3863     my $spice_port;
3864
3865     if ($qxlnum) {
3866         if ($qxlnum > 1) {
3867             if ($winversion){
3868                 for(my $i = 1; $i < $qxlnum; $i++){
3869                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3870                 }
3871             } else {
3872                 # assume other OS works like Linux
3873                 my ($ram, $vram) = ("134217728", "67108864");
3874                 if ($vga->{memory}) {
3875                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3876                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3877                 }
3878                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3879                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3880             }
3881         }
3882
3883         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3884
3885         my $nodename = PVE::INotify::nodename();
3886         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3887         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3888         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3889
3890         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3891         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3892         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3893
3894         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3895         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3896
3897         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3898         if ($spice_enhancement->{foldersharing}) {
3899             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3900             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3901         }
3902
3903         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3904         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3905         push @$devices, '-spice', "$spice_opts";
3906     }
3907
3908     # enable balloon by default, unless explicitly disabled
3909     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3910         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3911         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3912     }
3913
3914     if ($conf->{watchdog}) {
3915         my $wdopts = parse_watchdog($conf->{watchdog});
3916         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3917         my $watchdog = $wdopts->{model} || 'i6300esb';
3918         push @$devices, '-device', "$watchdog$pciaddr";
3919         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3920     }
3921
3922     my $vollist = [];
3923     my $scsicontroller = {};
3924     my $ahcicontroller = {};
3925     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3926
3927     # Add iscsi initiator name if available
3928     if (my $initiator = get_initiator_name()) {
3929         push @$devices, '-iscsi', "initiator-name=$initiator";
3930     }
3931
3932     foreach_drive($conf, sub {
3933         my ($ds, $drive) = @_;
3934
3935         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3936             push @$vollist, $drive->{file};
3937         }
3938
3939         # ignore efidisk here, already added in bios/fw handling code above
3940         return if $drive->{interface} eq 'efidisk';
3941
3942         $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944         if (drive_is_cdrom ($drive)) {
3945             if ($bootindex_hash->{d}) {
3946                 $drive->{bootindex} = $bootindex_hash->{d};
3947                 $bootindex_hash->{d} += 1;
3948             }
3949         } else {
3950             if ($bootindex_hash->{c}) {
3951                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3952                 $bootindex_hash->{c} += 1;
3953             }
3954         }
3955
3956         if($drive->{interface} eq 'virtio'){
3957            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3958         }
3959
3960         if ($drive->{interface} eq 'scsi') {
3961
3962             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3963
3964             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3965             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3966
3967             my $iothread = '';
3968             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3969                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3970                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3971             } elsif ($drive->{iothread}) {
3972                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3973             }
3974
3975             my $queues = '';
3976             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3977                 $queues = ",num_queues=$drive->{queues}";
3978             }
3979
3980             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3981             $scsicontroller->{$controller}=1;
3982         }
3983
3984         if ($drive->{interface} eq 'sata') {
3985            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3986            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3987            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3988            $ahcicontroller->{$controller}=1;
3989         }
3990
3991         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3992         push @$devices, '-drive',$drive_cmd;
3993         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3994     });
3995
3996     for (my $i = 0; $i < $MAX_NETS; $i++) {
3997          next if !$conf->{"net$i"};
3998          my $d = parse_net($conf->{"net$i"});
3999          next if !$d;
4000
4001          $use_virtio = 1 if $d->{model} eq 'virtio';
4002
4003          if ($bootindex_hash->{n}) {
4004             $d->{bootindex} = $bootindex_hash->{n};
4005             $bootindex_hash->{n} += 1;
4006          }
4007
4008          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4009          push @$devices, '-netdev', $netdevfull;
4010
4011          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4012          push @$devices, '-device', $netdevicefull;
4013     }
4014
4015     if ($conf->{ivshmem}) {
4016         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4017
4018         my $bus;
4019         if ($q35) {
4020             $bus = print_pcie_addr("ivshmem");
4021         } else {
4022             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4023         }
4024
4025         my $ivshmem_name = $ivshmem->{name} // $vmid;
4026         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4027
4028         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4029         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4030     }
4031
4032     if (!$q35) {
4033         # add pci bridges
4034         if (min_version($machine_version, 2, 3)) {
4035            $bridges->{1} = 1;
4036            $bridges->{2} = 1;
4037         }
4038
4039         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4040
4041         for my $k (sort {$b cmp $a} keys %$bridges) {
4042             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4043             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4044         }
4045     }
4046
4047     push @$cmd, @$devices;
4048     push @$cmd, '-rtc', join(',', @$rtcFlags)
4049         if scalar(@$rtcFlags);
4050     push @$cmd, '-machine', join(',', @$machineFlags)
4051         if scalar(@$machineFlags);
4052     push @$cmd, '-global', join(',', @$globalFlags)
4053         if scalar(@$globalFlags);
4054
4055     if (my $vmstate = $conf->{vmstate}) {
4056         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4057         push @$vollist, $vmstate;
4058         push @$cmd, '-loadstate', $statepath;
4059         print "activating and using '$vmstate' as vmstate\n";
4060     }
4061
4062     # add custom args
4063     if ($conf->{args}) {
4064         my $aa = PVE::Tools::split_args($conf->{args});
4065         push @$cmd, @$aa;
4066     }
4067
4068     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4069 }
4070
4071 sub spice_port {
4072     my ($vmid) = @_;
4073
4074     my $res = mon_cmd($vmid, 'query-spice');
4075
4076     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4077 }
4078
4079 sub vm_devices_list {
4080     my ($vmid) = @_;
4081
4082     my $res = mon_cmd($vmid, 'query-pci');
4083     my $devices_to_check = [];
4084     my $devices = {};
4085     foreach my $pcibus (@$res) {
4086         push @$devices_to_check, @{$pcibus->{devices}},
4087     }
4088
4089     while (@$devices_to_check) {
4090         my $to_check = [];
4091         for my $d (@$devices_to_check) {
4092             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4093             next if !$d->{'pci_bridge'};
4094
4095             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4096             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4097         }
4098         $devices_to_check = $to_check;
4099     }
4100
4101     my $resblock = mon_cmd($vmid, 'query-block');
4102     foreach my $block (@$resblock) {
4103         if($block->{device} =~ m/^drive-(\S+)/){
4104                 $devices->{$1} = 1;
4105         }
4106     }
4107
4108     my $resmice = mon_cmd($vmid, 'query-mice');
4109     foreach my $mice (@$resmice) {
4110         if ($mice->{name} eq 'QEMU HID Tablet') {
4111             $devices->{tablet} = 1;
4112             last;
4113         }
4114     }
4115
4116     # for usb devices there is no query-usb
4117     # but we can iterate over the entries in
4118     # qom-list path=/machine/peripheral
4119     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4120     foreach my $per (@$resperipheral) {
4121         if ($per->{name} =~ m/^usb\d+$/) {
4122             $devices->{$per->{name}} = 1;
4123         }
4124     }
4125
4126     return $devices;
4127 }
4128
4129 sub vm_deviceplug {
4130     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4131
4132     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4133
4134     my $devices_list = vm_devices_list($vmid);
4135     return 1 if defined($devices_list->{$deviceid});
4136
4137     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4138
4139     if ($deviceid eq 'tablet') {
4140
4141         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4142
4143     } elsif ($deviceid eq 'keyboard') {
4144
4145         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4146
4147     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4148
4149         die "usb hotplug currently not reliable\n";
4150         # since we can't reliably hot unplug all added usb devices
4151         # and usb passthrough disables live migration
4152         # we disable usb hotplugging for now
4153         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4154
4155     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4156
4157         qemu_iothread_add($vmid, $deviceid, $device);
4158
4159         qemu_driveadd($storecfg, $vmid, $device);
4160         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4161
4162         qemu_deviceadd($vmid, $devicefull);
4163         eval { qemu_deviceaddverify($vmid, $deviceid); };
4164         if (my $err = $@) {
4165             eval { qemu_drivedel($vmid, $deviceid); };
4166             warn $@ if $@;
4167             die $err;
4168         }
4169
4170     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4171
4172
4173         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4174         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4175         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4176
4177         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4178
4179         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4180             qemu_iothread_add($vmid, $deviceid, $device);
4181             $devicefull .= ",iothread=iothread-$deviceid";
4182         }
4183
4184         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4185             $devicefull .= ",num_queues=$device->{queues}";
4186         }
4187
4188         qemu_deviceadd($vmid, $devicefull);
4189         qemu_deviceaddverify($vmid, $deviceid);
4190
4191     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4192
4193         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4194         qemu_driveadd($storecfg, $vmid, $device);
4195
4196         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4197         eval { qemu_deviceadd($vmid, $devicefull); };
4198         if (my $err = $@) {
4199             eval { qemu_drivedel($vmid, $deviceid); };
4200             warn $@ if $@;
4201             die $err;
4202         }
4203
4204     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4205
4206         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4207
4208         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4209         my $use_old_bios_files = undef;
4210         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4211
4212         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4213         qemu_deviceadd($vmid, $netdevicefull);
4214         eval {
4215             qemu_deviceaddverify($vmid, $deviceid);
4216             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4217         };
4218         if (my $err = $@) {
4219             eval { qemu_netdevdel($vmid, $deviceid); };
4220             warn $@ if $@;
4221             die $err;
4222         }
4223
4224     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4225
4226         my $bridgeid = $2;
4227         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4228         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4229
4230         qemu_deviceadd($vmid, $devicefull);
4231         qemu_deviceaddverify($vmid, $deviceid);
4232
4233     } else {
4234         die "can't hotplug device '$deviceid'\n";
4235     }
4236
4237     return 1;
4238 }
4239
4240 # fixme: this should raise exceptions on error!
4241 sub vm_deviceunplug {
4242     my ($vmid, $conf, $deviceid) = @_;
4243
4244     my $devices_list = vm_devices_list($vmid);
4245     return 1 if !defined($devices_list->{$deviceid});
4246
4247     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4248
4249     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4250
4251         qemu_devicedel($vmid, $deviceid);
4252
4253     } elsif ($deviceid =~ m/^usb\d+$/) {
4254
4255         die "usb hotplug currently not reliable\n";
4256         # when unplugging usb devices this way,
4257         # there may be remaining usb controllers/hubs
4258         # so we disable it for now
4259         qemu_devicedel($vmid, $deviceid);
4260         qemu_devicedelverify($vmid, $deviceid);
4261
4262     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4263
4264         qemu_devicedel($vmid, $deviceid);
4265         qemu_devicedelverify($vmid, $deviceid);
4266         qemu_drivedel($vmid, $deviceid);
4267         qemu_iothread_del($conf, $vmid, $deviceid);
4268
4269     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4270
4271         qemu_devicedel($vmid, $deviceid);
4272         qemu_devicedelverify($vmid, $deviceid);
4273         qemu_iothread_del($conf, $vmid, $deviceid);
4274
4275     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4276
4277         qemu_devicedel($vmid, $deviceid);
4278         qemu_drivedel($vmid, $deviceid);
4279         qemu_deletescsihw($conf, $vmid, $deviceid);
4280
4281     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4282
4283         qemu_devicedel($vmid, $deviceid);
4284         qemu_devicedelverify($vmid, $deviceid);
4285         qemu_netdevdel($vmid, $deviceid);
4286
4287     } else {
4288         die "can't unplug device '$deviceid'\n";
4289     }
4290
4291     return 1;
4292 }
4293
4294 sub qemu_deviceadd {
4295     my ($vmid, $devicefull) = @_;
4296
4297     $devicefull = "driver=".$devicefull;
4298     my %options =  split(/[=,]/, $devicefull);
4299
4300     mon_cmd($vmid, "device_add" , %options);
4301 }
4302
4303 sub qemu_devicedel {
4304     my ($vmid, $deviceid) = @_;
4305
4306     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4307 }
4308
4309 sub qemu_iothread_add {
4310     my($vmid, $deviceid, $device) = @_;
4311
4312     if ($device->{iothread}) {
4313         my $iothreads = vm_iothreads_list($vmid);
4314         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4315     }
4316 }
4317
4318 sub qemu_iothread_del {
4319     my($conf, $vmid, $deviceid) = @_;
4320
4321     my $confid = $deviceid;
4322     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4323         $confid = 'scsi' . $1;
4324     }
4325     my $device = parse_drive($confid, $conf->{$confid});
4326     if ($device->{iothread}) {
4327         my $iothreads = vm_iothreads_list($vmid);
4328         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4329     }
4330 }
4331
4332 sub qemu_objectadd {
4333     my($vmid, $objectid, $qomtype) = @_;
4334
4335     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4336
4337     return 1;
4338 }
4339
4340 sub qemu_objectdel {
4341     my($vmid, $objectid) = @_;
4342
4343     mon_cmd($vmid, "object-del", id => $objectid);
4344
4345     return 1;
4346 }
4347
4348 sub qemu_driveadd {
4349     my ($storecfg, $vmid, $device) = @_;
4350
4351     my $drive = print_drive_full($storecfg, $vmid, $device);
4352     $drive =~ s/\\/\\\\/g;
4353     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4354
4355     # If the command succeeds qemu prints: "OK"
4356     return 1 if $ret =~ m/OK/s;
4357
4358     die "adding drive failed: $ret\n";
4359 }
4360
4361 sub qemu_drivedel {
4362     my($vmid, $deviceid) = @_;
4363
4364     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4365     $ret =~ s/^\s+//;
4366
4367     return 1 if $ret eq "";
4368
4369     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4370     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4371
4372     die "deleting drive $deviceid failed : $ret\n";
4373 }
4374
4375 sub qemu_deviceaddverify {
4376     my ($vmid, $deviceid) = @_;
4377
4378     for (my $i = 0; $i <= 5; $i++) {
4379          my $devices_list = vm_devices_list($vmid);
4380          return 1 if defined($devices_list->{$deviceid});
4381          sleep 1;
4382     }
4383
4384     die "error on hotplug device '$deviceid'\n";
4385 }
4386
4387
4388 sub qemu_devicedelverify {
4389     my ($vmid, $deviceid) = @_;
4390
4391     # need to verify that the device is correctly removed as device_del
4392     # is async and empty return is not reliable
4393
4394     for (my $i = 0; $i <= 5; $i++) {
4395          my $devices_list = vm_devices_list($vmid);
4396          return 1 if !defined($devices_list->{$deviceid});
4397          sleep 1;
4398     }
4399
4400     die "error on hot-unplugging device '$deviceid'\n";
4401 }
4402
4403 sub qemu_findorcreatescsihw {
4404     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4405
4406     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4407
4408     my $scsihwid="$controller_prefix$controller";
4409     my $devices_list = vm_devices_list($vmid);
4410
4411     if(!defined($devices_list->{$scsihwid})) {
4412         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4413     }
4414
4415     return 1;
4416 }
4417
4418 sub qemu_deletescsihw {
4419     my ($conf, $vmid, $opt) = @_;
4420
4421     my $device = parse_drive($opt, $conf->{$opt});
4422
4423     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4424         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4425         return 1;
4426     }
4427
4428     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4429
4430     my $devices_list = vm_devices_list($vmid);
4431     foreach my $opt (keys %{$devices_list}) {
4432         if (PVE::QemuServer::is_valid_drivename($opt)) {
4433             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4434             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4435                 return 1;
4436             }
4437         }
4438     }
4439
4440     my $scsihwid="scsihw$controller";
4441
4442     vm_deviceunplug($vmid, $conf, $scsihwid);
4443
4444     return 1;
4445 }
4446
4447 sub qemu_add_pci_bridge {
4448     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4449
4450     my $bridges = {};
4451
4452     my $bridgeid;
4453
4454     print_pci_addr($device, $bridges, $arch, $machine_type);
4455
4456     while (my ($k, $v) = each %$bridges) {
4457         $bridgeid = $k;
4458     }
4459     return 1 if !defined($bridgeid) || $bridgeid < 1;
4460
4461     my $bridge = "pci.$bridgeid";
4462     my $devices_list = vm_devices_list($vmid);
4463
4464     if (!defined($devices_list->{$bridge})) {
4465         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4466     }
4467
4468     return 1;
4469 }
4470
4471 sub qemu_set_link_status {
4472     my ($vmid, $device, $up) = @_;
4473
4474     mon_cmd($vmid, "set_link", name => $device,
4475                up => $up ? JSON::true : JSON::false);
4476 }
4477
4478 sub qemu_netdevadd {
4479     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4480
4481     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4482     my %options =  split(/[=,]/, $netdev);
4483
4484     mon_cmd($vmid, "netdev_add",  %options);
4485     return 1;
4486 }
4487
4488 sub qemu_netdevdel {
4489     my ($vmid, $deviceid) = @_;
4490
4491     mon_cmd($vmid, "netdev_del", id => $deviceid);
4492 }
4493
4494 sub qemu_usb_hotplug {
4495     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4496
4497     return if !$device;
4498
4499     # remove the old one first
4500     vm_deviceunplug($vmid, $conf, $deviceid);
4501
4502     # check if xhci controller is necessary and available
4503     if ($device->{usb3}) {
4504
4505         my $devicelist = vm_devices_list($vmid);
4506
4507         if (!$devicelist->{xhci}) {
4508             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4509             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4510         }
4511     }
4512     my $d = parse_usb_device($device->{host});
4513     $d->{usb3} = $device->{usb3};
4514
4515     # add the new one
4516     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4517 }
4518
4519 sub qemu_cpu_hotplug {
4520     my ($vmid, $conf, $vcpus) = @_;
4521
4522     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4523
4524     my $sockets = 1;
4525     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4526     $sockets = $conf->{sockets} if  $conf->{sockets};
4527     my $cores = $conf->{cores} || 1;
4528     my $maxcpus = $sockets * $cores;
4529
4530     $vcpus = $maxcpus if !$vcpus;
4531
4532     die "you can't add more vcpus than maxcpus\n"
4533         if $vcpus > $maxcpus;
4534
4535     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4536
4537     if ($vcpus < $currentvcpus) {
4538
4539         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4540
4541             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4542                 qemu_devicedel($vmid, "cpu$i");
4543                 my $retry = 0;
4544                 my $currentrunningvcpus = undef;
4545                 while (1) {
4546                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4547                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4548                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4549                     $retry++;
4550                     sleep 1;
4551                 }
4552                 #update conf after each succesfull cpu unplug
4553                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4554                 PVE::QemuConfig->write_config($vmid, $conf);
4555             }
4556         } else {
4557             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4558         }
4559
4560         return;
4561     }
4562
4563     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4564     die "vcpus in running vm does not match its configuration\n"
4565         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4566
4567     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4568
4569         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4570             my $cpustr = print_cpu_device($conf, $i);
4571             qemu_deviceadd($vmid, $cpustr);
4572
4573             my $retry = 0;
4574             my $currentrunningvcpus = undef;
4575             while (1) {
4576                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4577                 last if scalar(@{$currentrunningvcpus}) == $i;
4578                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4579                 sleep 1;
4580                 $retry++;
4581             }
4582             #update conf after each succesfull cpu hotplug
4583             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4584             PVE::QemuConfig->write_config($vmid, $conf);
4585         }
4586     } else {
4587
4588         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4589             mon_cmd($vmid, "cpu-add", id => int($i));
4590         }
4591     }
4592 }
4593
4594 sub qemu_block_set_io_throttle {
4595     my ($vmid, $deviceid,
4596         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4597         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4598         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4599         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4600
4601     return if !check_running($vmid) ;
4602
4603     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4604         bps => int($bps),
4605         bps_rd => int($bps_rd),
4606         bps_wr => int($bps_wr),
4607         iops => int($iops),
4608         iops_rd => int($iops_rd),
4609         iops_wr => int($iops_wr),
4610         bps_max => int($bps_max),
4611         bps_rd_max => int($bps_rd_max),
4612         bps_wr_max => int($bps_wr_max),
4613         iops_max => int($iops_max),
4614         iops_rd_max => int($iops_rd_max),
4615         iops_wr_max => int($iops_wr_max),
4616         bps_max_length => int($bps_max_length),
4617         bps_rd_max_length => int($bps_rd_max_length),
4618         bps_wr_max_length => int($bps_wr_max_length),
4619         iops_max_length => int($iops_max_length),
4620         iops_rd_max_length => int($iops_rd_max_length),
4621         iops_wr_max_length => int($iops_wr_max_length),
4622     );
4623
4624 }
4625
4626 # old code, only used to shutdown old VM after update
4627 sub __read_avail {
4628     my ($fh, $timeout) = @_;
4629
4630     my $sel = new IO::Select;
4631     $sel->add($fh);
4632
4633     my $res = '';
4634     my $buf;
4635
4636     my @ready;
4637     while (scalar (@ready = $sel->can_read($timeout))) {
4638         my $count;
4639         if ($count = $fh->sysread($buf, 8192)) {
4640             if ($buf =~ /^(.*)\(qemu\) $/s) {
4641                 $res .= $1;
4642                 last;
4643             } else {
4644                 $res .= $buf;
4645             }
4646         } else {
4647             if (!defined($count)) {
4648                 die "$!\n";
4649             }
4650             last;
4651         }
4652     }
4653
4654     die "monitor read timeout\n" if !scalar(@ready);
4655
4656     return $res;
4657 }
4658
4659 sub qemu_block_resize {
4660     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4661
4662     my $running = check_running($vmid);
4663
4664     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4665
4666     return if !$running;
4667
4668     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4669
4670 }
4671
4672 sub qemu_volume_snapshot {
4673     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4674
4675     my $running = check_running($vmid);
4676
4677     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4678         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4679     } else {
4680         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4681     }
4682 }
4683
4684 sub qemu_volume_snapshot_delete {
4685     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4686
4687     my $running = check_running($vmid);
4688
4689     if($running) {
4690
4691         $running = undef;
4692         my $conf = PVE::QemuConfig->load_config($vmid);
4693         foreach_drive($conf, sub {
4694             my ($ds, $drive) = @_;
4695             $running = 1 if $drive->{file} eq $volid;
4696         });
4697     }
4698
4699     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4700         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4701     } else {
4702         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4703     }
4704 }
4705
4706 sub set_migration_caps {
4707     my ($vmid) = @_;
4708
4709     my $cap_ref = [];
4710
4711     my $enabled_cap = {
4712         "auto-converge" => 1,
4713         "xbzrle" => 1,
4714         "x-rdma-pin-all" => 0,
4715         "zero-blocks" => 0,
4716         "compress" => 0
4717     };
4718
4719     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4720
4721     for my $supported_capability (@$supported_capabilities) {
4722         push @$cap_ref, {
4723             capability => $supported_capability->{capability},
4724             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4725         };
4726     }
4727
4728     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4729 }
4730
4731 my $fast_plug_option = {
4732     'lock' => 1,
4733     'name' => 1,
4734     'onboot' => 1,
4735     'shares' => 1,
4736     'startup' => 1,
4737     'description' => 1,
4738     'protection' => 1,
4739     'vmstatestorage' => 1,
4740     'hookscript' => 1,
4741     'tags' => 1,
4742 };
4743
4744 # hotplug changes in [PENDING]
4745 # $selection hash can be used to only apply specified options, for
4746 # example: { cores => 1 } (only apply changed 'cores')
4747 # $errors ref is used to return error messages
4748 sub vmconfig_hotplug_pending {
4749     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4750
4751     my $defaults = load_defaults();
4752     my $arch = get_vm_arch($conf);
4753     my $machine_type = get_vm_machine($conf, undef, $arch);
4754
4755     # commit values which do not have any impact on running VM first
4756     # Note: those option cannot raise errors, we we do not care about
4757     # $selection and always apply them.
4758
4759     my $add_error = sub {
4760         my ($opt, $msg) = @_;
4761         $errors->{$opt} = "hotplug problem - $msg";
4762     };
4763
4764     my $changes = 0;
4765     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4766         if ($fast_plug_option->{$opt}) {
4767             $conf->{$opt} = $conf->{pending}->{$opt};
4768             delete $conf->{pending}->{$opt};
4769             $changes = 1;
4770         }
4771     }
4772
4773     if ($changes) {
4774         PVE::QemuConfig->write_config($vmid, $conf);
4775         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4776     }
4777
4778     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4779
4780     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4781     foreach my $opt (sort keys %$pending_delete_hash) {
4782         next if $selection && !$selection->{$opt};
4783         my $force = $pending_delete_hash->{$opt}->{force};
4784         eval {
4785             if ($opt eq 'hotplug') {
4786                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4787             } elsif ($opt eq 'tablet') {
4788                 die "skip\n" if !$hotplug_features->{usb};
4789                 if ($defaults->{tablet}) {
4790                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4791                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4792                         if $arch eq 'aarch64';
4793                 } else {
4794                     vm_deviceunplug($vmid, $conf, 'tablet');
4795                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4796                 }
4797             } elsif ($opt =~ m/^usb\d+/) {
4798                 die "skip\n";
4799                 # since we cannot reliably hot unplug usb devices
4800                 # we are disabling it
4801                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4802                 vm_deviceunplug($vmid, $conf, $opt);
4803             } elsif ($opt eq 'vcpus') {
4804                 die "skip\n" if !$hotplug_features->{cpu};
4805                 qemu_cpu_hotplug($vmid, $conf, undef);
4806             } elsif ($opt eq 'balloon') {
4807                 # enable balloon device is not hotpluggable
4808                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4809                 # here we reset the ballooning value to memory
4810                 my $balloon = $conf->{memory} || $defaults->{memory};
4811                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4812             } elsif ($fast_plug_option->{$opt}) {
4813                 # do nothing
4814             } elsif ($opt =~ m/^net(\d+)$/) {
4815                 die "skip\n" if !$hotplug_features->{network};
4816                 vm_deviceunplug($vmid, $conf, $opt);
4817             } elsif (is_valid_drivename($opt)) {
4818                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4819                 vm_deviceunplug($vmid, $conf, $opt);
4820                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4821             } elsif ($opt =~ m/^memory$/) {
4822                 die "skip\n" if !$hotplug_features->{memory};
4823                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4824             } elsif ($opt eq 'cpuunits') {
4825                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4826             } elsif ($opt eq 'cpulimit') {
4827                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4828             } else {
4829                 die "skip\n";
4830             }
4831         };
4832         if (my $err = $@) {
4833             &$add_error($opt, $err) if $err ne "skip\n";
4834         } else {
4835             # save new config if hotplug was successful
4836             delete $conf->{$opt};
4837             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4838             PVE::QemuConfig->write_config($vmid, $conf);
4839             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4840         }
4841     }
4842
4843     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4844     $apply_pending_cloudinit = sub {
4845         return if $apply_pending_cloudinit_done; # once is enough
4846         $apply_pending_cloudinit_done = 1; # once is enough
4847
4848         my ($key, $value) = @_;
4849
4850         my @cloudinit_opts = keys %$confdesc_cloudinit;
4851         foreach my $opt (keys %{$conf->{pending}}) {
4852             next if !grep { $_ eq $opt } @cloudinit_opts;
4853             $conf->{$opt} = delete $conf->{pending}->{$opt};
4854         }
4855
4856         my $new_conf = { %$conf };
4857         $new_conf->{$key} = $value;
4858         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4859     };
4860
4861     foreach my $opt (keys %{$conf->{pending}}) {
4862         next if $selection && !$selection->{$opt};
4863         my $value = $conf->{pending}->{$opt};
4864         eval {
4865             if ($opt eq 'hotplug') {
4866                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4867             } elsif ($opt eq 'tablet') {
4868                 die "skip\n" if !$hotplug_features->{usb};
4869                 if ($value == 1) {
4870                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4871                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4872                         if $arch eq 'aarch64';
4873                 } elsif ($value == 0) {
4874                     vm_deviceunplug($vmid, $conf, 'tablet');
4875                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4876                 }
4877             } elsif ($opt =~ m/^usb\d+$/) {
4878                 die "skip\n";
4879                 # since we cannot reliably hot unplug usb devices
4880                 # we are disabling it
4881                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4882                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4883                 die "skip\n" if !$d;
4884                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4885             } elsif ($opt eq 'vcpus') {
4886                 die "skip\n" if !$hotplug_features->{cpu};
4887                 qemu_cpu_hotplug($vmid, $conf, $value);
4888             } elsif ($opt eq 'balloon') {
4889                 # enable/disable balloning device is not hotpluggable
4890                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4891                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4892                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4893
4894                 # allow manual ballooning if shares is set to zero
4895                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4896                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4897                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4898                 }
4899             } elsif ($opt =~ m/^net(\d+)$/) {
4900                 # some changes can be done without hotplug
4901                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4902                                     $vmid, $opt, $value, $arch, $machine_type);
4903             } elsif (is_valid_drivename($opt)) {
4904                 die "skip\n" if $opt eq 'efidisk0';
4905                 # some changes can be done without hotplug
4906                 my $drive = parse_drive($opt, $value);
4907                 if (drive_is_cloudinit($drive)) {
4908                     &$apply_pending_cloudinit($opt, $value);
4909                 }
4910                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4911                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4912             } elsif ($opt =~ m/^memory$/) { #dimms
4913                 die "skip\n" if !$hotplug_features->{memory};
4914                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4915             } elsif ($opt eq 'cpuunits') {
4916                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4917             } elsif ($opt eq 'cpulimit') {
4918                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4919                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4920             } else {
4921                 die "skip\n";  # skip non-hot-pluggable options
4922             }
4923         };
4924         if (my $err = $@) {
4925             &$add_error($opt, $err) if $err ne "skip\n";
4926         } else {
4927             # save new config if hotplug was successful
4928             $conf->{$opt} = $value;
4929             delete $conf->{pending}->{$opt};
4930             PVE::QemuConfig->write_config($vmid, $conf);
4931             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4932         }
4933     }
4934 }
4935
4936 sub try_deallocate_drive {
4937     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4938
4939     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4940         my $volid = $drive->{file};
4941         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4942             my $sid = PVE::Storage::parse_volume_id($volid);
4943             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4944
4945             # check if the disk is really unused
4946             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4947                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4948             PVE::Storage::vdisk_free($storecfg, $volid);
4949             return 1;
4950         } else {
4951             # If vm is not owner of this disk remove from config
4952             return 1;
4953         }
4954     }
4955
4956     return undef;
4957 }
4958
4959 sub vmconfig_delete_or_detach_drive {
4960     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4961
4962     my $drive = parse_drive($opt, $conf->{$opt});
4963
4964     my $rpcenv = PVE::RPCEnvironment::get();
4965     my $authuser = $rpcenv->get_user();
4966
4967     if ($force) {
4968         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4969         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4970     } else {
4971         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4972     }
4973 }
4974
4975
4976
4977 sub vmconfig_apply_pending {
4978     my ($vmid, $conf, $storecfg) = @_;
4979
4980     # cold plug
4981
4982     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4983     foreach my $opt (sort keys %$pending_delete_hash) {
4984         die "internal error" if $opt =~ m/^unused/;
4985         my $force = $pending_delete_hash->{$opt}->{force};
4986         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4987         if (!defined($conf->{$opt})) {
4988             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4989             PVE::QemuConfig->write_config($vmid, $conf);
4990         } elsif (is_valid_drivename($opt)) {
4991             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4992             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4993             delete $conf->{$opt};
4994             PVE::QemuConfig->write_config($vmid, $conf);
4995         } else {
4996             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4997             delete $conf->{$opt};
4998             PVE::QemuConfig->write_config($vmid, $conf);
4999         }
5000     }
5001
5002     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5003
5004     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5005         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5006
5007         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5008             # skip if nothing changed
5009         } elsif (is_valid_drivename($opt)) {
5010             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5011                 if defined($conf->{$opt});
5012             $conf->{$opt} = $conf->{pending}->{$opt};
5013         } else {
5014             $conf->{$opt} = $conf->{pending}->{$opt};
5015         }
5016
5017         delete $conf->{pending}->{$opt};
5018         PVE::QemuConfig->write_config($vmid, $conf);
5019     }
5020 }
5021
5022 my $safe_num_ne = sub {
5023     my ($a, $b) = @_;
5024
5025     return 0 if !defined($a) && !defined($b);
5026     return 1 if !defined($a);
5027     return 1 if !defined($b);
5028
5029     return $a != $b;
5030 };
5031
5032 my $safe_string_ne = sub {
5033     my ($a, $b) = @_;
5034
5035     return 0 if !defined($a) && !defined($b);
5036     return 1 if !defined($a);
5037     return 1 if !defined($b);
5038
5039     return $a ne $b;
5040 };
5041
5042 sub vmconfig_update_net {
5043     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5044
5045     my $newnet = parse_net($value);
5046
5047     if ($conf->{$opt}) {
5048         my $oldnet = parse_net($conf->{$opt});
5049
5050         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5051             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5052             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5053             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5054
5055             # for non online change, we try to hot-unplug
5056             die "skip\n" if !$hotplug;
5057             vm_deviceunplug($vmid, $conf, $opt);
5058         } else {
5059
5060             die "internal error" if $opt !~ m/net(\d+)/;
5061             my $iface = "tap${vmid}i$1";
5062
5063             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5064                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5065                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5066                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5067                 PVE::Network::tap_unplug($iface);
5068                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5069             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5070                 # Rate can be applied on its own but any change above needs to
5071                 # include the rate in tap_plug since OVS resets everything.
5072                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5073             }
5074
5075             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5076                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5077             }
5078
5079             return 1;
5080         }
5081     }
5082
5083     if ($hotplug) {
5084         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5085     } else {
5086         die "skip\n";
5087     }
5088 }
5089
5090 sub vmconfig_update_disk {
5091     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5092
5093     # fixme: do we need force?
5094
5095     my $drive = parse_drive($opt, $value);
5096
5097     if ($conf->{$opt}) {
5098
5099         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5100
5101             my $media = $drive->{media} || 'disk';
5102             my $oldmedia = $old_drive->{media} || 'disk';
5103             die "unable to change media type\n" if $media ne $oldmedia;
5104
5105             if (!drive_is_cdrom($old_drive)) {
5106
5107                 if ($drive->{file} ne $old_drive->{file}) {
5108
5109                     die "skip\n" if !$hotplug;
5110
5111                     # unplug and register as unused
5112                     vm_deviceunplug($vmid, $conf, $opt);
5113                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5114
5115                 } else {
5116                     # update existing disk
5117
5118                     # skip non hotpluggable value
5119                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5120                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5121                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5122                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5123                         die "skip\n";
5124                     }
5125
5126                     # apply throttle
5127                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5128                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5129                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5130                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5131                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5132                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5133                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5134                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5135                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5136                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5137                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5138                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5139                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5140                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5141                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5142                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5143                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5144                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5145
5146                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5147                                                    ($drive->{mbps} || 0)*1024*1024,
5148                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5149                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5150                                                    $drive->{iops} || 0,
5151                                                    $drive->{iops_rd} || 0,
5152                                                    $drive->{iops_wr} || 0,
5153                                                    ($drive->{mbps_max} || 0)*1024*1024,
5154                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5155                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5156                                                    $drive->{iops_max} || 0,
5157                                                    $drive->{iops_rd_max} || 0,
5158                                                    $drive->{iops_wr_max} || 0,
5159                                                    $drive->{bps_max_length} || 1,
5160                                                    $drive->{bps_rd_max_length} || 1,
5161                                                    $drive->{bps_wr_max_length} || 1,
5162                                                    $drive->{iops_max_length} || 1,
5163                                                    $drive->{iops_rd_max_length} || 1,
5164                                                    $drive->{iops_wr_max_length} || 1);
5165
5166                     }
5167
5168                     return 1;
5169                 }
5170
5171             } else { # cdrom
5172
5173                 if ($drive->{file} eq 'none') {
5174                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5175                     if (drive_is_cloudinit($old_drive)) {
5176                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5177                     }
5178                 } else {
5179                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5180                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5181                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5182                 }
5183
5184                 return 1;
5185             }
5186         }
5187     }
5188
5189     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5190     # hotplug new disks
5191     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5192     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5193 }
5194
5195 sub vm_start {
5196     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5197         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5198
5199     PVE::QemuConfig->lock_config($vmid, sub {
5200         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5201
5202         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5203
5204         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5205
5206         PVE::QemuConfig->check_lock($conf)
5207             if !($skiplock || $is_suspended);
5208
5209         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5210
5211         # clean up leftover reboot request files
5212         eval { clear_reboot_request($vmid); };
5213         warn $@ if $@;
5214
5215         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5216             vmconfig_apply_pending($vmid, $conf, $storecfg);
5217             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5218         }
5219
5220         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5221
5222         my $defaults = load_defaults();
5223
5224         # set environment variable useful inside network script
5225         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5226
5227         my $local_volumes = {};
5228
5229         if ($targetstorage) {
5230             foreach_drive($conf, sub {
5231                 my ($ds, $drive) = @_;
5232
5233                 return if drive_is_cdrom($drive);
5234
5235                 my $volid = $drive->{file};
5236
5237                 return if !$volid;
5238
5239                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5240
5241                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5242                 return if $scfg->{shared};
5243                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5244             });
5245
5246             my $format = undef;
5247
5248             foreach my $opt (sort keys %$local_volumes) {
5249
5250                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5251                 my $drive = parse_drive($opt, $conf->{$opt});
5252
5253                 #if remote storage is specified, use default format
5254                 if ($targetstorage && $targetstorage ne "1") {
5255                     $storeid = $targetstorage;
5256                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5257                     $format = $defFormat;
5258                 } else {
5259                     #else we use same format than original
5260                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5261                     $format = qemu_img_format($scfg, $volid);
5262                 }
5263
5264                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5265                 my $newdrive = $drive;
5266                 $newdrive->{format} = $format;
5267                 $newdrive->{file} = $newvolid;
5268                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5269                 $local_volumes->{$opt} = $drivestr;
5270                 #pass drive to conf for command line
5271                 $conf->{$opt} = $drivestr;
5272             }
5273         }
5274
5275         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5276
5277         if ($is_suspended) {
5278             # enforce machine type on suspended vm to ensure HW compatibility
5279             $forcemachine = $conf->{runningmachine};
5280             print "Resuming suspended VM\n";
5281         }
5282
5283         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5284
5285         my $migration_ip;
5286         my $get_migration_ip = sub {
5287             my ($cidr, $nodename) = @_;
5288
5289             return $migration_ip if defined($migration_ip);
5290
5291             if (!defined($cidr)) {
5292                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5293                 $cidr = $dc_conf->{migration}->{network};
5294             }
5295
5296             if (defined($cidr)) {
5297                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5298
5299                 die "could not get IP: no address configured on local " .
5300                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5301
5302                 die "could not get IP: multiple addresses configured on local " .
5303                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5304
5305                 $migration_ip = @$ips[0];
5306             }
5307
5308             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5309                 if !defined($migration_ip);
5310
5311             return $migration_ip;
5312         };
5313
5314         my $migrate_uri;
5315         if ($statefile) {
5316             if ($statefile eq 'tcp') {
5317                 my $localip = "localhost";
5318                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5319                 my $nodename = PVE::INotify::nodename();
5320
5321                 if (!defined($migration_type)) {
5322                     if (defined($datacenterconf->{migration}->{type})) {
5323                         $migration_type = $datacenterconf->{migration}->{type};
5324                     } else {
5325                         $migration_type = 'secure';
5326                     }
5327                 }
5328
5329                 if ($migration_type eq 'insecure') {
5330                     $localip = $get_migration_ip->($migration_network, $nodename);
5331                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5332                 }
5333
5334                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5335                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5336                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5337                 push @$cmd, '-incoming', $migrate_uri;
5338                 push @$cmd, '-S';
5339
5340             } elsif ($statefile eq 'unix') {
5341                 # should be default for secure migrations as a ssh TCP forward
5342                 # tunnel is not deterministic reliable ready and fails regurarly
5343                 # to set up in time, so use UNIX socket forwards
5344                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5345                 unlink $socket_addr;
5346
5347                 $migrate_uri = "unix:$socket_addr";
5348
5349                 push @$cmd, '-incoming', $migrate_uri;
5350                 push @$cmd, '-S';
5351
5352             } elsif (-e $statefile) {
5353                 push @$cmd, '-loadstate', $statefile;
5354             } else {
5355                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5356                 push @$vollist, $statefile;
5357                 push @$cmd, '-loadstate', $statepath;
5358             }
5359         } elsif ($paused) {
5360             push @$cmd, '-S';
5361         }
5362
5363         # host pci devices
5364         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5365           my $d = parse_hostpci($conf->{"hostpci$i"});
5366           next if !$d;
5367           my $pcidevices = $d->{pciid};
5368           foreach my $pcidevice (@$pcidevices) {
5369                 my $pciid = $pcidevice->{id};
5370
5371                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5372                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5373                 die "no pci device info for device '$pciid'\n" if !$info;
5374
5375                 if ($d->{mdev}) {
5376                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5377                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5378                 } else {
5379                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5380                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5381                     die "can't reset pci device '$pciid'\n"
5382                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5383                 }
5384           }
5385         }
5386
5387         PVE::Storage::activate_volumes($storecfg, $vollist);
5388
5389         eval {
5390             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5391                 outfunc => sub {}, errfunc => sub {});
5392         };
5393         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5394         # timeout should be more than enough here...
5395         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5396
5397         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5398                                                   : $defaults->{cpuunits};
5399
5400         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5401         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5402
5403         my %properties = (
5404             Slice => 'qemu.slice',
5405             KillMode => 'none',
5406             CPUShares => $cpuunits
5407         );
5408
5409         if (my $cpulimit = $conf->{cpulimit}) {
5410             $properties{CPUQuota} = int($cpulimit * 100);
5411         }
5412         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5413
5414         my $run_qemu = sub {
5415             PVE::Tools::run_fork sub {
5416                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5417                 run_command($cmd, %run_params);
5418             };
5419         };
5420
5421         if ($conf->{hugepages}) {
5422
5423             my $code = sub {
5424                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5425                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5426
5427                 PVE::QemuServer::Memory::hugepages_mount();
5428                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5429
5430                 eval { $run_qemu->() };
5431                 if (my $err = $@) {
5432                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5433                     die $err;
5434                 }
5435
5436                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5437             };
5438             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5439
5440         } else {
5441             eval { $run_qemu->() };
5442         }
5443
5444         if (my $err = $@) {
5445             # deactivate volumes if start fails
5446             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5447             die "start failed: $err";
5448         }
5449
5450         print "migration listens on $migrate_uri\n" if $migrate_uri;
5451
5452         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5453             eval { mon_cmd($vmid, "cont"); };
5454             warn $@ if $@;
5455         }
5456
5457         #start nbd server for storage migration
5458         if ($targetstorage) {
5459             my $nodename = PVE::INotify::nodename();
5460             my $localip = $get_migration_ip->($migration_network, $nodename);
5461             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5462             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5463
5464             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5465
5466             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5467
5468             foreach my $opt (sort keys %$local_volumes) {
5469                 my $volid = $local_volumes->{$opt};
5470                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5471                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5472                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5473             }
5474         }
5475
5476         if ($migratedfrom) {
5477             eval {
5478                 set_migration_caps($vmid);
5479             };
5480             warn $@ if $@;
5481
5482             if ($spice_port) {
5483                 print "spice listens on port $spice_port\n";
5484                 if ($spice_ticket) {
5485                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5486                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5487                 }
5488             }
5489
5490         } else {
5491             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5492                 if !$statefile && $conf->{balloon};
5493
5494             foreach my $opt (keys %$conf) {
5495                 next if $opt !~  m/^net\d+$/;
5496                 my $nicconf = parse_net($conf->{$opt});
5497                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5498             }
5499         }
5500
5501         mon_cmd($vmid, 'qom-set',
5502                     path => "machine/peripheral/balloon0",
5503                     property => "guest-stats-polling-interval",
5504                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5505
5506         if ($is_suspended) {
5507             print "Resumed VM, removing state\n";
5508             if (my $vmstate = $conf->{vmstate}) {
5509                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5510                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5511             }
5512             delete $conf->@{qw(lock vmstate runningmachine)};
5513             PVE::QemuConfig->write_config($vmid, $conf);
5514         }
5515
5516         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5517     });
5518 }
5519
5520 sub vm_commandline {
5521     my ($storecfg, $vmid, $snapname) = @_;
5522
5523     my $conf = PVE::QemuConfig->load_config($vmid);
5524     my $forcemachine;
5525
5526     if ($snapname) {
5527         my $snapshot = $conf->{snapshots}->{$snapname};
5528         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5529
5530         # check for a 'runningmachine' in snapshot
5531         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5532
5533         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5534
5535         $conf = $snapshot;
5536     }
5537
5538     my $defaults = load_defaults();
5539
5540     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5541
5542     return PVE::Tools::cmd2string($cmd);
5543 }
5544
5545 sub vm_reset {
5546     my ($vmid, $skiplock) = @_;
5547
5548     PVE::QemuConfig->lock_config($vmid, sub {
5549
5550         my $conf = PVE::QemuConfig->load_config($vmid);
5551
5552         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5553
5554         mon_cmd($vmid, "system_reset");
5555     });
5556 }
5557
5558 sub get_vm_volumes {
5559     my ($conf) = @_;
5560
5561     my $vollist = [];
5562     foreach_volid($conf, sub {
5563         my ($volid, $attr) = @_;
5564
5565         return if $volid =~ m|^/|;
5566
5567         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5568         return if !$sid;
5569
5570         push @$vollist, $volid;
5571     });
5572
5573     return $vollist;
5574 }
5575
5576 sub vm_stop_cleanup {
5577     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5578
5579     eval {
5580
5581         if (!$keepActive) {
5582             my $vollist = get_vm_volumes($conf);
5583             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5584         }
5585
5586         foreach my $ext (qw(mon qmp pid vnc qga)) {
5587             unlink "/var/run/qemu-server/${vmid}.$ext";
5588         }
5589
5590         if ($conf->{ivshmem}) {
5591             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5592             # just delete it for now, VMs which have this already open do not
5593             # are affected, but new VMs will get a separated one. If this
5594             # becomes an issue we either add some sort of ref-counting or just
5595             # add a "don't delete on stop" flag to the ivshmem format.
5596             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5597         }
5598
5599         foreach my $key (keys %$conf) {
5600             next if $key !~ m/^hostpci(\d+)$/;
5601             my $hostpciindex = $1;
5602             my $d = parse_hostpci($conf->{$key});
5603             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5604
5605             foreach my $pci (@{$d->{pciid}}) {
5606                 my $pciid = $pci->{id};
5607                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5608             }
5609         }
5610
5611         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5612     };
5613     warn $@ if $@; # avoid errors - just warn
5614 }
5615
5616 # call only in locked context
5617 sub _do_vm_stop {
5618     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5619
5620     my $pid = check_running($vmid, $nocheck);
5621     return if !$pid;
5622
5623     my $conf;
5624     if (!$nocheck) {
5625         $conf = PVE::QemuConfig->load_config($vmid);
5626         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5627         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5628             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5629             $timeout = $opts->{down} if $opts->{down};
5630         }
5631         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5632     }
5633
5634     eval {
5635         if ($shutdown) {
5636             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5637                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5638             } else {
5639                 mon_cmd($vmid, "system_powerdown");
5640             }
5641         } else {
5642             mon_cmd($vmid, "quit");
5643         }
5644     };
5645     my $err = $@;
5646
5647     if (!$err) {
5648         $timeout = 60 if !defined($timeout);
5649
5650         my $count = 0;
5651         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5652             $count++;
5653             sleep 1;
5654         }
5655
5656         if ($count >= $timeout) {
5657             if ($force) {
5658                 warn "VM still running - terminating now with SIGTERM\n";
5659                 kill 15, $pid;
5660             } else {
5661                 die "VM quit/powerdown failed - got timeout\n";
5662             }
5663         } else {
5664             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5665             return;
5666         }
5667     } else {
5668         if ($force) {
5669             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5670             kill 15, $pid;
5671         } else {
5672             die "VM quit/powerdown failed\n";
5673         }
5674     }
5675
5676     # wait again
5677     $timeout = 10;
5678
5679     my $count = 0;
5680     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5681         $count++;
5682         sleep 1;
5683     }
5684
5685     if ($count >= $timeout) {
5686         warn "VM still running - terminating now with SIGKILL\n";
5687         kill 9, $pid;
5688         sleep 1;
5689     }
5690
5691     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5692 }
5693
5694 # Note: use $nocheck to skip tests if VM configuration file exists.
5695 # We need that when migration VMs to other nodes (files already moved)
5696 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5697 sub vm_stop {
5698     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5699
5700     $force = 1 if !defined($force) && !$shutdown;
5701
5702     if ($migratedfrom){
5703         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5704         kill 15, $pid if $pid;
5705         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5706         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5707         return;
5708     }
5709
5710     PVE::QemuConfig->lock_config($vmid, sub {
5711         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5712    });
5713 }
5714
5715 sub vm_reboot {
5716     my ($vmid, $timeout) = @_;
5717
5718     PVE::QemuConfig->lock_config($vmid, sub {
5719         eval {
5720
5721             # only reboot if running, as qmeventd starts it again on a stop event
5722             return if !check_running($vmid);
5723
5724             create_reboot_request($vmid);
5725
5726             my $storecfg = PVE::Storage::config();
5727             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5728
5729         };
5730         if (my $err = $@) {
5731             # avoid that the next normal shutdown will be confused for a reboot
5732             clear_reboot_request($vmid);
5733             die $err;
5734         }
5735    });
5736 }
5737
5738 sub vm_suspend {
5739     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5740
5741     my $conf;
5742     my $path;
5743     my $storecfg;
5744     my $vmstate;
5745
5746     PVE::QemuConfig->lock_config($vmid, sub {
5747
5748         $conf = PVE::QemuConfig->load_config($vmid);
5749
5750         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5751         PVE::QemuConfig->check_lock($conf)
5752             if !($skiplock || $is_backing_up);
5753
5754         die "cannot suspend to disk during backup\n"
5755             if $is_backing_up && $includestate;
5756
5757         if ($includestate) {
5758             $conf->{lock} = 'suspending';
5759             my $date = strftime("%Y-%m-%d", localtime(time()));
5760             $storecfg = PVE::Storage::config();
5761             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5762             $path = PVE::Storage::path($storecfg, $vmstate);
5763             PVE::QemuConfig->write_config($vmid, $conf);
5764         } else {
5765             mon_cmd($vmid, "stop");
5766         }
5767     });
5768
5769     if ($includestate) {
5770         # save vm state
5771         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5772
5773         eval {
5774             mon_cmd($vmid, "savevm-start", statefile => $path);
5775             for(;;) {
5776                 my $state = mon_cmd($vmid, "query-savevm");
5777                 if (!$state->{status}) {
5778                     die "savevm not active\n";
5779                 } elsif ($state->{status} eq 'active') {
5780                     sleep(1);
5781                     next;
5782                 } elsif ($state->{status} eq 'completed') {
5783                     print "State saved, quitting\n";
5784                     last;
5785                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5786                     die "query-savevm failed with error '$state->{error}'\n"
5787                 } else {
5788                     die "query-savevm returned status '$state->{status}'\n";
5789                 }
5790             }
5791         };
5792         my $err = $@;
5793
5794         PVE::QemuConfig->lock_config($vmid, sub {
5795             $conf = PVE::QemuConfig->load_config($vmid);
5796             if ($err) {
5797                 # cleanup, but leave suspending lock, to indicate something went wrong
5798                 eval {
5799                     mon_cmd($vmid, "savevm-end");
5800                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5801                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5802                     delete $conf->@{qw(vmstate runningmachine)};
5803                     PVE::QemuConfig->write_config($vmid, $conf);
5804                 };
5805                 warn $@ if $@;
5806                 die $err;
5807             }
5808
5809             die "lock changed unexpectedly\n"
5810                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5811
5812             mon_cmd($vmid, "quit");
5813             $conf->{lock} = 'suspended';
5814             PVE::QemuConfig->write_config($vmid, $conf);
5815         });
5816     }
5817 }
5818
5819 sub vm_resume {
5820     my ($vmid, $skiplock, $nocheck) = @_;
5821
5822     PVE::QemuConfig->lock_config($vmid, sub {
5823         my $res = mon_cmd($vmid, 'query-status');
5824         my $resume_cmd = 'cont';
5825
5826         if ($res->{status} && $res->{status} eq 'suspended') {
5827             $resume_cmd = 'system_wakeup';
5828         }
5829
5830         if (!$nocheck) {
5831
5832             my $conf = PVE::QemuConfig->load_config($vmid);
5833
5834             PVE::QemuConfig->check_lock($conf)
5835                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5836         }
5837
5838         mon_cmd($vmid, $resume_cmd);
5839     });
5840 }
5841
5842 sub vm_sendkey {
5843     my ($vmid, $skiplock, $key) = @_;
5844
5845     PVE::QemuConfig->lock_config($vmid, sub {
5846
5847         my $conf = PVE::QemuConfig->load_config($vmid);
5848
5849         # there is no qmp command, so we use the human monitor command
5850         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5851         die $res if $res ne '';
5852     });
5853 }
5854
5855 # vzdump restore implementaion
5856
5857 sub tar_archive_read_firstfile {
5858     my $archive = shift;
5859
5860     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5861
5862     # try to detect archive type first
5863     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5864         die "unable to open file '$archive'\n";
5865     my $firstfile = <$fh>;
5866     kill 15, $pid;
5867     close $fh;
5868
5869     die "ERROR: archive contaions no data\n" if !$firstfile;
5870     chomp $firstfile;
5871
5872     return $firstfile;
5873 }
5874
5875 sub tar_restore_cleanup {
5876     my ($storecfg, $statfile) = @_;
5877
5878     print STDERR "starting cleanup\n";
5879
5880     if (my $fd = IO::File->new($statfile, "r")) {
5881         while (defined(my $line = <$fd>)) {
5882             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5883                 my $volid = $2;
5884                 eval {
5885                     if ($volid =~ m|^/|) {
5886                         unlink $volid || die 'unlink failed\n';
5887                     } else {
5888                         PVE::Storage::vdisk_free($storecfg, $volid);
5889                     }
5890                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5891                 };
5892                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5893             } else {
5894                 print STDERR "unable to parse line in statfile - $line";
5895             }
5896         }
5897         $fd->close();
5898     }
5899 }
5900
5901 sub restore_archive {
5902     my ($archive, $vmid, $user, $opts) = @_;
5903
5904     my $format = $opts->{format};
5905     my $comp;
5906
5907     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5908         $format = 'tar' if !$format;
5909         $comp = 'gzip';
5910     } elsif ($archive =~ m/\.tar$/) {
5911         $format = 'tar' if !$format;
5912     } elsif ($archive =~ m/.tar.lzo$/) {
5913         $format = 'tar' if !$format;
5914         $comp = 'lzop';
5915     } elsif ($archive =~ m/\.vma$/) {
5916         $format = 'vma' if !$format;
5917     } elsif ($archive =~ m/\.vma\.gz$/) {
5918         $format = 'vma' if !$format;
5919         $comp = 'gzip';
5920     } elsif ($archive =~ m/\.vma\.lzo$/) {
5921         $format = 'vma' if !$format;
5922         $comp = 'lzop';
5923     } else {
5924         $format = 'vma' if !$format; # default
5925     }
5926
5927     # try to detect archive format
5928     if ($format eq 'tar') {
5929         return restore_tar_archive($archive, $vmid, $user, $opts);
5930     } else {
5931         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5932     }
5933 }
5934
5935 sub restore_update_config_line {
5936     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5937
5938     return if $line =~ m/^\#qmdump\#/;
5939     return if $line =~ m/^\#vzdump\#/;
5940     return if $line =~ m/^lock:/;
5941     return if $line =~ m/^unused\d+:/;
5942     return if $line =~ m/^parent:/;
5943
5944     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5945     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5946         # try to convert old 1.X settings
5947         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5948         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5949             my ($model, $macaddr) = split(/\=/, $devconfig);
5950             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5951             my $net = {
5952                 model => $model,
5953                 bridge => "vmbr$ind",
5954                 macaddr => $macaddr,
5955             };
5956             my $netstr = print_net($net);
5957
5958             print $outfd "net$cookie->{netcount}: $netstr\n";
5959             $cookie->{netcount}++;
5960         }
5961     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5962         my ($id, $netstr) = ($1, $2);
5963         my $net = parse_net($netstr);
5964         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5965         $netstr = print_net($net);
5966         print $outfd "$id: $netstr\n";
5967     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5968         my $virtdev = $1;
5969         my $value = $3;
5970         my $di = parse_drive($virtdev, $value);
5971         if (defined($di->{backup}) && !$di->{backup}) {
5972             print $outfd "#$line";
5973         } elsif ($map->{$virtdev}) {
5974             delete $di->{format}; # format can change on restore
5975             $di->{file} = $map->{$virtdev};
5976             $value = print_drive($vmid, $di);
5977             print $outfd "$virtdev: $value\n";
5978         } else {
5979             print $outfd $line;
5980         }
5981     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5982         my $vmgenid = $1;
5983         if ($vmgenid ne '0') {
5984             # always generate a new vmgenid if there was a valid one setup
5985             $vmgenid = generate_uuid();
5986         }
5987         print $outfd "vmgenid: $vmgenid\n";
5988     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5989         my ($uuid, $uuid_str);
5990         UUID::generate($uuid);
5991         UUID::unparse($uuid, $uuid_str);
5992         my $smbios1 = parse_smbios1($2);
5993         $smbios1->{uuid} = $uuid_str;
5994         print $outfd $1.print_smbios1($smbios1)."\n";
5995     } else {
5996         print $outfd $line;
5997     }
5998 }
5999
6000 sub scan_volids {
6001     my ($cfg, $vmid) = @_;
6002
6003     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6004
6005     my $volid_hash = {};
6006     foreach my $storeid (keys %$info) {
6007         foreach my $item (@{$info->{$storeid}}) {
6008             next if !($item->{volid} && $item->{size});
6009             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6010             $volid_hash->{$item->{volid}} = $item;
6011         }
6012     }
6013
6014     return $volid_hash;
6015 }
6016
6017 sub is_volume_in_use {
6018     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6019
6020     my $path = PVE::Storage::path($storecfg, $volid);
6021
6022     my $scan_config = sub {
6023         my ($cref, $snapname) = @_;
6024
6025         foreach my $key (keys %$cref) {
6026             my $value = $cref->{$key};
6027             if (is_valid_drivename($key)) {
6028                 next if $skip_drive && $key eq $skip_drive;
6029                 my $drive = parse_drive($key, $value);
6030                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6031                 return 1 if $volid eq $drive->{file};
6032                 if ($drive->{file} =~ m!^/!) {
6033                     return 1 if $drive->{file} eq $path;
6034                 } else {
6035                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6036                     next if !$storeid;
6037                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6038                     next if !$scfg;
6039                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6040                 }
6041             }
6042         }
6043
6044         return 0;
6045     };
6046
6047     return 1 if &$scan_config($conf);
6048
6049     undef $skip_drive;
6050
6051     foreach my $snapname (keys %{$conf->{snapshots}}) {
6052         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6053     }
6054
6055     return 0;
6056 }
6057
6058 sub update_disksize {
6059     my ($vmid, $conf, $volid_hash) = @_;
6060
6061     my $changes;
6062     my $prefix = "VM $vmid:";
6063
6064     # used and unused disks
6065     my $referenced = {};
6066
6067     # Note: it is allowed to define multiple storages with same path (alias), so
6068     # we need to check both 'volid' and real 'path' (two different volid can point
6069     # to the same path).
6070
6071     my $referencedpath = {};
6072
6073     # update size info
6074     foreach my $opt (keys %$conf) {
6075         if (is_valid_drivename($opt)) {
6076             my $drive = parse_drive($opt, $conf->{$opt});
6077             my $volid = $drive->{file};
6078             next if !$volid;
6079
6080             $referenced->{$volid} = 1;
6081             if ($volid_hash->{$volid} &&
6082                 (my $path = $volid_hash->{$volid}->{path})) {
6083                 $referencedpath->{$path} = 1;
6084             }
6085
6086             next if drive_is_cdrom($drive);
6087             next if !$volid_hash->{$volid};
6088
6089             $drive->{size} = $volid_hash->{$volid}->{size};
6090             my $new = print_drive($vmid, $drive);
6091             if ($new ne $conf->{$opt}) {
6092                 $changes = 1;
6093                 $conf->{$opt} = $new;
6094                 print "$prefix update disk '$opt' information.\n";
6095             }
6096         }
6097     }
6098
6099     # remove 'unusedX' entry if volume is used
6100     foreach my $opt (keys %$conf) {
6101         next if $opt !~ m/^unused\d+$/;
6102         my $volid = $conf->{$opt};
6103         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6104         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6105             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6106             $changes = 1;
6107             delete $conf->{$opt};
6108         }
6109
6110         $referenced->{$volid} = 1;
6111         $referencedpath->{$path} = 1 if $path;
6112     }
6113
6114     foreach my $volid (sort keys %$volid_hash) {
6115         next if $volid =~ m/vm-$vmid-state-/;
6116         next if $referenced->{$volid};
6117         my $path = $volid_hash->{$volid}->{path};
6118         next if !$path; # just to be sure
6119         next if $referencedpath->{$path};
6120         $changes = 1;
6121         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6122         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6123         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6124     }
6125
6126     return $changes;
6127 }
6128
6129 sub rescan {
6130     my ($vmid, $nolock, $dryrun) = @_;
6131
6132     my $cfg = PVE::Storage::config();
6133
6134     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6135     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6136     foreach my $stor (keys %{$cfg->{ids}}) {
6137         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6138     }
6139
6140     print "rescan volumes...\n";
6141     my $volid_hash = scan_volids($cfg, $vmid);
6142
6143     my $updatefn =  sub {
6144         my ($vmid) = @_;
6145
6146         my $conf = PVE::QemuConfig->load_config($vmid);
6147
6148         PVE::QemuConfig->check_lock($conf);
6149
6150         my $vm_volids = {};
6151         foreach my $volid (keys %$volid_hash) {
6152             my $info = $volid_hash->{$volid};
6153             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6154         }
6155
6156         my $changes = update_disksize($vmid, $conf, $vm_volids);
6157
6158         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6159     };
6160
6161     if (defined($vmid)) {
6162         if ($nolock) {
6163             &$updatefn($vmid);
6164         } else {
6165             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6166         }
6167     } else {
6168         my $vmlist = config_list();
6169         foreach my $vmid (keys %$vmlist) {
6170             if ($nolock) {
6171                 &$updatefn($vmid);
6172             } else {
6173                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6174             }
6175         }
6176     }
6177 }
6178
6179 sub restore_vma_archive {
6180     my ($archive, $vmid, $user, $opts, $comp) = @_;
6181
6182     my $readfrom = $archive;
6183
6184     my $cfg = PVE::Storage::config();
6185     my $commands = [];
6186     my $bwlimit = $opts->{bwlimit};
6187
6188     my $dbg_cmdstring = '';
6189     my $add_pipe = sub {
6190         my ($cmd) = @_;
6191         push @$commands, $cmd;
6192         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6193         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6194         $readfrom = '-';
6195     };
6196
6197     my $input = undef;
6198     if ($archive eq '-') {
6199         $input = '<&STDIN';
6200     } else {
6201         # If we use a backup from a PVE defined storage we also consider that
6202         # storage's rate limit:
6203         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6204         if (defined($volid)) {
6205             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6206             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6207             if ($readlimit) {
6208                 print STDERR "applying read rate limit: $readlimit\n";
6209                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6210                 $add_pipe->($cstream);
6211             }
6212         }
6213     }
6214
6215     if ($comp) {
6216         my $cmd;
6217         if ($comp eq 'gzip') {
6218             $cmd = ['zcat', $readfrom];
6219         } elsif ($comp eq 'lzop') {
6220             $cmd = ['lzop', '-d', '-c', $readfrom];
6221         } else {
6222             die "unknown compression method '$comp'\n";
6223         }
6224         $add_pipe->($cmd);
6225     }
6226
6227     my $tmpdir = "/var/tmp/vzdumptmp$$";
6228     rmtree $tmpdir;
6229
6230     # disable interrupts (always do cleanups)
6231     local $SIG{INT} =
6232         local $SIG{TERM} =
6233         local $SIG{QUIT} =
6234         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6235
6236     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6237     POSIX::mkfifo($mapfifo, 0600);
6238     my $fifofh;
6239
6240     my $openfifo = sub {
6241         open($fifofh, '>', $mapfifo) || die $!;
6242     };
6243
6244     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6245
6246     my $oldtimeout;
6247     my $timeout = 5;
6248
6249     my $devinfo = {};
6250
6251     my $rpcenv = PVE::RPCEnvironment::get();
6252
6253     my $conffile = PVE::QemuConfig->config_file($vmid);
6254     my $tmpfn = "$conffile.$$.tmp";
6255
6256     # Note: $oldconf is undef if VM does not exists
6257     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6258     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6259
6260     my %storage_limits;
6261
6262     my $print_devmap = sub {
6263         my $virtdev_hash = {};
6264
6265         my $cfgfn = "$tmpdir/qemu-server.conf";
6266
6267         # we can read the config - that is already extracted
6268         my $fh = IO::File->new($cfgfn, "r") ||
6269             "unable to read qemu-server.conf - $!\n";
6270
6271         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6272         if (-f $fwcfgfn) {
6273             my $pve_firewall_dir = '/etc/pve/firewall';
6274             mkdir $pve_firewall_dir; # make sure the dir exists
6275             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6276         }
6277
6278         while (defined(my $line = <$fh>)) {
6279             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6280                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6281                 die "archive does not contain data for drive '$virtdev'\n"
6282                     if !$devinfo->{$devname};
6283                 if (defined($opts->{storage})) {
6284                     $storeid = $opts->{storage} || 'local';
6285                 } elsif (!$storeid) {
6286                     $storeid = 'local';
6287                 }
6288                 $format = 'raw' if !$format;
6289                 $devinfo->{$devname}->{devname} = $devname;
6290                 $devinfo->{$devname}->{virtdev} = $virtdev;
6291                 $devinfo->{$devname}->{format} = $format;
6292                 $devinfo->{$devname}->{storeid} = $storeid;
6293
6294                 # check permission on storage
6295                 my $pool = $opts->{pool}; # todo: do we need that?
6296                 if ($user ne 'root@pam') {
6297                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6298                 }
6299
6300                 $storage_limits{$storeid} = $bwlimit;
6301
6302                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6303             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6304                 my $virtdev = $1;
6305                 my $drive = parse_drive($virtdev, $2);
6306                 if (drive_is_cloudinit($drive)) {
6307                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6308                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6309                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6310
6311                     my $d = {
6312                         format => $format,
6313                         storeid => $opts->{storage} // $storeid,
6314                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6315                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6316                         name => "vm-$vmid-cloudinit",
6317                         is_cloudinit => 1,
6318                     };
6319                     $virtdev_hash->{$virtdev} = $d;
6320                 }
6321             }
6322         }
6323
6324         foreach my $key (keys %storage_limits) {
6325             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6326             next if !$limit;
6327             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6328             $storage_limits{$key} = $limit * 1024;
6329         }
6330
6331         foreach my $devname (keys %$devinfo) {
6332             die "found no device mapping information for device '$devname'\n"
6333                 if !$devinfo->{$devname}->{virtdev};
6334         }
6335
6336         # create empty/temp config
6337         if ($oldconf) {
6338             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6339             foreach_drive($oldconf, sub {
6340                 my ($ds, $drive) = @_;
6341
6342                 return if drive_is_cdrom($drive, 1);
6343
6344                 my $volid = $drive->{file};
6345                 return if !$volid || $volid =~ m|^/|;
6346
6347                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6348                 return if !$path || !$owner || ($owner != $vmid);
6349
6350                 # Note: only delete disk we want to restore
6351                 # other volumes will become unused
6352                 if ($virtdev_hash->{$ds}) {
6353                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6354                     if (my $err = $@) {
6355                         warn $err;
6356                     }
6357                 }
6358             });
6359
6360             # delete vmstate files, after the restore we have no snapshots anymore
6361             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6362                 my $snap = $oldconf->{snapshots}->{$snapname};
6363                 if ($snap->{vmstate}) {
6364                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6365                     if (my $err = $@) {
6366                         warn $err;
6367                     }
6368                 }
6369             }
6370         }
6371
6372         my $map = {};
6373         foreach my $virtdev (sort keys %$virtdev_hash) {
6374             my $d = $virtdev_hash->{$virtdev};
6375             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6376             my $storeid = $d->{storeid};
6377             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6378
6379             my $map_opts = '';
6380             if (my $limit = $storage_limits{$storeid}) {
6381                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6382             }
6383
6384             # test if requested format is supported
6385             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6386             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6387             $d->{format} = $defFormat if !$supported;
6388
6389             my $name;
6390             if ($d->{is_cloudinit}) {
6391                 $name = $d->{name};
6392                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6393             }
6394
6395             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6396             print STDERR "new volume ID is '$volid'\n";
6397             $d->{volid} = $volid;
6398
6399             PVE::Storage::activate_volumes($cfg, [$volid]);
6400
6401             my $write_zeros = 1;
6402             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6403                 $write_zeros = 0;
6404             }
6405
6406             if (!$d->{is_cloudinit}) {
6407                 my $path = PVE::Storage::path($cfg, $volid);
6408
6409                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6410
6411                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6412             }
6413             $map->{$virtdev} = $volid;
6414         }
6415
6416         $fh->seek(0, 0) || die "seek failed - $!\n";
6417
6418         my $outfd = new IO::File ($tmpfn, "w") ||
6419             die "unable to write config for VM $vmid\n";
6420
6421         my $cookie = { netcount => 0 };
6422         while (defined(my $line = <$fh>)) {
6423             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6424         }
6425
6426         $fh->close();
6427         $outfd->close();
6428     };
6429
6430     eval {
6431         # enable interrupts
6432         local $SIG{INT} =
6433             local $SIG{TERM} =
6434             local $SIG{QUIT} =
6435             local $SIG{HUP} =
6436             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6437         local $SIG{ALRM} = sub { die "got timeout\n"; };
6438
6439         $oldtimeout = alarm($timeout);
6440
6441         my $parser = sub {
6442             my $line = shift;
6443
6444             print "$line\n";
6445
6446             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6447                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6448                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6449             } elsif ($line =~ m/^CTIME: /) {
6450                 # we correctly received the vma config, so we can disable
6451                 # the timeout now for disk allocation (set to 10 minutes, so
6452                 # that we always timeout if something goes wrong)
6453                 alarm(600);
6454                 &$print_devmap();
6455                 print $fifofh "done\n";
6456                 my $tmp = $oldtimeout || 0;
6457                 $oldtimeout = undef;
6458                 alarm($tmp);
6459                 close($fifofh);
6460             }
6461         };
6462
6463         print "restore vma archive: $dbg_cmdstring\n";
6464         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6465     };
6466     my $err = $@;
6467
6468     alarm($oldtimeout) if $oldtimeout;
6469
6470     my $vollist = [];
6471     foreach my $devname (keys %$devinfo) {
6472         my $volid = $devinfo->{$devname}->{volid};
6473         push @$vollist, $volid if $volid;
6474     }
6475
6476     PVE::Storage::deactivate_volumes($cfg, $vollist);
6477
6478     unlink $mapfifo;
6479
6480     if ($err) {
6481         rmtree $tmpdir;
6482         unlink $tmpfn;
6483
6484         foreach my $devname (keys %$devinfo) {
6485             my $volid = $devinfo->{$devname}->{volid};
6486             next if !$volid;
6487             eval {
6488                 if ($volid =~ m|^/|) {
6489                     unlink $volid || die 'unlink failed\n';
6490                 } else {
6491                     PVE::Storage::vdisk_free($cfg, $volid);
6492                 }
6493                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6494             };
6495             print STDERR "unable to cleanup '$volid' - $@" if $@;
6496         }
6497         die $err;
6498     }
6499
6500     rmtree $tmpdir;
6501
6502     rename($tmpfn, $conffile) ||
6503         die "unable to commit configuration file '$conffile'\n";
6504
6505     PVE::Cluster::cfs_update(); # make sure we read new file
6506
6507     eval { rescan($vmid, 1); };
6508     warn $@ if $@;
6509 }
6510
6511 sub restore_tar_archive {
6512     my ($archive, $vmid, $user, $opts) = @_;
6513
6514     if ($archive ne '-') {
6515         my $firstfile = tar_archive_read_firstfile($archive);
6516         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6517             if $firstfile ne 'qemu-server.conf';
6518     }
6519
6520     my $storecfg = PVE::Storage::config();
6521
6522     # avoid zombie disks when restoring over an existing VM -> cleanup first
6523     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6524     # skiplock=1 because qmrestore has set the 'create' lock itself already
6525     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6526     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6527
6528     my $tocmd = "/usr/lib/qemu-server/qmextract";
6529
6530     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6531     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6532     $tocmd .= ' --prealloc' if $opts->{prealloc};
6533     $tocmd .= ' --info' if $opts->{info};
6534
6535     # tar option "xf" does not autodetect compression when read from STDIN,
6536     # so we pipe to zcat
6537     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6538         PVE::Tools::shellquote("--to-command=$tocmd");
6539
6540     my $tmpdir = "/var/tmp/vzdumptmp$$";
6541     mkpath $tmpdir;
6542
6543     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6544     local $ENV{VZDUMP_VMID} = $vmid;
6545     local $ENV{VZDUMP_USER} = $user;
6546
6547     my $conffile = PVE::QemuConfig->config_file($vmid);
6548     my $tmpfn = "$conffile.$$.tmp";
6549
6550     # disable interrupts (always do cleanups)
6551     local $SIG{INT} =
6552         local $SIG{TERM} =
6553         local $SIG{QUIT} =
6554         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6555
6556     eval {
6557         # enable interrupts
6558         local $SIG{INT} =
6559             local $SIG{TERM} =
6560             local $SIG{QUIT} =
6561             local $SIG{HUP} =
6562             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6563
6564         if ($archive eq '-') {
6565             print "extracting archive from STDIN\n";
6566             run_command($cmd, input => "<&STDIN");
6567         } else {
6568             print "extracting archive '$archive'\n";
6569             run_command($cmd);
6570         }
6571
6572         return if $opts->{info};
6573
6574         # read new mapping
6575         my $map = {};
6576         my $statfile = "$tmpdir/qmrestore.stat";
6577         if (my $fd = IO::File->new($statfile, "r")) {
6578             while (defined (my $line = <$fd>)) {
6579                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6580                     $map->{$1} = $2 if $1;
6581                 } else {
6582                     print STDERR "unable to parse line in statfile - $line\n";
6583                 }
6584             }
6585             $fd->close();
6586         }
6587
6588         my $confsrc = "$tmpdir/qemu-server.conf";
6589
6590         my $srcfd = new IO::File($confsrc, "r") ||
6591             die "unable to open file '$confsrc'\n";
6592
6593         my $outfd = new IO::File ($tmpfn, "w") ||
6594             die "unable to write config for VM $vmid\n";
6595
6596         my $cookie = { netcount => 0 };
6597         while (defined (my $line = <$srcfd>)) {
6598             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6599         }
6600
6601         $srcfd->close();
6602         $outfd->close();
6603     };
6604     if (my $err = $@) {
6605         unlink $tmpfn;
6606         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6607         die $err;
6608     }
6609
6610     rmtree $tmpdir;
6611
6612     rename $tmpfn, $conffile ||
6613         die "unable to commit configuration file '$conffile'\n";
6614
6615     PVE::Cluster::cfs_update(); # make sure we read new file
6616
6617     eval { rescan($vmid, 1); };
6618     warn $@ if $@;
6619 };
6620
6621 sub foreach_storage_used_by_vm {
6622     my ($conf, $func) = @_;
6623
6624     my $sidhash = {};
6625
6626     foreach_drive($conf, sub {
6627         my ($ds, $drive) = @_;
6628         return if drive_is_cdrom($drive);
6629
6630         my $volid = $drive->{file};
6631
6632         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6633         $sidhash->{$sid} = $sid if $sid;
6634     });
6635
6636     foreach my $sid (sort keys %$sidhash) {
6637         &$func($sid);
6638     }
6639 }
6640
6641 my $qemu_snap_storage = {
6642     rbd => 1,
6643 };
6644 sub do_snapshots_with_qemu {
6645     my ($storecfg, $volid) = @_;
6646
6647     my $storage_name = PVE::Storage::parse_volume_id($volid);
6648     my $scfg = $storecfg->{ids}->{$storage_name};
6649
6650     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6651         return 1;
6652     }
6653
6654     if ($volid =~ m/\.(qcow2|qed)$/){
6655         return 1;
6656     }
6657
6658     return undef;
6659 }
6660
6661 sub qga_check_running {
6662     my ($vmid, $nowarn) = @_;
6663
6664     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6665     if ($@) {
6666         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6667         return 0;
6668     }
6669     return 1;
6670 }
6671
6672 sub template_create {
6673     my ($vmid, $conf, $disk) = @_;
6674
6675     my $storecfg = PVE::Storage::config();
6676
6677     foreach_drive($conf, sub {
6678         my ($ds, $drive) = @_;
6679
6680         return if drive_is_cdrom($drive);
6681         return if $disk && $ds ne $disk;
6682
6683         my $volid = $drive->{file};
6684         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6685
6686         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6687         $drive->{file} = $voliddst;
6688         $conf->{$ds} = print_drive($vmid, $drive);
6689         PVE::QemuConfig->write_config($vmid, $conf);
6690     });
6691 }
6692
6693 sub convert_iscsi_path {
6694     my ($path) = @_;
6695
6696     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6697         my $portal = $1;
6698         my $target = $2;
6699         my $lun = $3;
6700
6701         my $initiator_name = get_initiator_name();
6702
6703         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6704                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6705     }
6706
6707     die "cannot convert iscsi path '$path', unkown format\n";
6708 }
6709
6710 sub qemu_img_convert {
6711     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6712
6713     my $storecfg = PVE::Storage::config();
6714     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6715     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6716
6717     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6718
6719     my $cachemode;
6720     my $src_path;
6721     my $src_is_iscsi = 0;
6722     my $src_format;
6723
6724     if ($src_storeid) {
6725         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6726         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6727         $src_format = qemu_img_format($src_scfg, $src_volname);
6728         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6729         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6730         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6731     } elsif (-f $src_volid) {
6732         $src_path = $src_volid;
6733         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6734             $src_format = $1;
6735         }
6736     }
6737
6738     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6739
6740     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6741     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6742     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6743     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6744
6745     my $cmd = [];
6746     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6747     push @$cmd, '-l', "snapshot.name=$snapname"
6748         if $snapname && $src_format && $src_format eq "qcow2";
6749     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6750     push @$cmd, '-T', $cachemode if defined($cachemode);
6751
6752     if ($src_is_iscsi) {
6753         push @$cmd, '--image-opts';
6754         $src_path = convert_iscsi_path($src_path);
6755     } elsif ($src_format) {
6756         push @$cmd, '-f', $src_format;
6757     }
6758
6759     if ($dst_is_iscsi) {
6760         push @$cmd, '--target-image-opts';
6761         $dst_path = convert_iscsi_path($dst_path);
6762     } else {
6763         push @$cmd, '-O', $dst_format;
6764     }
6765
6766     push @$cmd, $src_path;
6767
6768     if (!$dst_is_iscsi && $is_zero_initialized) {
6769         push @$cmd, "zeroinit:$dst_path";
6770     } else {
6771         push @$cmd, $dst_path;
6772     }
6773
6774     my $parser = sub {
6775         my $line = shift;
6776         if($line =~ m/\((\S+)\/100\%\)/){
6777             my $percent = $1;
6778             my $transferred = int($size * $percent / 100);
6779             my $remaining = $size - $transferred;
6780
6781             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6782         }
6783
6784     };
6785
6786     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6787     my $err = $@;
6788     die "copy failed: $err" if $err;
6789 }
6790
6791 sub qemu_img_format {
6792     my ($scfg, $volname) = @_;
6793
6794     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6795         return $1;
6796     } else {
6797         return "raw";
6798     }
6799 }
6800
6801 sub qemu_drive_mirror {
6802     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6803
6804     $jobs = {} if !$jobs;
6805
6806     my $qemu_target;
6807     my $format;
6808     $jobs->{"drive-$drive"} = {};
6809
6810     if ($dst_volid =~ /^nbd:/) {
6811         $qemu_target = $dst_volid;
6812         $format = "nbd";
6813     } else {
6814         my $storecfg = PVE::Storage::config();
6815         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6816
6817         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6818
6819         $format = qemu_img_format($dst_scfg, $dst_volname);
6820
6821         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6822
6823         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6824     }
6825
6826     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6827     $opts->{format} = $format if $format;
6828
6829     if (defined($bwlimit)) {
6830         $opts->{speed} = $bwlimit * 1024;
6831         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6832     } else {
6833         print "drive mirror is starting for drive-$drive\n";
6834     }
6835
6836     # if a job already runs for this device we get an error, catch it for cleanup
6837     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6838     if (my $err = $@) {
6839         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6840         warn "$@\n" if $@;
6841         die "mirroring error: $err\n";
6842     }
6843
6844     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6845 }
6846
6847 sub qemu_drive_mirror_monitor {
6848     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6849
6850     eval {
6851         my $err_complete = 0;
6852
6853         while (1) {
6854             die "storage migration timed out\n" if $err_complete > 300;
6855
6856             my $stats = mon_cmd($vmid, "query-block-jobs");
6857
6858             my $running_mirror_jobs = {};
6859             foreach my $stat (@$stats) {
6860                 next if $stat->{type} ne 'mirror';
6861                 $running_mirror_jobs->{$stat->{device}} = $stat;
6862             }
6863
6864             my $readycounter = 0;
6865
6866             foreach my $job (keys %$jobs) {
6867
6868                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6869                     print "$job : finished\n";
6870                     delete $jobs->{$job};
6871                     next;
6872                 }
6873
6874                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6875
6876                 my $busy = $running_mirror_jobs->{$job}->{busy};
6877                 my $ready = $running_mirror_jobs->{$job}->{ready};
6878                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6879                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6880                     my $remaining = $total - $transferred;
6881                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6882
6883                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6884                 }
6885
6886                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6887             }
6888
6889             last if scalar(keys %$jobs) == 0;
6890
6891             if ($readycounter == scalar(keys %$jobs)) {
6892                 print "all mirroring jobs are ready \n";
6893                 last if $skipcomplete; #do the complete later
6894
6895                 if ($vmiddst && $vmiddst != $vmid) {
6896                     my $agent_running = $qga && qga_check_running($vmid);
6897                     if ($agent_running) {
6898                         print "freeze filesystem\n";
6899                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6900                     } else {
6901                         print "suspend vm\n";
6902                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6903                     }
6904
6905                     # if we clone a disk for a new target vm, we don't switch the disk
6906                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6907
6908                     if ($agent_running) {
6909                         print "unfreeze filesystem\n";
6910                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6911                     } else {
6912                         print "resume vm\n";
6913                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6914                     }
6915
6916                     last;
6917                 } else {
6918
6919                     foreach my $job (keys %$jobs) {
6920                         # try to switch the disk if source and destination are on the same guest
6921                         print "$job: Completing block job...\n";
6922
6923                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6924                         if ($@ =~ m/cannot be completed/) {
6925                             print "$job: Block job cannot be completed, try again.\n";
6926                             $err_complete++;
6927                         }else {
6928                             print "$job: Completed successfully.\n";
6929                             $jobs->{$job}->{complete} = 1;
6930                         }
6931                     }
6932                 }
6933             }
6934             sleep 1;
6935         }
6936     };
6937     my $err = $@;
6938
6939     if ($err) {
6940         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6941         die "mirroring error: $err";
6942     }
6943
6944 }
6945
6946 sub qemu_blockjobs_cancel {
6947     my ($vmid, $jobs) = @_;
6948
6949     foreach my $job (keys %$jobs) {
6950         print "$job: Cancelling block job\n";
6951         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6952         $jobs->{$job}->{cancel} = 1;
6953     }
6954
6955     while (1) {
6956         my $stats = mon_cmd($vmid, "query-block-jobs");
6957
6958         my $running_jobs = {};
6959         foreach my $stat (@$stats) {
6960             $running_jobs->{$stat->{device}} = $stat;
6961         }
6962
6963         foreach my $job (keys %$jobs) {
6964
6965             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6966                 print "$job: Done.\n";
6967                 delete $jobs->{$job};
6968             }
6969         }
6970
6971         last if scalar(keys %$jobs) == 0;
6972
6973         sleep 1;
6974     }
6975 }
6976
6977 sub clone_disk {
6978     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6979         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6980
6981     my $newvolid;
6982
6983     if (!$full) {
6984         print "create linked clone of drive $drivename ($drive->{file})\n";
6985         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6986         push @$newvollist, $newvolid;
6987     } else {
6988
6989         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6990         $storeid = $storage if $storage;
6991
6992         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6993         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6994
6995         print "create full clone of drive $drivename ($drive->{file})\n";
6996         my $name = undef;
6997         if (drive_is_cloudinit($drive)) {
6998             $name = "vm-$newvmid-cloudinit";
6999             $name .= ".$dst_format" if $dst_format ne 'raw';
7000             $snapname = undef;
7001             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7002         }
7003         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7004         push @$newvollist, $newvolid;
7005
7006         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7007
7008         if (drive_is_cloudinit($drive)) {
7009             goto no_data_clone;
7010         }
7011
7012         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7013         if (!$running || $snapname) {
7014             # TODO: handle bwlimits
7015             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7016         } else {
7017
7018             my $kvmver = get_running_qemu_version ($vmid);
7019             if (!min_version($kvmver, 2, 7)) {
7020                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7021                     if $drive->{iothread};
7022             }
7023
7024             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7025         }
7026     }
7027
7028 no_data_clone:
7029     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7030
7031     my $disk = $drive;
7032     $disk->{format} = undef;
7033     $disk->{file} = $newvolid;
7034     $disk->{size} = $size;
7035
7036     return $disk;
7037 }
7038
7039 sub get_running_qemu_version {
7040     my ($vmid) = @_;
7041     my $res = mon_cmd($vmid, "query-version");
7042     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7043 }
7044
7045 sub qemu_use_old_bios_files {
7046     my ($machine_type) = @_;
7047
7048     return if !$machine_type;
7049
7050     my $use_old_bios_files = undef;
7051
7052     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7053         $machine_type = $1;
7054         $use_old_bios_files = 1;
7055     } else {
7056         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7057         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7058         # load new efi bios files on migration. So this hack is required to allow
7059         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7060         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7061         $use_old_bios_files = !min_version($version, 2, 4);
7062     }
7063
7064     return ($use_old_bios_files, $machine_type);
7065 }
7066
7067 sub create_efidisk($$$$$) {
7068     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7069
7070     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7071     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7072
7073     my $vars_size_b = -s $ovmf_vars;
7074     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7075     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7076     PVE::Storage::activate_volumes($storecfg, [$volid]);
7077
7078     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7079     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7080
7081     return ($volid, $size/1024);
7082 }
7083
7084 sub vm_iothreads_list {
7085     my ($vmid) = @_;
7086
7087     my $res = mon_cmd($vmid, 'query-iothreads');
7088
7089     my $iothreads = {};
7090     foreach my $iothread (@$res) {
7091         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7092     }
7093
7094     return $iothreads;
7095 }
7096
7097 sub scsihw_infos {
7098     my ($conf, $drive) = @_;
7099
7100     my $maxdev = 0;
7101
7102     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7103         $maxdev = 7;
7104     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7105         $maxdev = 1;
7106     } else {
7107         $maxdev = 256;
7108     }
7109
7110     my $controller = int($drive->{index} / $maxdev);
7111     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7112
7113     return ($maxdev, $controller, $controller_prefix);
7114 }
7115
7116 sub add_hyperv_enlightenments {
7117     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7118
7119     return if $winversion < 6;
7120     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7121
7122     if ($gpu_passthrough || defined($hv_vendor_id)) {
7123         $hv_vendor_id //= 'proxmox';
7124         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7125     }
7126
7127     if (min_version($machine_version, 2, 3)) {
7128         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7129         push @$cpuFlags , 'hv_vapic';
7130         push @$cpuFlags , 'hv_time';
7131     } else {
7132         push @$cpuFlags , 'hv_spinlocks=0xffff';
7133     }
7134
7135     if (min_version($machine_version, 2, 6)) {
7136         push @$cpuFlags , 'hv_reset';
7137         push @$cpuFlags , 'hv_vpindex';
7138         push @$cpuFlags , 'hv_runtime';
7139     }
7140
7141     if ($winversion >= 7) {
7142         push @$cpuFlags , 'hv_relaxed';
7143
7144         if (min_version($machine_version, 2, 12)) {
7145             push @$cpuFlags , 'hv_synic';
7146             push @$cpuFlags , 'hv_stimer';
7147         }
7148
7149         if (min_version($machine_version, 3, 1)) {
7150             push @$cpuFlags , 'hv_ipi';
7151         }
7152     }
7153 }
7154
7155 sub windows_version {
7156     my ($ostype) = @_;
7157
7158     return 0 if !$ostype;
7159
7160     my $winversion = 0;
7161
7162     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7163         $winversion = 5;
7164     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7165         $winversion = 6;
7166     } elsif ($ostype =~ m/^win(\d+)$/) {
7167         $winversion = $1;
7168     }
7169
7170     return $winversion;
7171 }
7172
7173 sub resolve_dst_disk_format {
7174         my ($storecfg, $storeid, $src_volname, $format) = @_;
7175         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7176
7177         if (!$format) {
7178             # if no target format is specified, use the source disk format as hint
7179             if ($src_volname) {
7180                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7181                 $format = qemu_img_format($scfg, $src_volname);
7182             } else {
7183                 return $defFormat;
7184             }
7185         }
7186
7187         # test if requested format is supported - else use default
7188         my $supported = grep { $_ eq $format } @$validFormats;
7189         $format = $defFormat if !$supported;
7190         return $format;
7191 }
7192
7193 sub resolve_first_disk {
7194     my $conf = shift;
7195     my @disks = PVE::QemuServer::valid_drive_names();
7196     my $firstdisk;
7197     foreach my $ds (reverse @disks) {
7198         next if !$conf->{$ds};
7199         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7200         next if PVE::QemuServer::drive_is_cdrom($disk);
7201         $firstdisk = $ds;
7202     }
7203     return $firstdisk;
7204 }
7205
7206 sub generate_uuid {
7207     my ($uuid, $uuid_str);
7208     UUID::generate($uuid);
7209     UUID::unparse($uuid, $uuid_str);
7210     return $uuid_str;
7211 }
7212
7213 sub generate_smbios1_uuid {
7214     return "uuid=".generate_uuid();
7215 }
7216
7217 sub nbd_stop {
7218     my ($vmid) = @_;
7219
7220     mon_cmd($vmid, 'nbd-server-stop');
7221 }
7222
7223 sub create_reboot_request {
7224     my ($vmid) = @_;
7225     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7226         or die "failed to create reboot trigger file: $!\n";
7227     close($fh);
7228 }
7229
7230 sub clear_reboot_request {
7231     my ($vmid) = @_;
7232     my $path = "/run/qemu-server/$vmid.reboot";
7233     my $res = 0;
7234
7235     $res = unlink($path);
7236     die "could not remove reboot request for $vmid: $!"
7237         if !$res && $! != POSIX::ENOENT;
7238
7239     return $res;
7240 }
7241
7242 # bash completion helper
7243
7244 sub complete_backup_archives {
7245     my ($cmdname, $pname, $cvalue) = @_;
7246
7247     my $cfg = PVE::Storage::config();
7248
7249     my $storeid;
7250
7251     if ($cvalue =~ m/^([^:]+):/) {
7252         $storeid = $1;
7253     }
7254
7255     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7256
7257     my $res = [];
7258     foreach my $id (keys %$data) {
7259         foreach my $item (@{$data->{$id}}) {
7260             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7261             push @$res, $item->{volid} if defined($item->{volid});
7262         }
7263     }
7264
7265     return $res;
7266 }
7267
7268 my $complete_vmid_full = sub {
7269     my ($running) = @_;
7270
7271     my $idlist = vmstatus();
7272
7273     my $res = [];
7274
7275     foreach my $id (keys %$idlist) {
7276         my $d = $idlist->{$id};
7277         if (defined($running)) {
7278             next if $d->{template};
7279             next if $running && $d->{status} ne 'running';
7280             next if !$running && $d->{status} eq 'running';
7281         }
7282         push @$res, $id;
7283
7284     }
7285     return $res;
7286 };
7287
7288 sub complete_vmid {
7289     return &$complete_vmid_full();
7290 }
7291
7292 sub complete_vmid_stopped {
7293     return &$complete_vmid_full(0);
7294 }
7295
7296 sub complete_vmid_running {
7297     return &$complete_vmid_full(1);
7298 }
7299
7300 sub complete_storage {
7301
7302     my $cfg = PVE::Storage::config();
7303     my $ids = $cfg->{ids};
7304
7305     my $res = [];
7306     foreach my $sid (keys %$ids) {
7307         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7308         next if !$ids->{$sid}->{content}->{images};
7309         push @$res, $sid;
7310     }
7311
7312     return $res;
7313 }
7314
7315 sub complete_migration_storage {
7316     my ($cmd, $param, $current_value, $all_args) = @_;
7317
7318     my $targetnode = @$all_args[1];
7319
7320     my $cfg = PVE::Storage::config();
7321     my $ids = $cfg->{ids};
7322
7323     my $res = [];
7324     foreach my $sid (keys %$ids) {
7325         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7326         next if !$ids->{$sid}->{content}->{images};
7327         push @$res, $sid;
7328     }
7329
7330     return $res;
7331 }
7332
7333 1;