PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers;
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Memory;
  47 use PVE::QemuServer::Monitor qw(mon_cmd);
  48 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  49 use PVE::QemuServer::USB qw(parse_usb_device);
  50
  51 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  52 my $OVMF = {
  53     x86_64 => [
  54         "$EDK2_FW_BASE/OVMF_CODE.fd",
  55         "$EDK2_FW_BASE/OVMF_VARS.fd"
  56     ],
  57     aarch64 => [
  58         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  59         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  60     ],
  61 };
  62
  63 my $qemu_snap_storage = { rbd => 1 };
  64
  65 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  66
  67 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  68
  69 # Note about locking: we use flock on the config file protect
  70 # against concurent actions.
  71 # Aditionaly, we have a 'lock' setting in the config file. This
  72 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  73 # allowed when such lock is set. But you can ignore this kind of
  74 # lock with the --skiplock flag.
  75
  76 cfs_register_file('/qemu-server/',
  77                   \&parse_vm_config,
  78                   \&write_vm_config);
  79
  80 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  81     description => "Some command save/restore state from this location.",
  82     type => 'string',
  83     maxLength => 128,
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  88     type => 'string',
  89     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  90     description => "The drive's backing file's data format.",
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102 #no warnings 'redefine';
 103
 104 sub cgroups_write {
 105    my ($controller, $vmid, $option, $value) = @_;
 106
 107    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 108    PVE::ProcFSTools::write_proc_entry($path, $value);
 109
 110 }
 111
 112 my $nodename = PVE::INotify::nodename();
 113
 114 my $cpu_vendor_list = {
 115     # Intel CPUs
 116     486 => 'GenuineIntel',
 117     pentium => 'GenuineIntel',
 118     pentium2  => 'GenuineIntel',
 119     pentium3  => 'GenuineIntel',
 120     coreduo => 'GenuineIntel',
 121     core2duo => 'GenuineIntel',
 122     Conroe  => 'GenuineIntel',
 123     Penryn  => 'GenuineIntel',
 124     Nehalem  => 'GenuineIntel',
 125     'Nehalem-IBRS'  => 'GenuineIntel',
 126     Westmere => 'GenuineIntel',
 127     'Westmere-IBRS' => 'GenuineIntel',
 128     SandyBridge => 'GenuineIntel',
 129     'SandyBridge-IBRS' => 'GenuineIntel',
 130     IvyBridge => 'GenuineIntel',
 131     'IvyBridge-IBRS' => 'GenuineIntel',
 132     Haswell => 'GenuineIntel',
 133     'Haswell-IBRS' => 'GenuineIntel',
 134     'Haswell-noTSX' => 'GenuineIntel',
 135     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 136     Broadwell => 'GenuineIntel',
 137     'Broadwell-IBRS' => 'GenuineIntel',
 138     'Broadwell-noTSX' => 'GenuineIntel',
 139     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 140     'Skylake-Client' => 'GenuineIntel',
 141     'Skylake-Client-IBRS' => 'GenuineIntel',
 142     'Skylake-Server' => 'GenuineIntel',
 143     'Skylake-Server-IBRS' => 'GenuineIntel',
 144     'Cascadelake-Server' => 'GenuineIntel',
 145     KnightsMill => 'GenuineIntel',
 146
 147
 148     # AMD CPUs
 149     athlon => 'AuthenticAMD',
 150     phenom  => 'AuthenticAMD',
 151     Opteron_G1  => 'AuthenticAMD',
 152     Opteron_G2  => 'AuthenticAMD',
 153     Opteron_G3  => 'AuthenticAMD',
 154     Opteron_G4  => 'AuthenticAMD',
 155     Opteron_G5  => 'AuthenticAMD',
 156     EPYC => 'AuthenticAMD',
 157     'EPYC-IBPB' => 'AuthenticAMD',
 158
 159     # generic types, use vendor from host node
 160     host => 'default',
 161     kvm32 => 'default',
 162     kvm64 => 'default',
 163     qemu32 => 'default',
 164     qemu64 => 'default',
 165     max => 'default',
 166 };
 167
 168 my @supported_cpu_flags = (
 169     'pcid',
 170     'spec-ctrl',
 171     'ibpb',
 172     'ssbd',
 173     'virt-ssbd',
 174     'amd-ssbd',
 175     'amd-no-ssb',
 176     'pdpe1gb',
 177     'md-clear',
 178     'hv-tlbflush',
 179     'hv-evmcs',
 180     'aes'
 181 );
 182 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 183
 184 my $cpu_fmt = {
 185     cputype => {
 186         description => "Emulated CPU type.",
 187         type => 'string',
 188         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 189         default => 'kvm64',
 190         default_key => 1,
 191     },
 192     hidden => {
 193         description => "Do not identify as a KVM virtual machine.",
 194         type => 'boolean',
 195         optional => 1,
 196         default => 0
 197     },
 198     'hv-vendor-id' => {
 199         type => 'string',
 200         pattern => qr/[a-zA-Z0-9]{1,12}/,
 201         format_description => 'vendor-id',
 202         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 203         optional => 1,
 204     },
 205     flags => {
 206         description => "List of additional CPU flags separated by ';'."
 207                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 208                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 209         format_description => '+FLAG[;-FLAG...]',
 210         type => 'string',
 211         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 212         optional => 1,
 213     },
 214 };
 215
 216 my $watchdog_fmt = {
 217     model => {
 218         default_key => 1,
 219         type => 'string',
 220         enum => [qw(i6300esb ib700)],
 221         description => "Watchdog type to emulate.",
 222         default => 'i6300esb',
 223         optional => 1,
 224     },
 225     action => {
 226         type => 'string',
 227         enum => [qw(reset shutdown poweroff pause debug none)],
 228         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 229         optional => 1,
 230     },
 231 };
 232 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 233
 234 my $agent_fmt = {
 235     enabled => {
 236         description => "Enable/disable Qemu GuestAgent.",
 237         type => 'boolean',
 238         default => 0,
 239         default_key => 1,
 240     },
 241     fstrim_cloned_disks => {
 242         description => "Run fstrim after cloning/moving a disk.",
 243         type => 'boolean',
 244         optional => 1,
 245         default => 0
 246     },
 247     type => {
 248         description => "Select the agent type",
 249         type => 'string',
 250         default => 'virtio',
 251         optional => 1,
 252         enum => [qw(virtio isa)],
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6 - 5.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 my $cdrom_path;
1557 sub get_cdrom_path {
1558
1559     return  $cdrom_path if $cdrom_path;
1560
1561     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1562     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1563     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1564 }
1565
1566 sub get_iso_path {
1567     my ($storecfg, $vmid, $cdrom) = @_;
1568
1569     if ($cdrom eq 'cdrom') {
1570         return get_cdrom_path();
1571     } elsif ($cdrom eq 'none') {
1572         return '';
1573     } elsif ($cdrom =~ m|^/|) {
1574         return $cdrom;
1575     } else {
1576         return PVE::Storage::path($storecfg, $cdrom);
1577     }
1578 }
1579
1580 # try to convert old style file names to volume IDs
1581 sub filename_to_volume_id {
1582     my ($vmid, $file, $media) = @_;
1583
1584      if (!($file eq 'none' || $file eq 'cdrom' ||
1585           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1586
1587         return undef if $file =~ m|/|;
1588
1589         if ($media && $media eq 'cdrom') {
1590             $file = "local:iso/$file";
1591         } else {
1592             $file = "local:$vmid/$file";
1593         }
1594     }
1595
1596     return $file;
1597 }
1598
1599 sub verify_media_type {
1600     my ($opt, $vtype, $media) = @_;
1601
1602     return if !$media;
1603
1604     my $etype;
1605     if ($media eq 'disk') {
1606         $etype = 'images';
1607     } elsif ($media eq 'cdrom') {
1608         $etype = 'iso';
1609     } else {
1610         die "internal error";
1611     }
1612
1613     return if ($vtype eq $etype);
1614
1615     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1616 }
1617
1618 sub cleanup_drive_path {
1619     my ($opt, $storecfg, $drive) = @_;
1620
1621     # try to convert filesystem paths to volume IDs
1622
1623     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1624         ($drive->{file} !~ m|^/dev/.+|) &&
1625         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1626         ($drive->{file} !~ m/^\d+$/)) {
1627         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1628         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1629         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1630         verify_media_type($opt, $vtype, $drive->{media});
1631         $drive->{file} = $volid;
1632     }
1633
1634     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1635 }
1636
1637 sub parse_hotplug_features {
1638     my ($data) = @_;
1639
1640     my $res = {};
1641
1642     return $res if $data eq '0';
1643
1644     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1645
1646     foreach my $feature (PVE::Tools::split_list($data)) {
1647         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1648             $res->{$1} = 1;
1649         } else {
1650             die "invalid hotplug feature '$feature'\n";
1651         }
1652     }
1653     return $res;
1654 }
1655
1656 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1657 sub pve_verify_hotplug_features {
1658     my ($value, $noerr) = @_;
1659
1660     return $value if parse_hotplug_features($value);
1661
1662     return undef if $noerr;
1663
1664     die "unable to parse hotplug option\n";
1665 }
1666
1667 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1668 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1669 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1670 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1671 #        [,iothread=on][,serial=serial][,model=model]
1672
1673 sub parse_drive {
1674     my ($key, $data) = @_;
1675
1676     my ($interface, $index);
1677
1678     if ($key =~ m/^([^\d]+)(\d+)$/) {
1679         $interface = $1;
1680         $index = $2;
1681     } else {
1682         return undef;
1683     }
1684
1685     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1686                                      : $confdesc->{$key}->{format};
1687     if (!$desc) {
1688         warn "invalid drive key: $key\n";
1689         return undef;
1690     }
1691     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1692     return undef if !$res;
1693     $res->{interface} = $interface;
1694     $res->{index} = $index;
1695
1696     my $error = 0;
1697     foreach my $opt (qw(bps bps_rd bps_wr)) {
1698         if (my $bps = defined(delete $res->{$opt})) {
1699             if (defined($res->{"m$opt"})) {
1700                 warn "both $opt and m$opt specified\n";
1701                 ++$error;
1702                 next;
1703             }
1704             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1705         }
1706     }
1707
1708     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1709     for my $requirement (
1710         [mbps_max => 'mbps'],
1711         [mbps_rd_max => 'mbps_rd'],
1712         [mbps_wr_max => 'mbps_wr'],
1713         [miops_max => 'miops'],
1714         [miops_rd_max => 'miops_rd'],
1715         [miops_wr_max => 'miops_wr'],
1716         [bps_max_length => 'mbps_max'],
1717         [bps_rd_max_length => 'mbps_rd_max'],
1718         [bps_wr_max_length => 'mbps_wr_max'],
1719         [iops_max_length => 'iops_max'],
1720         [iops_rd_max_length => 'iops_rd_max'],
1721         [iops_wr_max_length => 'iops_wr_max']) {
1722         my ($option, $requires) = @$requirement;
1723         if ($res->{$option} && !$res->{$requires}) {
1724             warn "$option requires $requires\n";
1725             ++$error;
1726         }
1727     }
1728
1729     return undef if $error;
1730
1731     return undef if $res->{mbps_rd} && $res->{mbps};
1732     return undef if $res->{mbps_wr} && $res->{mbps};
1733     return undef if $res->{iops_rd} && $res->{iops};
1734     return undef if $res->{iops_wr} && $res->{iops};
1735
1736     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1737         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1738         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1739         return undef if $res->{interface} eq 'virtio';
1740     }
1741
1742     if (my $size = $res->{size}) {
1743         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1744     }
1745
1746     return $res;
1747 }
1748
1749 sub print_drive {
1750     my ($vmid, $drive) = @_;
1751     my $data = { %$drive };
1752     delete $data->{$_} for qw(index interface);
1753     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1754 }
1755
1756 sub scsi_inquiry {
1757     my($fh, $noerr) = @_;
1758
1759     my $SG_IO = 0x2285;
1760     my $SG_GET_VERSION_NUM = 0x2282;
1761
1762     my $versionbuf = "\x00" x 8;
1763     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1764     if (!$ret) {
1765         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1766         return undef;
1767     }
1768     my $version = unpack("I", $versionbuf);
1769     if ($version < 30000) {
1770         die "scsi generic interface too old\n"  if !$noerr;
1771         return undef;
1772     }
1773
1774     my $buf = "\x00" x 36;
1775     my $sensebuf = "\x00" x 8;
1776     my $cmd = pack("C x3 C x1", 0x12, 36);
1777
1778     # see /usr/include/scsi/sg.h
1779     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1780
1781     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1782                       length($sensebuf), 0, length($buf), $buf,
1783                       $cmd, $sensebuf, 6000);
1784
1785     $ret = ioctl($fh, $SG_IO, $packet);
1786     if (!$ret) {
1787         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790
1791     my @res = unpack($sg_io_hdr_t, $packet);
1792     if ($res[17] || $res[18]) {
1793         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1794         return undef;
1795     }
1796
1797     my $res = {};
1798     (my $byte0, my $byte1, $res->{vendor},
1799      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1800
1801     $res->{removable} = $byte1 & 128 ? 1 : 0;
1802     $res->{type} = $byte0 & 31;
1803
1804     return $res;
1805 }
1806
1807 sub path_is_scsi {
1808     my ($path) = @_;
1809
1810     my $fh = IO::File->new("+<$path") || return undef;
1811     my $res = scsi_inquiry($fh, 1);
1812     close($fh);
1813
1814     return $res;
1815 }
1816
1817 sub machine_type_is_q35 {
1818     my ($conf) = @_;
1819
1820     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1821 }
1822
1823 sub print_tabletdevice_full {
1824     my ($conf, $arch) = @_;
1825
1826     my $q35 = machine_type_is_q35($conf);
1827
1828     # we use uhci for old VMs because tablet driver was buggy in older qemu
1829     my $usbbus;
1830     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1831         $usbbus = 'ehci';
1832     } else {
1833         $usbbus = 'uhci';
1834     }
1835
1836     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1837 }
1838
1839 sub print_keyboarddevice_full {
1840     my ($conf, $arch, $machine) = @_;
1841
1842     return undef if $arch ne 'aarch64';
1843
1844     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1845 }
1846
1847 sub print_drivedevice_full {
1848     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1849
1850     my $device = '';
1851     my $maxdev = 0;
1852
1853     if ($drive->{interface} eq 'virtio') {
1854         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1855         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1856         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1857     } elsif ($drive->{interface} eq 'scsi') {
1858
1859         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1860         my $unit = $drive->{index} % $maxdev;
1861         my $devicetype = 'hd';
1862         my $path = '';
1863         if (drive_is_cdrom($drive)) {
1864             $devicetype = 'cd';
1865         } else {
1866             if ($drive->{file} =~ m|^/|) {
1867                 $path = $drive->{file};
1868                 if (my $info = path_is_scsi($path)) {
1869                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1870                         $devicetype = 'block';
1871                     } elsif ($info->{type} == 1) { # tape
1872                         $devicetype = 'generic';
1873                     }
1874                 }
1875             } else {
1876                  $path = PVE::Storage::path($storecfg, $drive->{file});
1877             }
1878
1879             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1880             if ($path =~ m/^iscsi\:\/\// &&
1881                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1882                 $devicetype = 'generic';
1883             }
1884         }
1885
1886         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1887             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1888         } else {
1889             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1890         }
1891
1892         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1893             $device .= ",rotation_rate=1";
1894         }
1895         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1896
1897     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1898         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1899         my $controller = int($drive->{index} / $maxdev);
1900         my $unit = $drive->{index} % $maxdev;
1901         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1902
1903         $device = "ide-$devicetype";
1904         if ($drive->{interface} eq 'ide') {
1905             $device .= ",bus=ide.$controller,unit=$unit";
1906         } else {
1907             $device .= ",bus=ahci$controller.$unit";
1908         }
1909         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910
1911         if ($devicetype eq 'hd') {
1912             if (my $model = $drive->{model}) {
1913                 $model = URI::Escape::uri_unescape($model);
1914                 $device .= ",model=$model";
1915             }
1916             if ($drive->{ssd}) {
1917                 $device .= ",rotation_rate=1";
1918             }
1919         }
1920         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1921     } elsif ($drive->{interface} eq 'usb') {
1922         die "implement me";
1923         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1924     } else {
1925         die "unsupported interface type";
1926     }
1927
1928     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1929
1930     if (my $serial = $drive->{serial}) {
1931         $serial = URI::Escape::uri_unescape($serial);
1932         $device .= ",serial=$serial";
1933     }
1934
1935
1936     return $device;
1937 }
1938
1939 sub get_initiator_name {
1940     my $initiator;
1941
1942     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1943     while (defined(my $line = <$fh>)) {
1944         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1945         $initiator = $1;
1946         last;
1947     }
1948     $fh->close();
1949
1950     return $initiator;
1951 }
1952
1953 sub print_drive_full {
1954     my ($storecfg, $vmid, $drive) = @_;
1955
1956     my $path;
1957     my $volid = $drive->{file};
1958     my $format;
1959
1960     if (drive_is_cdrom($drive)) {
1961         $path = get_iso_path($storecfg, $vmid, $volid);
1962     } else {
1963         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1964         if ($storeid) {
1965             $path = PVE::Storage::path($storecfg, $volid);
1966             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1967             $format = qemu_img_format($scfg, $volname);
1968         } else {
1969             $path = $volid;
1970             $format = "raw";
1971         }
1972    }
1973
1974     my $opts = '';
1975     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1976     foreach my $o (@qemu_drive_options) {
1977         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1978     }
1979
1980     # snapshot only accepts on|off
1981     if (defined($drive->{snapshot})) {
1982         my $v = $drive->{snapshot} ? 'on' : 'off';
1983         $opts .= ",snapshot=$v";
1984     }
1985
1986     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1987         my ($dir, $qmpname) = @$type;
1988         if (my $v = $drive->{"mbps$dir"}) {
1989             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1990         }
1991         if (my $v = $drive->{"mbps${dir}_max"}) {
1992             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1993         }
1994         if (my $v = $drive->{"bps${dir}_max_length"}) {
1995             $opts .= ",throttling.bps$qmpname-max-length=$v";
1996         }
1997         if (my $v = $drive->{"iops${dir}"}) {
1998             $opts .= ",throttling.iops$qmpname=$v";
1999         }
2000         if (my $v = $drive->{"iops${dir}_max"}) {
2001             $opts .= ",throttling.iops$qmpname-max=$v";
2002         }
2003         if (my $v = $drive->{"iops${dir}_max_length"}) {
2004             $opts .= ",throttling.iops$qmpname-max-length=$v";
2005         }
2006     }
2007
2008     $opts .= ",format=$format" if $format && !$drive->{format};
2009
2010     my $cache_direct = 0;
2011
2012     if (my $cache = $drive->{cache}) {
2013         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2014     } elsif (!drive_is_cdrom($drive)) {
2015         $opts .= ",cache=none";
2016         $cache_direct = 1;
2017     }
2018
2019     # aio native works only with O_DIRECT
2020     if (!$drive->{aio}) {
2021         if($cache_direct) {
2022             $opts .= ",aio=native";
2023         } else {
2024             $opts .= ",aio=threads";
2025         }
2026     }
2027
2028     if (!drive_is_cdrom($drive)) {
2029         my $detectzeroes;
2030         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2031             $detectzeroes = 'off';
2032         } elsif ($drive->{discard}) {
2033             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2034         } else {
2035             # This used to be our default with discard not being specified:
2036             $detectzeroes = 'on';
2037         }
2038         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2039     }
2040
2041     my $pathinfo = $path ? "file=$path," : '';
2042
2043     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2044 }
2045
2046 sub print_netdevice_full {
2047     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2048
2049     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2050
2051     my $device = $net->{model};
2052     if ($net->{model} eq 'virtio') {
2053          $device = 'virtio-net-pci';
2054      };
2055
2056     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2057     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2058     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2059         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2060         my $vectors = $net->{queues} * 2 + 2;
2061         $tmpstr .= ",vectors=$vectors,mq=on";
2062     }
2063     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2064
2065     if ($use_old_bios_files) {
2066         my $romfile;
2067         if ($device eq 'virtio-net-pci') {
2068             $romfile = 'pxe-virtio.rom';
2069         } elsif ($device eq 'e1000') {
2070             $romfile = 'pxe-e1000.rom';
2071         } elsif ($device eq 'ne2k') {
2072             $romfile = 'pxe-ne2k_pci.rom';
2073         } elsif ($device eq 'pcnet') {
2074             $romfile = 'pxe-pcnet.rom';
2075         } elsif ($device eq 'rtl8139') {
2076             $romfile = 'pxe-rtl8139.rom';
2077         }
2078         $tmpstr .= ",romfile=$romfile" if $romfile;
2079     }
2080
2081     return $tmpstr;
2082 }
2083
2084 sub print_netdev_full {
2085     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2086
2087     my $i = '';
2088     if ($netid =~ m/^net(\d+)$/) {
2089         $i = int($1);
2090     }
2091
2092     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2093
2094     my $ifname = "tap${vmid}i$i";
2095
2096     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2097     die "interface name '$ifname' is too long (max 15 character)\n"
2098         if length($ifname) >= 16;
2099
2100     my $vhostparam = '';
2101     if (is_native($arch)) {
2102         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2103     }
2104
2105     my $vmname = $conf->{name} || "vm$vmid";
2106
2107     my $netdev = "";
2108     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2109
2110     if ($net->{bridge}) {
2111         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2112     } else {
2113         $netdev = "type=user,id=$netid,hostname=$vmname";
2114     }
2115
2116     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2117
2118     return $netdev;
2119 }
2120
2121
2122 sub print_cpu_device {
2123     my ($conf, $id) = @_;
2124
2125     my $kvm = $conf->{kvm} // 1;
2126     my $cpu = $kvm ? "kvm64" : "qemu64";
2127     if (my $cputype = $conf->{cpu}) {
2128         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2129             or die "Cannot parse cpu description: $cputype\n";
2130         $cpu = $cpuconf->{cputype};
2131     }
2132
2133     my $cores = $conf->{cores} || 1;
2134
2135     my $current_core = ($id - 1) % $cores;
2136     my $current_socket = int(($id - 1 - $current_core)/$cores);
2137
2138     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2139 }
2140
2141 my $vga_map = {
2142     'cirrus' => 'cirrus-vga',
2143     'std' => 'VGA',
2144     'vmware' => 'vmware-svga',
2145     'virtio' => 'virtio-vga',
2146 };
2147
2148 sub print_vga_device {
2149     my ($conf, $vga, $arch, $kvmver, $machine, $id, $qxlnum, $bridges) = @_;
2150
2151     my $type = $vga_map->{$vga->{type}};
2152     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2153         $type = 'virtio-gpu';
2154     }
2155     my $vgamem_mb = $vga->{memory};
2156
2157     my $max_outputs = '';
2158     if ($qxlnum) {
2159         $type = $id ? 'qxl' : 'qxl-vga';
2160
2161         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2162             # set max outputs so linux can have up to 4 qxl displays with one device
2163             if (qemu_machine_feature_enabled($machine, $kvmver, 4, 1)) {
2164                 $max_outputs = ",max_outputs=4";
2165             }
2166         }
2167     }
2168
2169     die "no devicetype for $vga->{type}\n" if !$type;
2170
2171     my $memory = "";
2172     if ($vgamem_mb) {
2173         if ($vga->{type} eq 'virtio') {
2174             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2175             $memory = ",max_hostmem=$bytes";
2176         } elsif ($qxlnum) {
2177             # from https://www.spice-space.org/multiple-monitors.html
2178             $memory = ",vgamem_mb=$vga->{memory}";
2179             my $ram = $vgamem_mb * 4;
2180             my $vram = $vgamem_mb * 2;
2181             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2182         } else {
2183             $memory = ",vgamem_mb=$vga->{memory}";
2184         }
2185     } elsif ($qxlnum && $id) {
2186         $memory = ",ram_size=67108864,vram_size=33554432";
2187     }
2188
2189     my $q35 = machine_type_is_q35($conf);
2190     my $vgaid = "vga" . ($id // '');
2191     my $pciaddr;
2192
2193     if ($q35 && $vgaid eq 'vga') {
2194         # the first display uses pcie.0 bus on q35 machines
2195         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2196     } else {
2197         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2198     }
2199
2200     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2201 }
2202
2203 sub drive_is_cloudinit {
2204     my ($drive) = @_;
2205     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2206 }
2207
2208 sub drive_is_cdrom {
2209     my ($drive, $exclude_cloudinit) = @_;
2210
2211     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2212
2213     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2214
2215 }
2216
2217 sub parse_number_sets {
2218     my ($set) = @_;
2219     my $res = [];
2220     foreach my $part (split(/;/, $set)) {
2221         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2222             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2223             push @$res, [ $1, $2 ];
2224         } else {
2225             die "invalid range: $part\n";
2226         }
2227     }
2228     return $res;
2229 }
2230
2231 sub parse_numa {
2232     my ($data) = @_;
2233
2234     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2235     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2236     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2237     return $res;
2238 }
2239
2240 sub parse_hostpci {
2241     my ($value) = @_;
2242
2243     return undef if !$value;
2244
2245     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2246
2247     my @idlist = split(/;/, $res->{host});
2248     delete $res->{host};
2249     foreach my $id (@idlist) {
2250         if ($id =~ m/\./) { # full id 00:00.1
2251             push @{$res->{pciid}}, {
2252                 id => $id,
2253             };
2254         } else { # partial id 00:00
2255             $res->{pciid} = PVE::SysFSTools::lspci($id);
2256         }
2257     }
2258     return $res;
2259 }
2260
2261 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2262 sub parse_net {
2263     my ($data) = @_;
2264
2265     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2266     if ($@) {
2267         warn $@;
2268         return undef;
2269     }
2270     if (!defined($res->{macaddr})) {
2271         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2272         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2273     }
2274     return $res;
2275 }
2276
2277 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2278 sub parse_ipconfig {
2279     my ($data) = @_;
2280
2281     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2282     if ($@) {
2283         warn $@;
2284         return undef;
2285     }
2286
2287     if ($res->{gw} && !$res->{ip}) {
2288         warn 'gateway specified without specifying an IP address';
2289         return undef;
2290     }
2291     if ($res->{gw6} && !$res->{ip6}) {
2292         warn 'IPv6 gateway specified without specifying an IPv6 address';
2293         return undef;
2294     }
2295     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2296         warn 'gateway specified together with DHCP';
2297         return undef;
2298     }
2299     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2300         # gw6 + auto/dhcp
2301         warn "IPv6 gateway specified together with $res->{ip6} address";
2302         return undef;
2303     }
2304
2305     if (!$res->{ip} && !$res->{ip6}) {
2306         return { ip => 'dhcp', ip6 => 'dhcp' };
2307     }
2308
2309     return $res;
2310 }
2311
2312 sub print_net {
2313     my $net = shift;
2314
2315     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2316 }
2317
2318 sub add_random_macs {
2319     my ($settings) = @_;
2320
2321     foreach my $opt (keys %$settings) {
2322         next if $opt !~ m/^net(\d+)$/;
2323         my $net = parse_net($settings->{$opt});
2324         next if !$net;
2325         $settings->{$opt} = print_net($net);
2326     }
2327 }
2328
2329 sub vm_is_volid_owner {
2330     my ($storecfg, $vmid, $volid) = @_;
2331
2332     if ($volid !~  m|^/|) {
2333         my ($path, $owner);
2334         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2335         if ($owner && ($owner == $vmid)) {
2336             return 1;
2337         }
2338     }
2339
2340     return undef;
2341 }
2342
2343 sub vmconfig_register_unused_drive {
2344     my ($storecfg, $vmid, $conf, $drive) = @_;
2345
2346     if (drive_is_cloudinit($drive)) {
2347         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2348         warn $@ if $@;
2349     } elsif (!drive_is_cdrom($drive)) {
2350         my $volid = $drive->{file};
2351         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2352             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2353         }
2354     }
2355 }
2356
2357 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2358 my $smbios1_fmt = {
2359     uuid => {
2360         type => 'string',
2361         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2362         format_description => 'UUID',
2363         description => "Set SMBIOS1 UUID.",
2364         optional => 1,
2365     },
2366     version => {
2367         type => 'string',
2368         pattern => '[A-Za-z0-9+\/]+={0,2}',
2369         format_description => 'Base64 encoded string',
2370         description => "Set SMBIOS1 version.",
2371         optional => 1,
2372     },
2373     serial => {
2374         type => 'string',
2375         pattern => '[A-Za-z0-9+\/]+={0,2}',
2376         format_description => 'Base64 encoded string',
2377         description => "Set SMBIOS1 serial number.",
2378         optional => 1,
2379     },
2380     manufacturer => {
2381         type => 'string',
2382         pattern => '[A-Za-z0-9+\/]+={0,2}',
2383         format_description => 'Base64 encoded string',
2384         description => "Set SMBIOS1 manufacturer.",
2385         optional => 1,
2386     },
2387     product => {
2388         type => 'string',
2389         pattern => '[A-Za-z0-9+\/]+={0,2}',
2390         format_description => 'Base64 encoded string',
2391         description => "Set SMBIOS1 product ID.",
2392         optional => 1,
2393     },
2394     sku => {
2395         type => 'string',
2396         pattern => '[A-Za-z0-9+\/]+={0,2}',
2397         format_description => 'Base64 encoded string',
2398         description => "Set SMBIOS1 SKU string.",
2399         optional => 1,
2400     },
2401     family => {
2402         type => 'string',
2403         pattern => '[A-Za-z0-9+\/]+={0,2}',
2404         format_description => 'Base64 encoded string',
2405         description => "Set SMBIOS1 family string.",
2406         optional => 1,
2407     },
2408     base64 => {
2409         type => 'boolean',
2410         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2411         optional => 1,
2412     },
2413 };
2414
2415 sub parse_smbios1 {
2416     my ($data) = @_;
2417
2418     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2419     warn $@ if $@;
2420     return $res;
2421 }
2422
2423 sub print_smbios1 {
2424     my ($smbios1) = @_;
2425     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2426 }
2427
2428 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2429
2430 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2431 sub verify_bootdisk {
2432     my ($value, $noerr) = @_;
2433
2434     return $value if is_valid_drivename($value);
2435
2436     return undef if $noerr;
2437
2438     die "invalid boot disk '$value'\n";
2439 }
2440
2441 sub parse_watchdog {
2442     my ($value) = @_;
2443
2444     return undef if !$value;
2445
2446     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2447     warn $@ if $@;
2448     return $res;
2449 }
2450
2451 sub parse_guest_agent {
2452     my ($value) = @_;
2453
2454     return {} if !defined($value->{agent});
2455
2456     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2457     warn $@ if $@;
2458
2459     # if the agent is disabled ignore the other potentially set properties
2460     return {} if !$res->{enabled};
2461     return $res;
2462 }
2463
2464 sub parse_vga {
2465     my ($value) = @_;
2466
2467     return {} if !$value;
2468     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2469     warn $@ if $@;
2470     return $res;
2471 }
2472
2473 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2474 sub verify_usb_device {
2475     my ($value, $noerr) = @_;
2476
2477     return $value if parse_usb_device($value);
2478
2479     return undef if $noerr;
2480
2481     die "unable to parse usb device\n";
2482 }
2483
2484 # add JSON properties for create and set function
2485 sub json_config_properties {
2486     my $prop = shift;
2487
2488     foreach my $opt (keys %$confdesc) {
2489         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2490         $prop->{$opt} = $confdesc->{$opt};
2491     }
2492
2493     return $prop;
2494 }
2495
2496 # return copy of $confdesc_cloudinit to generate documentation
2497 sub cloudinit_config_properties {
2498
2499     return dclone($confdesc_cloudinit);
2500 }
2501
2502 sub check_type {
2503     my ($key, $value) = @_;
2504
2505     die "unknown setting '$key'\n" if !$confdesc->{$key};
2506
2507     my $type = $confdesc->{$key}->{type};
2508
2509     if (!defined($value)) {
2510         die "got undefined value\n";
2511     }
2512
2513     if ($value =~ m/[\n\r]/) {
2514         die "property contains a line feed\n";
2515     }
2516
2517     if ($type eq 'boolean') {
2518         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2519         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2520         die "type check ('boolean') failed - got '$value'\n";
2521     } elsif ($type eq 'integer') {
2522         return int($1) if $value =~ m/^(\d+)$/;
2523         die "type check ('integer') failed - got '$value'\n";
2524     } elsif ($type eq 'number') {
2525         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2526         die "type check ('number') failed - got '$value'\n";
2527     } elsif ($type eq 'string') {
2528         if (my $fmt = $confdesc->{$key}->{format}) {
2529             PVE::JSONSchema::check_format($fmt, $value);
2530             return $value;
2531         }
2532         $value =~ s/^\"(.*)\"$/$1/;
2533         return $value;
2534     } else {
2535         die "internal error"
2536     }
2537 }
2538
2539 sub destroy_vm {
2540     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2541
2542     my $conf = PVE::QemuConfig->load_config($vmid);
2543
2544     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2545
2546     if ($conf->{template}) {
2547         # check if any base image is still used by a linked clone
2548         foreach_drive($conf, sub {
2549                 my ($ds, $drive) = @_;
2550                 return if drive_is_cdrom($drive);
2551
2552                 my $volid = $drive->{file};
2553                 return if !$volid || $volid =~ m|^/|;
2554
2555                 die "base volume '$volid' is still in use by linked cloned\n"
2556                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2557
2558         });
2559     }
2560
2561     # only remove disks owned by this VM
2562     foreach_drive($conf, sub {
2563         my ($ds, $drive) = @_;
2564         return if drive_is_cdrom($drive, 1);
2565
2566         my $volid = $drive->{file};
2567         return if !$volid || $volid =~ m|^/|;
2568
2569         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2570         return if !$path || !$owner || ($owner != $vmid);
2571
2572         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2573         warn "Could not remove disk '$volid', check manually: $@" if $@;
2574     });
2575
2576     # also remove unused disk
2577     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2578     PVE::Storage::foreach_volid($vmdisks, sub {
2579         my ($volid, $sid, $volname, $d) = @_;
2580         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2581         warn $@ if $@;
2582     });
2583
2584     if (defined $replacement_conf) {
2585         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2586     } else {
2587         PVE::QemuConfig->destroy_config($vmid);
2588     }
2589 }
2590
2591 sub parse_vm_config {
2592     my ($filename, $raw) = @_;
2593
2594     return undef if !defined($raw);
2595
2596     my $res = {
2597         digest => Digest::SHA::sha1_hex($raw),
2598         snapshots => {},
2599         pending => {},
2600     };
2601
2602     $filename =~ m|/qemu-server/(\d+)\.conf$|
2603         || die "got strange filename '$filename'";
2604
2605     my $vmid = $1;
2606
2607     my $conf = $res;
2608     my $descr;
2609     my $section = '';
2610
2611     my @lines = split(/\n/, $raw);
2612     foreach my $line (@lines) {
2613         next if $line =~ m/^\s*$/;
2614
2615         if ($line =~ m/^\[PENDING\]\s*$/i) {
2616             $section = 'pending';
2617             if (defined($descr)) {
2618                 $descr =~ s/\s+$//;
2619                 $conf->{description} = $descr;
2620             }
2621             $descr = undef;
2622             $conf = $res->{$section} = {};
2623             next;
2624
2625         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2626             $section = $1;
2627             if (defined($descr)) {
2628                 $descr =~ s/\s+$//;
2629                 $conf->{description} = $descr;
2630             }
2631             $descr = undef;
2632             $conf = $res->{snapshots}->{$section} = {};
2633             next;
2634         }
2635
2636         if ($line =~ m/^\#(.*)\s*$/) {
2637             $descr = '' if !defined($descr);
2638             $descr .= PVE::Tools::decode_text($1) . "\n";
2639             next;
2640         }
2641
2642         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2643             $descr = '' if !defined($descr);
2644             $descr .= PVE::Tools::decode_text($2);
2645         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2646             $conf->{snapstate} = $1;
2647         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2648             my $key = $1;
2649             my $value = $2;
2650             $conf->{$key} = $value;
2651         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2652             my $value = $1;
2653             if ($section eq 'pending') {
2654                 $conf->{delete} = $value; # we parse this later
2655             } else {
2656                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2657             }
2658         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2659             my $key = $1;
2660             my $value = $2;
2661             eval { $value = check_type($key, $value); };
2662             if ($@) {
2663                 warn "vm $vmid - unable to parse value of '$key' - $@";
2664             } else {
2665                 $key = 'ide2' if $key eq 'cdrom';
2666                 my $fmt = $confdesc->{$key}->{format};
2667                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2668                     my $v = parse_drive($key, $value);
2669                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2670                         $v->{file} = $volid;
2671                         $value = print_drive($vmid, $v);
2672                     } else {
2673                         warn "vm $vmid - unable to parse value of '$key'\n";
2674                         next;
2675                     }
2676                 }
2677
2678                 $conf->{$key} = $value;
2679             }
2680         }
2681     }
2682
2683     if (defined($descr)) {
2684         $descr =~ s/\s+$//;
2685         $conf->{description} = $descr;
2686     }
2687     delete $res->{snapstate}; # just to be sure
2688
2689     return $res;
2690 }
2691
2692 sub write_vm_config {
2693     my ($filename, $conf) = @_;
2694
2695     delete $conf->{snapstate}; # just to be sure
2696
2697     if ($conf->{cdrom}) {
2698         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2699         $conf->{ide2} = $conf->{cdrom};
2700         delete $conf->{cdrom};
2701     }
2702
2703     # we do not use 'smp' any longer
2704     if ($conf->{sockets}) {
2705         delete $conf->{smp};
2706     } elsif ($conf->{smp}) {
2707         $conf->{sockets} = $conf->{smp};
2708         delete $conf->{cores};
2709         delete $conf->{smp};
2710     }
2711
2712     my $used_volids = {};
2713
2714     my $cleanup_config = sub {
2715         my ($cref, $pending, $snapname) = @_;
2716
2717         foreach my $key (keys %$cref) {
2718             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2719                 $key eq 'snapstate' || $key eq 'pending';
2720             my $value = $cref->{$key};
2721             if ($key eq 'delete') {
2722                 die "propertry 'delete' is only allowed in [PENDING]\n"
2723                     if !$pending;
2724                 # fixme: check syntax?
2725                 next;
2726             }
2727             eval { $value = check_type($key, $value); };
2728             die "unable to parse value of '$key' - $@" if $@;
2729
2730             $cref->{$key} = $value;
2731
2732             if (!$snapname && is_valid_drivename($key)) {
2733                 my $drive = parse_drive($key, $value);
2734                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2735             }
2736         }
2737     };
2738
2739     &$cleanup_config($conf);
2740
2741     &$cleanup_config($conf->{pending}, 1);
2742
2743     foreach my $snapname (keys %{$conf->{snapshots}}) {
2744         die "internal error" if $snapname eq 'pending';
2745         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2746     }
2747
2748     # remove 'unusedX' settings if we re-add a volume
2749     foreach my $key (keys %$conf) {
2750         my $value = $conf->{$key};
2751         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2752             delete $conf->{$key};
2753         }
2754     }
2755
2756     my $generate_raw_config = sub {
2757         my ($conf, $pending) = @_;
2758
2759         my $raw = '';
2760
2761         # add description as comment to top of file
2762         if (defined(my $descr = $conf->{description})) {
2763             if ($descr) {
2764                 foreach my $cl (split(/\n/, $descr)) {
2765                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2766                 }
2767             } else {
2768                 $raw .= "#\n" if $pending;
2769             }
2770         }
2771
2772         foreach my $key (sort keys %$conf) {
2773             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2774             $raw .= "$key: $conf->{$key}\n";
2775         }
2776         return $raw;
2777     };
2778
2779     my $raw = &$generate_raw_config($conf);
2780
2781     if (scalar(keys %{$conf->{pending}})){
2782         $raw .= "\n[PENDING]\n";
2783         $raw .= &$generate_raw_config($conf->{pending}, 1);
2784     }
2785
2786     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2787         $raw .= "\n[$snapname]\n";
2788         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2789     }
2790
2791     return $raw;
2792 }
2793
2794 sub load_defaults {
2795
2796     my $res = {};
2797
2798     # we use static defaults from our JSON schema configuration
2799     foreach my $key (keys %$confdesc) {
2800         if (defined(my $default = $confdesc->{$key}->{default})) {
2801             $res->{$key} = $default;
2802         }
2803     }
2804
2805     return $res;
2806 }
2807
2808 sub config_list {
2809     my $vmlist = PVE::Cluster::get_vmlist();
2810     my $res = {};
2811     return $res if !$vmlist || !$vmlist->{ids};
2812     my $ids = $vmlist->{ids};
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my @loc_res = ();
2828
2829     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2830     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2831
2832     push @loc_res, "ivshmem" if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2842
2843     return \@loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = PVE::INotify::nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_local_storage_availability {
2900     my ($conf, $storecfg) = @_;
2901
2902     my $nodelist = PVE::Cluster::get_nodelist();
2903     my $nodehash = { map { $_ => {} } @$nodelist };
2904
2905     foreach_drive($conf, sub {
2906         my ($ds, $drive) = @_;
2907
2908         my $volid = $drive->{file};
2909         return if !$volid;
2910
2911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2912         if ($storeid) {
2913             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2914
2915             if ($scfg->{disable}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                 }
2919             } elsif (my $avail = $scfg->{nodes}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     if (!$avail->{$node}) {
2922                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2923                     }
2924                 }
2925             }
2926         }
2927     });
2928
2929     foreach my $node (values %$nodehash) {
2930         if (my $unavail = $node->{unavailable_storages}) {
2931             $node->{unavailable_storages} = [ sort keys %$unavail ];
2932         }
2933     }
2934
2935     return $nodehash
2936 }
2937
2938 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2939 sub check_running {
2940     my ($vmid, $nocheck, $node) = @_;
2941
2942     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2943     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2944 }
2945
2946 sub vzlist {
2947
2948     my $vzlist = config_list();
2949
2950     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2951
2952     while (defined(my $de = $fd->read)) {
2953         next if $de !~ m/^(\d+)\.pid$/;
2954         my $vmid = $1;
2955         next if !defined($vzlist->{$vmid});
2956         if (my $pid = check_running($vmid)) {
2957             $vzlist->{$vmid}->{pid} = $pid;
2958         }
2959     }
2960
2961     return $vzlist;
2962 }
2963
2964 sub disksize {
2965     my ($storecfg, $conf) = @_;
2966
2967     my $bootdisk = $conf->{bootdisk};
2968     return undef if !$bootdisk;
2969     return undef if !is_valid_drivename($bootdisk);
2970
2971     return undef if !$conf->{$bootdisk};
2972
2973     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2974     return undef if !defined($drive);
2975
2976     return undef if drive_is_cdrom($drive);
2977
2978     my $volid = $drive->{file};
2979     return undef if !$volid;
2980
2981     return $drive->{size};
2982 }
2983
2984 our $vmstatus_return_properties = {
2985     vmid => get_standard_option('pve-vmid'),
2986     status => {
2987         description => "Qemu process status.",
2988         type => 'string',
2989         enum => ['stopped', 'running'],
2990     },
2991     maxmem => {
2992         description => "Maximum memory in bytes.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'bytes',
2996     },
2997     maxdisk => {
2998         description => "Root disk size in bytes.",
2999         type => 'integer',
3000         optional => 1,
3001         renderer => 'bytes',
3002     },
3003     name => {
3004         description => "VM name.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     qmpstatus => {
3009         description => "Qemu QMP agent status.",
3010         type => 'string',
3011         optional => 1,
3012     },
3013     pid => {
3014         description => "PID of running qemu process.",
3015         type => 'integer',
3016         optional => 1,
3017     },
3018     uptime => {
3019         description => "Uptime.",
3020         type => 'integer',
3021         optional => 1,
3022         renderer => 'duration',
3023     },
3024     cpus => {
3025         description => "Maximum usable CPUs.",
3026         type => 'number',
3027         optional => 1,
3028     },
3029     lock => {
3030         description => "The current config lock, if any.",
3031         type => 'string',
3032         optional => 1,
3033     }
3034 };
3035
3036 my $last_proc_pid_stat;
3037
3038 # get VM status information
3039 # This must be fast and should not block ($full == false)
3040 # We only query KVM using QMP if $full == true (this can be slow)
3041 sub vmstatus {
3042     my ($opt_vmid, $full) = @_;
3043
3044     my $res = {};
3045
3046     my $storecfg = PVE::Storage::config();
3047
3048     my $list = vzlist();
3049     my $defaults = load_defaults();
3050
3051     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3052
3053     my $cpucount = $cpuinfo->{cpus} || 1;
3054
3055     foreach my $vmid (keys %$list) {
3056         next if $opt_vmid && ($vmid ne $opt_vmid);
3057
3058         my $conf = PVE::QemuConfig->load_config($vmid);
3059
3060         my $d = { vmid => $vmid };
3061         $d->{pid} = $list->{$vmid}->{pid};
3062
3063         # fixme: better status?
3064         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3065
3066         my $size = disksize($storecfg, $conf);
3067         if (defined($size)) {
3068             $d->{disk} = 0; # no info available
3069             $d->{maxdisk} = $size;
3070         } else {
3071             $d->{disk} = 0;
3072             $d->{maxdisk} = 0;
3073         }
3074
3075         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3076             * ($conf->{cores} || $defaults->{cores});
3077         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3078         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3079
3080         $d->{name} = $conf->{name} || "VM $vmid";
3081         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3082             : $defaults->{memory}*(1024*1024);
3083
3084         if ($conf->{balloon}) {
3085             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3086             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3087                 : $defaults->{shares};
3088         }
3089
3090         $d->{uptime} = 0;
3091         $d->{cpu} = 0;
3092         $d->{mem} = 0;
3093
3094         $d->{netout} = 0;
3095         $d->{netin} = 0;
3096
3097         $d->{diskread} = 0;
3098         $d->{diskwrite} = 0;
3099
3100         $d->{template} = PVE::QemuConfig->is_template($conf);
3101
3102         $d->{serial} = 1 if conf_has_serial($conf);
3103         $d->{lock} = $conf->{lock} if $conf->{lock};
3104
3105         $res->{$vmid} = $d;
3106     }
3107
3108     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3109     foreach my $dev (keys %$netdev) {
3110         next if $dev !~ m/^tap([1-9]\d*)i/;
3111         my $vmid = $1;
3112         my $d = $res->{$vmid};
3113         next if !$d;
3114
3115         $d->{netout} += $netdev->{$dev}->{receive};
3116         $d->{netin} += $netdev->{$dev}->{transmit};
3117
3118         if ($full) {
3119             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3120             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3121         }
3122
3123     }
3124
3125     my $ctime = gettimeofday;
3126
3127     foreach my $vmid (keys %$list) {
3128
3129         my $d = $res->{$vmid};
3130         my $pid = $d->{pid};
3131         next if !$pid;
3132
3133         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3134         next if !$pstat; # not running
3135
3136         my $used = $pstat->{utime} + $pstat->{stime};
3137
3138         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3139
3140         if ($pstat->{vsize}) {
3141             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3142         }
3143
3144         my $old = $last_proc_pid_stat->{$pid};
3145         if (!$old) {
3146             $last_proc_pid_stat->{$pid} = {
3147                 time => $ctime,
3148                 used => $used,
3149                 cpu => 0,
3150             };
3151             next;
3152         }
3153
3154         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3155
3156         if ($dtime > 1000) {
3157             my $dutime = $used -  $old->{used};
3158
3159             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3160             $last_proc_pid_stat->{$pid} = {
3161                 time => $ctime,
3162                 used => $used,
3163                 cpu => $d->{cpu},
3164             };
3165         } else {
3166             $d->{cpu} = $old->{cpu};
3167         }
3168     }
3169
3170     return $res if !$full;
3171
3172     my $qmpclient = PVE::QMPClient->new();
3173
3174     my $ballooncb = sub {
3175         my ($vmid, $resp) = @_;
3176
3177         my $info = $resp->{'return'};
3178         return if !$info->{max_mem};
3179
3180         my $d = $res->{$vmid};
3181
3182         # use memory assigned to VM
3183         $d->{maxmem} = $info->{max_mem};
3184         $d->{balloon} = $info->{actual};
3185
3186         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3187             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3188             $d->{freemem} = $info->{free_mem};
3189         }
3190
3191         $d->{ballooninfo} = $info;
3192     };
3193
3194     my $blockstatscb = sub {
3195         my ($vmid, $resp) = @_;
3196         my $data = $resp->{'return'} || [];
3197         my $totalrdbytes = 0;
3198         my $totalwrbytes = 0;
3199
3200         for my $blockstat (@$data) {
3201             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3202             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3203
3204             $blockstat->{device} =~ s/drive-//;
3205             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3206         }
3207         $res->{$vmid}->{diskread} = $totalrdbytes;
3208         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3209     };
3210
3211     my $statuscb = sub {
3212         my ($vmid, $resp) = @_;
3213
3214         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3215         # this fails if ballon driver is not loaded, so this must be
3216         # the last commnand (following command are aborted if this fails).
3217         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3218
3219         my $status = 'unknown';
3220         if (!defined($status = $resp->{'return'}->{status})) {
3221             warn "unable to get VM status\n";
3222             return;
3223         }
3224
3225         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3226     };
3227
3228     foreach my $vmid (keys %$list) {
3229         next if $opt_vmid && ($vmid ne $opt_vmid);
3230         next if !$res->{$vmid}->{pid}; # not running
3231         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3232     }
3233
3234     $qmpclient->queue_execute(undef, 2);
3235
3236     foreach my $vmid (keys %$list) {
3237         next if $opt_vmid && ($vmid ne $opt_vmid);
3238         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3239     }
3240
3241     return $res;
3242 }
3243
3244 sub foreach_drive {
3245     my ($conf, $func, @param) = @_;
3246
3247     foreach my $ds (valid_drive_names()) {
3248         next if !defined($conf->{$ds});
3249
3250         my $drive = parse_drive($ds, $conf->{$ds});
3251         next if !$drive;
3252
3253         &$func($ds, $drive, @param);
3254     }
3255 }
3256
3257 sub foreach_volid {
3258     my ($conf, $func, @param) = @_;
3259
3260     my $volhash = {};
3261
3262     my $test_volid = sub {
3263         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3264
3265         return if !$volid;
3266
3267         $volhash->{$volid}->{cdrom} //= 1;
3268         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3269
3270         $volhash->{$volid}->{replicate} //= 0;
3271         $volhash->{$volid}->{replicate} = 1 if $replicate;
3272
3273         $volhash->{$volid}->{shared} //= 0;
3274         $volhash->{$volid}->{shared} = 1 if $shared;
3275
3276         $volhash->{$volid}->{referenced_in_config} //= 0;
3277         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3278
3279         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3280             if defined($snapname);
3281         $volhash->{$volid}->{size} = $size if $size;
3282     };
3283
3284     foreach_drive($conf, sub {
3285         my ($ds, $drive) = @_;
3286         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3287     });
3288
3289     foreach my $snapname (keys %{$conf->{snapshots}}) {
3290         my $snap = $conf->{snapshots}->{$snapname};
3291         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3292         foreach_drive($snap, sub {
3293             my ($ds, $drive) = @_;
3294             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3295         });
3296     }
3297
3298     foreach my $volid (keys %$volhash) {
3299         &$func($volid, $volhash->{$volid}, @param);
3300     }
3301 }
3302
3303 sub conf_has_serial {
3304     my ($conf) = @_;
3305
3306     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3307         if ($conf->{"serial$i"}) {
3308             return 1;
3309         }
3310     }
3311
3312     return 0;
3313 }
3314
3315 sub conf_has_audio {
3316     my ($conf, $id) = @_;
3317
3318     $id //= 0;
3319     my $audio = $conf->{"audio$id"};
3320     return undef if !defined($audio);
3321
3322     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3323     my $audiodriver = $audioproperties->{driver} // 'spice';
3324
3325     return {
3326         dev => $audioproperties->{device},
3327         dev_id => "audiodev$id",
3328         backend => $audiodriver,
3329         backend_id => "$audiodriver-backend${id}",
3330     };
3331 }
3332
3333 sub vga_conf_has_spice {
3334     my ($vga) = @_;
3335
3336     my $vgaconf = parse_vga($vga);
3337     my $vgatype = $vgaconf->{type};
3338     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3339
3340     return $1 || 1;
3341 }
3342
3343 sub is_native($) {
3344     my ($arch) = @_;
3345     return get_host_arch() eq $arch;
3346 }
3347
3348 my $default_machines = {
3349     x86_64 => 'pc',
3350     aarch64 => 'virt',
3351 };
3352
3353 sub get_basic_machine_info {
3354     my ($conf, $forcemachine) = @_;
3355
3356     my $arch = $conf->{arch} // get_host_arch();
3357     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3358     return ($arch, $machine);
3359 }
3360
3361 sub get_ovmf_files($) {
3362     my ($arch) = @_;
3363
3364     my $ovmf = $OVMF->{$arch}
3365         or die "no OVMF images known for architecture '$arch'\n";
3366
3367     return @$ovmf;
3368 }
3369
3370 my $Arch2Qemu = {
3371     aarch64 => '/usr/bin/qemu-system-aarch64',
3372     x86_64 => '/usr/bin/qemu-system-x86_64',
3373 };
3374 sub get_command_for_arch($) {
3375     my ($arch) = @_;
3376     return '/usr/bin/kvm' if is_native($arch);
3377
3378     my $cmd = $Arch2Qemu->{$arch}
3379         or die "don't know how to emulate architecture '$arch'\n";
3380     return $cmd;
3381 }
3382
3383 sub get_cpu_options {
3384     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3385
3386     my $cpuFlags = [];
3387     my $ostype = $conf->{ostype};
3388
3389     my $cpu = $kvm ? "kvm64" : "qemu64";
3390     if ($arch eq 'aarch64') {
3391         $cpu = 'cortex-a57';
3392     }
3393     my $hv_vendor_id;
3394     if (my $cputype = $conf->{cpu}) {
3395         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3396             or die "Cannot parse cpu description: $cputype\n";
3397         $cpu = $cpuconf->{cputype};
3398         $kvm_off = 1 if $cpuconf->{hidden};
3399         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3400
3401         if (defined(my $flags = $cpuconf->{flags})) {
3402             push @$cpuFlags, split(";", $flags);
3403         }
3404     }
3405
3406     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3407
3408     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3409
3410     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3411
3412     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3413
3414     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3415
3416         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3417         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3418     }
3419
3420     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3421
3422     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3423
3424     push @$cpuFlags, 'kvm=off' if $kvm_off;
3425
3426     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3427         push @$cpuFlags, "vendor=${cpu_vendor}"
3428             if $cpu_vendor ne 'default';
3429     } elsif ($arch ne 'aarch64') {
3430         die "internal error"; # should not happen
3431     }
3432
3433     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3434
3435     return ('-cpu', $cpu);
3436 }
3437
3438 sub config_to_command {
3439     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3440
3441     my $cmd = [];
3442     my $globalFlags = [];
3443     my $machineFlags = [];
3444     my $rtcFlags = [];
3445     my $devices = [];
3446     my $pciaddr = '';
3447     my $bridges = {};
3448     my $vernum = 0; # unknown
3449     my $ostype = $conf->{ostype};
3450     my $winversion = windows_version($ostype);
3451     my $kvm = $conf->{kvm};
3452
3453     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3454     my $kvm_binary = get_command_for_arch($arch);
3455     my $kvmver = kvm_user_version($kvm_binary);
3456     $kvm //= 1 if is_native($arch);
3457
3458     if ($kvm) {
3459         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3460             if !defined kvm_version();
3461     }
3462
3463     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3464         $vernum = $1*1000000+$2*1000;
3465     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3466         $vernum = $1*1000000+$2*1000+$3;
3467     }
3468
3469     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3470
3471     my $q35 = machine_type_is_q35($conf);
3472     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3473     my $use_old_bios_files = undef;
3474     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3475
3476     my $cpuunits = defined($conf->{cpuunits}) ?
3477             $conf->{cpuunits} : $defaults->{cpuunits};
3478
3479     push @$cmd, $kvm_binary;
3480
3481     push @$cmd, '-id', $vmid;
3482
3483     my $vmname = $conf->{name} || "vm$vmid";
3484
3485     push @$cmd, '-name', $vmname;
3486
3487     my $use_virtio = 0;
3488
3489     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3490     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3491     push @$cmd, '-mon', "chardev=qmp,mode=control";
3492
3493     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3494         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3495         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3496     }
3497
3498     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3499
3500     push @$cmd, '-daemonize';
3501
3502     if ($conf->{smbios1}) {
3503         my $smbios_conf = parse_smbios1($conf->{smbios1});
3504         if ($smbios_conf->{base64}) {
3505             # Do not pass base64 flag to qemu
3506             delete $smbios_conf->{base64};
3507             my $smbios_string = "";
3508             foreach my $key (keys %$smbios_conf) {
3509                 my $value;
3510                 if ($key eq "uuid") {
3511                     $value = $smbios_conf->{uuid}
3512                 } else {
3513                     $value = decode_base64($smbios_conf->{$key});
3514                 }
3515                 # qemu accepts any binary data, only commas need escaping by double comma
3516                 $value =~ s/,/,,/g;
3517                 $smbios_string .= "," . $key . "=" . $value if $value;
3518             }
3519             push @$cmd, '-smbios', "type=1" . $smbios_string;
3520         } else {
3521             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3522         }
3523     }
3524
3525     if ($conf->{vmgenid}) {
3526         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3527     }
3528
3529     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3530     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3531         die "uefi base image not found\n" if ! -f $ovmf_code;
3532
3533         my $path;
3534         my $format;
3535         if (my $efidisk = $conf->{efidisk0}) {
3536             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3537             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3538             $format = $d->{format};
3539             if ($storeid) {
3540                 $path = PVE::Storage::path($storecfg, $d->{file});
3541                 if (!defined($format)) {
3542                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3543                     $format = qemu_img_format($scfg, $volname);
3544                 }
3545             } else {
3546                 $path = $d->{file};
3547                 die "efidisk format must be specified\n"
3548                     if !defined($format);
3549             }
3550         } else {
3551             warn "no efidisk configured! Using temporary efivars disk.\n";
3552             $path = "/tmp/$vmid-ovmf.fd";
3553             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3554             $format = 'raw';
3555         }
3556
3557         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3558         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3559     }
3560
3561     # load q35 config
3562     if ($q35) {
3563         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3564         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3565             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3566         } else {
3567             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3568         }
3569     }
3570
3571     # add usb controllers
3572     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3573     push @$devices, @usbcontrollers if @usbcontrollers;
3574     my $vga = parse_vga($conf->{vga});
3575
3576     my $qxlnum = vga_conf_has_spice($conf->{vga});
3577     $vga->{type} = 'qxl' if $qxlnum;
3578
3579     if (!$vga->{type}) {
3580         if ($arch eq 'aarch64') {
3581             $vga->{type} = 'virtio';
3582         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3583             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3584         } else {
3585             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3586         }
3587     }
3588
3589     # enable absolute mouse coordinates (needed by vnc)
3590     my $tablet;
3591     if (defined($conf->{tablet})) {
3592         $tablet = $conf->{tablet};
3593     } else {
3594         $tablet = $defaults->{tablet};
3595         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3596         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3597     }
3598
3599     if ($tablet) {
3600         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3601         my $kbd = print_keyboarddevice_full($conf, $arch);
3602         push @$devices, '-device', $kbd if defined($kbd);
3603     }
3604
3605     my $kvm_off = 0;
3606     my $gpu_passthrough;
3607
3608     # host pci devices
3609     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3610         my $id = "hostpci$i";
3611         my $d = parse_hostpci($conf->{$id});
3612         next if !$d;
3613
3614         if (my $pcie = $d->{pcie}) {
3615             die "q35 machine model is not enabled" if !$q35;
3616             # win7 wants to have the pcie devices directly on the pcie bus
3617             # instead of in the root port
3618             if ($winversion == 7) {
3619                 $pciaddr = print_pcie_addr("${id}bus0");
3620             } else {
3621                 # add more root ports if needed, 4 are present by default
3622                 # by pve-q35 cfgs, rest added here on demand.
3623                 if ($i > 3) {
3624                     push @$devices, '-device', print_pcie_root_port($i);
3625                 }
3626                 $pciaddr = print_pcie_addr($id);
3627             }
3628         } else {
3629             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3630         }
3631
3632         my $xvga = '';
3633         if ($d->{'x-vga'}) {
3634             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3635             $kvm_off = 1;
3636             $vga->{type} = 'none' if !defined($conf->{vga});
3637             $gpu_passthrough = 1;
3638         }
3639
3640         my $pcidevices = $d->{pciid};
3641         my $multifunction = 1 if @$pcidevices > 1;
3642
3643         my $sysfspath;
3644         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3645             my $pci_id = $pcidevices->[0]->{id};
3646             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3647             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3648         } elsif ($d->{mdev}) {
3649             warn "ignoring mediated device '$id' with multifunction device\n";
3650         }
3651
3652         my $j=0;
3653         foreach my $pcidevice (@$pcidevices) {
3654             my $devicestr = "vfio-pci";
3655
3656             if ($sysfspath) {
3657                 $devicestr .= ",sysfsdev=$sysfspath";
3658             } else {
3659                 $devicestr .= ",host=$pcidevice->{id}";
3660             }
3661
3662             my $mf_addr = $multifunction ? ".$j" : '';
3663             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3664
3665             if ($j == 0) {
3666                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3667                 $devicestr .= "$xvga";
3668                 $devicestr .= ",multifunction=on" if $multifunction;
3669                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3670             }
3671
3672             push @$devices, '-device', $devicestr;
3673             $j++;
3674         }
3675     }
3676
3677     # usb devices
3678     my $usb_dev_features = {};
3679     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3680
3681     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3682     push @$devices, @usbdevices if @usbdevices;
3683     # serial devices
3684     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3685         if (my $path = $conf->{"serial$i"}) {
3686             if ($path eq 'socket') {
3687                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3688                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3689                 # On aarch64, serial0 is the UART device. Qemu only allows
3690                 # connecting UART devices via the '-serial' command line, as
3691                 # the device has a fixed slot on the hardware...
3692                 if ($arch eq 'aarch64' && $i == 0) {
3693                     push @$devices, '-serial', "chardev:serial$i";
3694                 } else {
3695                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3696                 }
3697             } else {
3698                 die "no such serial device\n" if ! -c $path;
3699                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3700                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3701             }
3702         }
3703     }
3704
3705     # parallel devices
3706     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3707         if (my $path = $conf->{"parallel$i"}) {
3708             die "no such parallel device\n" if ! -c $path;
3709             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3710             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3711             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3712         }
3713     }
3714
3715     if (my $audio = conf_has_audio($conf)) {
3716
3717         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3718
3719         my $id = $audio->{dev_id};
3720         if ($audio->{dev} eq 'AC97') {
3721             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3722         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3723             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3724             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3725             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3726         } else {
3727             die "unkown audio device '$audio->{dev}', implement me!";
3728         }
3729
3730         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3731     }
3732
3733     my $sockets = 1;
3734     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3735     $sockets = $conf->{sockets} if  $conf->{sockets};
3736
3737     my $cores = $conf->{cores} || 1;
3738
3739     my $maxcpus = $sockets * $cores;
3740
3741     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3742
3743     my $allowed_vcpus = $cpuinfo->{cpus};
3744
3745     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3746         if ($allowed_vcpus < $maxcpus);
3747
3748     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3749
3750         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3751         for (my $i = 2; $i <= $vcpus; $i++)  {
3752             my $cpustr = print_cpu_device($conf,$i);
3753             push @$cmd, '-device', $cpustr;
3754         }
3755
3756     } else {
3757
3758         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3759     }
3760     push @$cmd, '-nodefaults';
3761
3762     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3763
3764     my $bootindex_hash = {};
3765     my $i = 1;
3766     foreach my $o (split(//, $bootorder)) {
3767         $bootindex_hash->{$o} = $i*100;
3768         $i++;
3769     }
3770
3771     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3772
3773     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3774
3775     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3776
3777     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3778         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $kvmver, $machine_type, undef, $qxlnum, $bridges);
3779         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3780         push @$cmd,  '-vnc', "unix:$socket,password";
3781     } else {
3782         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3783         push @$cmd, '-nographic';
3784     }
3785
3786     # time drift fix
3787     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3788
3789     my $useLocaltime = $conf->{localtime};
3790
3791     if ($winversion >= 5) { # windows
3792         $useLocaltime = 1 if !defined($conf->{localtime});
3793
3794         # use time drift fix when acpi is enabled
3795         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3796             $tdf = 1 if !defined($conf->{tdf});
3797         }
3798     }
3799
3800     if ($winversion >= 6) {
3801         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3802         push @$cmd, '-no-hpet';
3803     }
3804
3805     push @$rtcFlags, 'driftfix=slew' if $tdf;
3806
3807     if (!$kvm) {
3808         push @$machineFlags, 'accel=tcg';
3809     }
3810
3811     if ($machine_type) {
3812         push @$machineFlags, "type=${machine_type}";
3813     }
3814
3815     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3816         push @$rtcFlags, "base=$conf->{startdate}";
3817     } elsif ($useLocaltime) {
3818         push @$rtcFlags, 'base=localtime';
3819     }
3820
3821     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3822
3823     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3824
3825     push @$cmd, '-S' if $conf->{freeze};
3826
3827     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3828
3829     my $guest_agent = parse_guest_agent($conf);
3830
3831     if ($guest_agent->{enabled}) {
3832         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3833         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3834
3835         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3836             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3837             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3838             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3839         } elsif ($guest_agent->{type} eq 'isa') {
3840             push @$devices, '-device', "isa-serial,chardev=qga0";
3841         }
3842     }
3843
3844     my $spice_port;
3845
3846     if ($qxlnum) {
3847         if ($qxlnum > 1) {
3848             if ($winversion){
3849                 for(my $i = 1; $i < $qxlnum; $i++){
3850                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $kvmver, $machine_type, $i, $qxlnum, $bridges);
3851                 }
3852             } else {
3853                 # assume other OS works like Linux
3854                 my ($ram, $vram) = ("134217728", "67108864");
3855                 if ($vga->{memory}) {
3856                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3857                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3858                 }
3859                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3860                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3861             }
3862         }
3863
3864         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3865
3866         my $nodename = PVE::INotify::nodename();
3867         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3868         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3869         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3870
3871         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3872         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3873         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3874
3875         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3876         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3877
3878         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3879         if ($spice_enhancement->{foldersharing}) {
3880             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3881             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3882         }
3883
3884         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3885         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3886         push @$devices, '-spice', "$spice_opts";
3887     }
3888
3889     # enable balloon by default, unless explicitly disabled
3890     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3891         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3892         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3893     }
3894
3895     if ($conf->{watchdog}) {
3896         my $wdopts = parse_watchdog($conf->{watchdog});
3897         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3898         my $watchdog = $wdopts->{model} || 'i6300esb';
3899         push @$devices, '-device', "$watchdog$pciaddr";
3900         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3901     }
3902
3903     my $vollist = [];
3904     my $scsicontroller = {};
3905     my $ahcicontroller = {};
3906     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3907
3908     # Add iscsi initiator name if available
3909     if (my $initiator = get_initiator_name()) {
3910         push @$devices, '-iscsi', "initiator-name=$initiator";
3911     }
3912
3913     foreach_drive($conf, sub {
3914         my ($ds, $drive) = @_;
3915
3916         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3917             push @$vollist, $drive->{file};
3918         }
3919
3920         # ignore efidisk here, already added in bios/fw handling code above
3921         return if $drive->{interface} eq 'efidisk';
3922
3923         $use_virtio = 1 if $ds =~ m/^virtio/;
3924
3925         if (drive_is_cdrom ($drive)) {
3926             if ($bootindex_hash->{d}) {
3927                 $drive->{bootindex} = $bootindex_hash->{d};
3928                 $bootindex_hash->{d} += 1;
3929             }
3930         } else {
3931             if ($bootindex_hash->{c}) {
3932                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3933                 $bootindex_hash->{c} += 1;
3934             }
3935         }
3936
3937         if($drive->{interface} eq 'virtio'){
3938            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3939         }
3940
3941         if ($drive->{interface} eq 'scsi') {
3942
3943             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3944
3945             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3946             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3947
3948             my $iothread = '';
3949             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3950                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3951                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3952             } elsif ($drive->{iothread}) {
3953                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3954             }
3955
3956             my $queues = '';
3957             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3958                 $queues = ",num_queues=$drive->{queues}";
3959             }
3960
3961             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3962             $scsicontroller->{$controller}=1;
3963         }
3964
3965         if ($drive->{interface} eq 'sata') {
3966            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3967            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3968            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3969            $ahcicontroller->{$controller}=1;
3970         }
3971
3972         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3973         push @$devices, '-drive',$drive_cmd;
3974         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3975     });
3976
3977     for (my $i = 0; $i < $MAX_NETS; $i++) {
3978          next if !$conf->{"net$i"};
3979          my $d = parse_net($conf->{"net$i"});
3980          next if !$d;
3981
3982          $use_virtio = 1 if $d->{model} eq 'virtio';
3983
3984          if ($bootindex_hash->{n}) {
3985             $d->{bootindex} = $bootindex_hash->{n};
3986             $bootindex_hash->{n} += 1;
3987          }
3988
3989          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3990          push @$devices, '-netdev', $netdevfull;
3991
3992          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3993          push @$devices, '-device', $netdevicefull;
3994     }
3995
3996     if ($conf->{ivshmem}) {
3997         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3998
3999         my $bus;
4000         if ($q35) {
4001             $bus = print_pcie_addr("ivshmem");
4002         } else {
4003             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4004         }
4005
4006         my $ivshmem_name = $ivshmem->{name} // $vmid;
4007         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4008
4009         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4010         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4011     }
4012
4013     if (!$q35) {
4014         # add pci bridges
4015         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4016            $bridges->{1} = 1;
4017            $bridges->{2} = 1;
4018         }
4019
4020         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4021
4022         for my $k (sort {$b cmp $a} keys %$bridges) {
4023             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4024             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4025         }
4026     }
4027
4028     push @$cmd, @$devices;
4029     push @$cmd, '-rtc', join(',', @$rtcFlags)
4030         if scalar(@$rtcFlags);
4031     push @$cmd, '-machine', join(',', @$machineFlags)
4032         if scalar(@$machineFlags);
4033     push @$cmd, '-global', join(',', @$globalFlags)
4034         if scalar(@$globalFlags);
4035
4036     if (my $vmstate = $conf->{vmstate}) {
4037         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4038         push @$vollist, $vmstate;
4039         push @$cmd, '-loadstate', $statepath;
4040     }
4041
4042     # add custom args
4043     if ($conf->{args}) {
4044         my $aa = PVE::Tools::split_args($conf->{args});
4045         push @$cmd, @$aa;
4046     }
4047
4048     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4049 }
4050
4051 sub spice_port {
4052     my ($vmid) = @_;
4053
4054     my $res = mon_cmd($vmid, 'query-spice');
4055
4056     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4057 }
4058
4059 sub vm_devices_list {
4060     my ($vmid) = @_;
4061
4062     my $res = mon_cmd($vmid, 'query-pci');
4063     my $devices_to_check = [];
4064     my $devices = {};
4065     foreach my $pcibus (@$res) {
4066         push @$devices_to_check, @{$pcibus->{devices}},
4067     }
4068
4069     while (@$devices_to_check) {
4070         my $to_check = [];
4071         for my $d (@$devices_to_check) {
4072             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4073             next if !$d->{'pci_bridge'};
4074
4075             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4076             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4077         }
4078         $devices_to_check = $to_check;
4079     }
4080
4081     my $resblock = mon_cmd($vmid, 'query-block');
4082     foreach my $block (@$resblock) {
4083         if($block->{device} =~ m/^drive-(\S+)/){
4084                 $devices->{$1} = 1;
4085         }
4086     }
4087
4088     my $resmice = mon_cmd($vmid, 'query-mice');
4089     foreach my $mice (@$resmice) {
4090         if ($mice->{name} eq 'QEMU HID Tablet') {
4091             $devices->{tablet} = 1;
4092             last;
4093         }
4094     }
4095
4096     # for usb devices there is no query-usb
4097     # but we can iterate over the entries in
4098     # qom-list path=/machine/peripheral
4099     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4100     foreach my $per (@$resperipheral) {
4101         if ($per->{name} =~ m/^usb\d+$/) {
4102             $devices->{$per->{name}} = 1;
4103         }
4104     }
4105
4106     return $devices;
4107 }
4108
4109 sub vm_deviceplug {
4110     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4111
4112     my $q35 = machine_type_is_q35($conf);
4113
4114     my $devices_list = vm_devices_list($vmid);
4115     return 1 if defined($devices_list->{$deviceid});
4116
4117     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4118
4119     if ($deviceid eq 'tablet') {
4120
4121         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4122
4123     } elsif ($deviceid eq 'keyboard') {
4124
4125         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4126
4127     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4128
4129         die "usb hotplug currently not reliable\n";
4130         # since we can't reliably hot unplug all added usb devices
4131         # and usb passthrough disables live migration
4132         # we disable usb hotplugging for now
4133         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4134
4135     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4136
4137         qemu_iothread_add($vmid, $deviceid, $device);
4138
4139         qemu_driveadd($storecfg, $vmid, $device);
4140         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4141
4142         qemu_deviceadd($vmid, $devicefull);
4143         eval { qemu_deviceaddverify($vmid, $deviceid); };
4144         if (my $err = $@) {
4145             eval { qemu_drivedel($vmid, $deviceid); };
4146             warn $@ if $@;
4147             die $err;
4148         }
4149
4150     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4151
4152
4153         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4154         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4155         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4156
4157         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4158
4159         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4160             qemu_iothread_add($vmid, $deviceid, $device);
4161             $devicefull .= ",iothread=iothread-$deviceid";
4162         }
4163
4164         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4165             $devicefull .= ",num_queues=$device->{queues}";
4166         }
4167
4168         qemu_deviceadd($vmid, $devicefull);
4169         qemu_deviceaddverify($vmid, $deviceid);
4170
4171     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4172
4173         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4174         qemu_driveadd($storecfg, $vmid, $device);
4175
4176         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4177         eval { qemu_deviceadd($vmid, $devicefull); };
4178         if (my $err = $@) {
4179             eval { qemu_drivedel($vmid, $deviceid); };
4180             warn $@ if $@;
4181             die $err;
4182         }
4183
4184     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4185
4186         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4187
4188         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4189         my $use_old_bios_files = undef;
4190         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4191
4192         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4193         qemu_deviceadd($vmid, $netdevicefull);
4194         eval {
4195             qemu_deviceaddverify($vmid, $deviceid);
4196             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4197         };
4198         if (my $err = $@) {
4199             eval { qemu_netdevdel($vmid, $deviceid); };
4200             warn $@ if $@;
4201             die $err;
4202         }
4203
4204     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4205
4206         my $bridgeid = $2;
4207         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4208         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4209
4210         qemu_deviceadd($vmid, $devicefull);
4211         qemu_deviceaddverify($vmid, $deviceid);
4212
4213     } else {
4214         die "can't hotplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 # fixme: this should raise exceptions on error!
4221 sub vm_deviceunplug {
4222     my ($vmid, $conf, $deviceid) = @_;
4223
4224     my $devices_list = vm_devices_list($vmid);
4225     return 1 if !defined($devices_list->{$deviceid});
4226
4227     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4228
4229     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4230
4231         qemu_devicedel($vmid, $deviceid);
4232
4233     } elsif ($deviceid =~ m/^usb\d+$/) {
4234
4235         die "usb hotplug currently not reliable\n";
4236         # when unplugging usb devices this way,
4237         # there may be remaining usb controllers/hubs
4238         # so we disable it for now
4239         qemu_devicedel($vmid, $deviceid);
4240         qemu_devicedelverify($vmid, $deviceid);
4241
4242     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4243
4244         qemu_devicedel($vmid, $deviceid);
4245         qemu_devicedelverify($vmid, $deviceid);
4246         qemu_drivedel($vmid, $deviceid);
4247         qemu_iothread_del($conf, $vmid, $deviceid);
4248
4249     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_devicedelverify($vmid, $deviceid);
4253         qemu_iothread_del($conf, $vmid, $deviceid);
4254
4255     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4256
4257         qemu_devicedel($vmid, $deviceid);
4258         qemu_drivedel($vmid, $deviceid);
4259         qemu_deletescsihw($conf, $vmid, $deviceid);
4260
4261     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4262
4263         qemu_devicedel($vmid, $deviceid);
4264         qemu_devicedelverify($vmid, $deviceid);
4265         qemu_netdevdel($vmid, $deviceid);
4266
4267     } else {
4268         die "can't unplug device '$deviceid'\n";
4269     }
4270
4271     return 1;
4272 }
4273
4274 sub qemu_deviceadd {
4275     my ($vmid, $devicefull) = @_;
4276
4277     $devicefull = "driver=".$devicefull;
4278     my %options =  split(/[=,]/, $devicefull);
4279
4280     mon_cmd($vmid, "device_add" , %options);
4281 }
4282
4283 sub qemu_devicedel {
4284     my ($vmid, $deviceid) = @_;
4285
4286     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4287 }
4288
4289 sub qemu_iothread_add {
4290     my($vmid, $deviceid, $device) = @_;
4291
4292     if ($device->{iothread}) {
4293         my $iothreads = vm_iothreads_list($vmid);
4294         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4295     }
4296 }
4297
4298 sub qemu_iothread_del {
4299     my($conf, $vmid, $deviceid) = @_;
4300
4301     my $confid = $deviceid;
4302     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4303         $confid = 'scsi' . $1;
4304     }
4305     my $device = parse_drive($confid, $conf->{$confid});
4306     if ($device->{iothread}) {
4307         my $iothreads = vm_iothreads_list($vmid);
4308         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4309     }
4310 }
4311
4312 sub qemu_objectadd {
4313     my($vmid, $objectid, $qomtype) = @_;
4314
4315     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4316
4317     return 1;
4318 }
4319
4320 sub qemu_objectdel {
4321     my($vmid, $objectid) = @_;
4322
4323     mon_cmd($vmid, "object-del", id => $objectid);
4324
4325     return 1;
4326 }
4327
4328 sub qemu_driveadd {
4329     my ($storecfg, $vmid, $device) = @_;
4330
4331     my $drive = print_drive_full($storecfg, $vmid, $device);
4332     $drive =~ s/\\/\\\\/g;
4333     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4334
4335     # If the command succeeds qemu prints: "OK"
4336     return 1 if $ret =~ m/OK/s;
4337
4338     die "adding drive failed: $ret\n";
4339 }
4340
4341 sub qemu_drivedel {
4342     my($vmid, $deviceid) = @_;
4343
4344     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4345     $ret =~ s/^\s+//;
4346
4347     return 1 if $ret eq "";
4348
4349     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4350     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4351
4352     die "deleting drive $deviceid failed : $ret\n";
4353 }
4354
4355 sub qemu_deviceaddverify {
4356     my ($vmid, $deviceid) = @_;
4357
4358     for (my $i = 0; $i <= 5; $i++) {
4359          my $devices_list = vm_devices_list($vmid);
4360          return 1 if defined($devices_list->{$deviceid});
4361          sleep 1;
4362     }
4363
4364     die "error on hotplug device '$deviceid'\n";
4365 }
4366
4367
4368 sub qemu_devicedelverify {
4369     my ($vmid, $deviceid) = @_;
4370
4371     # need to verify that the device is correctly removed as device_del
4372     # is async and empty return is not reliable
4373
4374     for (my $i = 0; $i <= 5; $i++) {
4375          my $devices_list = vm_devices_list($vmid);
4376          return 1 if !defined($devices_list->{$deviceid});
4377          sleep 1;
4378     }
4379
4380     die "error on hot-unplugging device '$deviceid'\n";
4381 }
4382
4383 sub qemu_findorcreatescsihw {
4384     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4385
4386     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4387
4388     my $scsihwid="$controller_prefix$controller";
4389     my $devices_list = vm_devices_list($vmid);
4390
4391     if(!defined($devices_list->{$scsihwid})) {
4392         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4393     }
4394
4395     return 1;
4396 }
4397
4398 sub qemu_deletescsihw {
4399     my ($conf, $vmid, $opt) = @_;
4400
4401     my $device = parse_drive($opt, $conf->{$opt});
4402
4403     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4404         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4405         return 1;
4406     }
4407
4408     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4409
4410     my $devices_list = vm_devices_list($vmid);
4411     foreach my $opt (keys %{$devices_list}) {
4412         if (PVE::QemuServer::is_valid_drivename($opt)) {
4413             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4414             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4415                 return 1;
4416             }
4417         }
4418     }
4419
4420     my $scsihwid="scsihw$controller";
4421
4422     vm_deviceunplug($vmid, $conf, $scsihwid);
4423
4424     return 1;
4425 }
4426
4427 sub qemu_add_pci_bridge {
4428     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4429
4430     my $bridges = {};
4431
4432     my $bridgeid;
4433
4434     print_pci_addr($device, $bridges, $arch, $machine_type);
4435
4436     while (my ($k, $v) = each %$bridges) {
4437         $bridgeid = $k;
4438     }
4439     return 1 if !defined($bridgeid) || $bridgeid < 1;
4440
4441     my $bridge = "pci.$bridgeid";
4442     my $devices_list = vm_devices_list($vmid);
4443
4444     if (!defined($devices_list->{$bridge})) {
4445         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4446     }
4447
4448     return 1;
4449 }
4450
4451 sub qemu_set_link_status {
4452     my ($vmid, $device, $up) = @_;
4453
4454     mon_cmd($vmid, "set_link", name => $device,
4455                up => $up ? JSON::true : JSON::false);
4456 }
4457
4458 sub qemu_netdevadd {
4459     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4460
4461     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4462     my %options =  split(/[=,]/, $netdev);
4463
4464     mon_cmd($vmid, "netdev_add",  %options);
4465     return 1;
4466 }
4467
4468 sub qemu_netdevdel {
4469     my ($vmid, $deviceid) = @_;
4470
4471     mon_cmd($vmid, "netdev_del", id => $deviceid);
4472 }
4473
4474 sub qemu_usb_hotplug {
4475     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4476
4477     return if !$device;
4478
4479     # remove the old one first
4480     vm_deviceunplug($vmid, $conf, $deviceid);
4481
4482     # check if xhci controller is necessary and available
4483     if ($device->{usb3}) {
4484
4485         my $devicelist = vm_devices_list($vmid);
4486
4487         if (!$devicelist->{xhci}) {
4488             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4489             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4490         }
4491     }
4492     my $d = parse_usb_device($device->{host});
4493     $d->{usb3} = $device->{usb3};
4494
4495     # add the new one
4496     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4497 }
4498
4499 sub qemu_cpu_hotplug {
4500     my ($vmid, $conf, $vcpus) = @_;
4501
4502     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4503
4504     my $sockets = 1;
4505     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4506     $sockets = $conf->{sockets} if  $conf->{sockets};
4507     my $cores = $conf->{cores} || 1;
4508     my $maxcpus = $sockets * $cores;
4509
4510     $vcpus = $maxcpus if !$vcpus;
4511
4512     die "you can't add more vcpus than maxcpus\n"
4513         if $vcpus > $maxcpus;
4514
4515     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4516
4517     if ($vcpus < $currentvcpus) {
4518
4519         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4520
4521             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4522                 qemu_devicedel($vmid, "cpu$i");
4523                 my $retry = 0;
4524                 my $currentrunningvcpus = undef;
4525                 while (1) {
4526                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4527                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4528                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4529                     $retry++;
4530                     sleep 1;
4531                 }
4532                 #update conf after each succesfull cpu unplug
4533                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4534                 PVE::QemuConfig->write_config($vmid, $conf);
4535             }
4536         } else {
4537             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4538         }
4539
4540         return;
4541     }
4542
4543     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4544     die "vcpus in running vm does not match its configuration\n"
4545         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4546
4547     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4548
4549         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4550             my $cpustr = print_cpu_device($conf, $i);
4551             qemu_deviceadd($vmid, $cpustr);
4552
4553             my $retry = 0;
4554             my $currentrunningvcpus = undef;
4555             while (1) {
4556                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4557                 last if scalar(@{$currentrunningvcpus}) == $i;
4558                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4559                 sleep 1;
4560                 $retry++;
4561             }
4562             #update conf after each succesfull cpu hotplug
4563             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4564             PVE::QemuConfig->write_config($vmid, $conf);
4565         }
4566     } else {
4567
4568         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4569             mon_cmd($vmid, "cpu-add", id => int($i));
4570         }
4571     }
4572 }
4573
4574 sub qemu_block_set_io_throttle {
4575     my ($vmid, $deviceid,
4576         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4577         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4578         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4579         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4580
4581     return if !check_running($vmid) ;
4582
4583     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4584         bps => int($bps),
4585         bps_rd => int($bps_rd),
4586         bps_wr => int($bps_wr),
4587         iops => int($iops),
4588         iops_rd => int($iops_rd),
4589         iops_wr => int($iops_wr),
4590         bps_max => int($bps_max),
4591         bps_rd_max => int($bps_rd_max),
4592         bps_wr_max => int($bps_wr_max),
4593         iops_max => int($iops_max),
4594         iops_rd_max => int($iops_rd_max),
4595         iops_wr_max => int($iops_wr_max),
4596         bps_max_length => int($bps_max_length),
4597         bps_rd_max_length => int($bps_rd_max_length),
4598         bps_wr_max_length => int($bps_wr_max_length),
4599         iops_max_length => int($iops_max_length),
4600         iops_rd_max_length => int($iops_rd_max_length),
4601         iops_wr_max_length => int($iops_wr_max_length),
4602     );
4603
4604 }
4605
4606 # old code, only used to shutdown old VM after update
4607 sub __read_avail {
4608     my ($fh, $timeout) = @_;
4609
4610     my $sel = new IO::Select;
4611     $sel->add($fh);
4612
4613     my $res = '';
4614     my $buf;
4615
4616     my @ready;
4617     while (scalar (@ready = $sel->can_read($timeout))) {
4618         my $count;
4619         if ($count = $fh->sysread($buf, 8192)) {
4620             if ($buf =~ /^(.*)\(qemu\) $/s) {
4621                 $res .= $1;
4622                 last;
4623             } else {
4624                 $res .= $buf;
4625             }
4626         } else {
4627             if (!defined($count)) {
4628                 die "$!\n";
4629             }
4630             last;
4631         }
4632     }
4633
4634     die "monitor read timeout\n" if !scalar(@ready);
4635
4636     return $res;
4637 }
4638
4639 sub qemu_block_resize {
4640     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4641
4642     my $running = check_running($vmid);
4643
4644     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4645
4646     return if !$running;
4647
4648     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4649
4650 }
4651
4652 sub qemu_volume_snapshot {
4653     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4654
4655     my $running = check_running($vmid);
4656
4657     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4658         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4659     } else {
4660         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4661     }
4662 }
4663
4664 sub qemu_volume_snapshot_delete {
4665     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4666
4667     my $running = check_running($vmid);
4668
4669     if($running) {
4670
4671         $running = undef;
4672         my $conf = PVE::QemuConfig->load_config($vmid);
4673         foreach_drive($conf, sub {
4674             my ($ds, $drive) = @_;
4675             $running = 1 if $drive->{file} eq $volid;
4676         });
4677     }
4678
4679     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4680         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4681     } else {
4682         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4683     }
4684 }
4685
4686 sub set_migration_caps {
4687     my ($vmid) = @_;
4688
4689     my $cap_ref = [];
4690
4691     my $enabled_cap = {
4692         "auto-converge" => 1,
4693         "xbzrle" => 1,
4694         "x-rdma-pin-all" => 0,
4695         "zero-blocks" => 0,
4696         "compress" => 0
4697     };
4698
4699     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4700
4701     for my $supported_capability (@$supported_capabilities) {
4702         push @$cap_ref, {
4703             capability => $supported_capability->{capability},
4704             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4705         };
4706     }
4707
4708     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4709 }
4710
4711 my $fast_plug_option = {
4712     'lock' => 1,
4713     'name' => 1,
4714     'onboot' => 1,
4715     'shares' => 1,
4716     'startup' => 1,
4717     'description' => 1,
4718     'protection' => 1,
4719     'vmstatestorage' => 1,
4720     'hookscript' => 1,
4721 };
4722
4723 # hotplug changes in [PENDING]
4724 # $selection hash can be used to only apply specified options, for
4725 # example: { cores => 1 } (only apply changed 'cores')
4726 # $errors ref is used to return error messages
4727 sub vmconfig_hotplug_pending {
4728     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4729
4730     my $defaults = load_defaults();
4731     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4732
4733     # commit values which do not have any impact on running VM first
4734     # Note: those option cannot raise errors, we we do not care about
4735     # $selection and always apply them.
4736
4737     my $add_error = sub {
4738         my ($opt, $msg) = @_;
4739         $errors->{$opt} = "hotplug problem - $msg";
4740     };
4741
4742     my $changes = 0;
4743     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4744         if ($fast_plug_option->{$opt}) {
4745             $conf->{$opt} = $conf->{pending}->{$opt};
4746             delete $conf->{pending}->{$opt};
4747             $changes = 1;
4748         }
4749     }
4750
4751     if ($changes) {
4752         PVE::QemuConfig->write_config($vmid, $conf);
4753         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4754     }
4755
4756     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4757
4758     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4759     foreach my $opt (sort keys %$pending_delete_hash) {
4760         next if $selection && !$selection->{$opt};
4761         my $force = $pending_delete_hash->{$opt}->{force};
4762         eval {
4763             if ($opt eq 'hotplug') {
4764                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4765             } elsif ($opt eq 'tablet') {
4766                 die "skip\n" if !$hotplug_features->{usb};
4767                 if ($defaults->{tablet}) {
4768                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4769                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4770                         if $arch eq 'aarch64';
4771                 } else {
4772                     vm_deviceunplug($vmid, $conf, 'tablet');
4773                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4774                 }
4775             } elsif ($opt =~ m/^usb\d+/) {
4776                 die "skip\n";
4777                 # since we cannot reliably hot unplug usb devices
4778                 # we are disabling it
4779                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4780                 vm_deviceunplug($vmid, $conf, $opt);
4781             } elsif ($opt eq 'vcpus') {
4782                 die "skip\n" if !$hotplug_features->{cpu};
4783                 qemu_cpu_hotplug($vmid, $conf, undef);
4784             } elsif ($opt eq 'balloon') {
4785                 # enable balloon device is not hotpluggable
4786                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4787                 # here we reset the ballooning value to memory
4788                 my $balloon = $conf->{memory} || $defaults->{memory};
4789                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4790             } elsif ($fast_plug_option->{$opt}) {
4791                 # do nothing
4792             } elsif ($opt =~ m/^net(\d+)$/) {
4793                 die "skip\n" if !$hotplug_features->{network};
4794                 vm_deviceunplug($vmid, $conf, $opt);
4795             } elsif (is_valid_drivename($opt)) {
4796                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4797                 vm_deviceunplug($vmid, $conf, $opt);
4798                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4799             } elsif ($opt =~ m/^memory$/) {
4800                 die "skip\n" if !$hotplug_features->{memory};
4801                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4802             } elsif ($opt eq 'cpuunits') {
4803                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4804             } elsif ($opt eq 'cpulimit') {
4805                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4806             } else {
4807                 die "skip\n";
4808             }
4809         };
4810         if (my $err = $@) {
4811             &$add_error($opt, $err) if $err ne "skip\n";
4812         } else {
4813             # save new config if hotplug was successful
4814             delete $conf->{$opt};
4815             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4816             PVE::QemuConfig->write_config($vmid, $conf);
4817             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4818         }
4819     }
4820
4821     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4822     $apply_pending_cloudinit = sub {
4823         return if $apply_pending_cloudinit_done; # once is enough
4824         $apply_pending_cloudinit_done = 1; # once is enough
4825
4826         my ($key, $value) = @_;
4827
4828         my @cloudinit_opts = keys %$confdesc_cloudinit;
4829         foreach my $opt (keys %{$conf->{pending}}) {
4830             next if !grep { $_ eq $opt } @cloudinit_opts;
4831             $conf->{$opt} = delete $conf->{pending}->{$opt};
4832         }
4833
4834         my $new_conf = { %$conf };
4835         $new_conf->{$key} = $value;
4836         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4837     };
4838
4839     foreach my $opt (keys %{$conf->{pending}}) {
4840         next if $selection && !$selection->{$opt};
4841         my $value = $conf->{pending}->{$opt};
4842         eval {
4843             if ($opt eq 'hotplug') {
4844                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4845             } elsif ($opt eq 'tablet') {
4846                 die "skip\n" if !$hotplug_features->{usb};
4847                 if ($value == 1) {
4848                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4849                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4850                         if $arch eq 'aarch64';
4851                 } elsif ($value == 0) {
4852                     vm_deviceunplug($vmid, $conf, 'tablet');
4853                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4854                 }
4855             } elsif ($opt =~ m/^usb\d+$/) {
4856                 die "skip\n";
4857                 # since we cannot reliably hot unplug usb devices
4858                 # we are disabling it
4859                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4860                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4861                 die "skip\n" if !$d;
4862                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4863             } elsif ($opt eq 'vcpus') {
4864                 die "skip\n" if !$hotplug_features->{cpu};
4865                 qemu_cpu_hotplug($vmid, $conf, $value);
4866             } elsif ($opt eq 'balloon') {
4867                 # enable/disable balloning device is not hotpluggable
4868                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4869                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4870                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4871
4872                 # allow manual ballooning if shares is set to zero
4873                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4874                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4875                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4876                 }
4877             } elsif ($opt =~ m/^net(\d+)$/) {
4878                 # some changes can be done without hotplug
4879                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4880                                     $vmid, $opt, $value, $arch, $machine_type);
4881             } elsif (is_valid_drivename($opt)) {
4882                 # some changes can be done without hotplug
4883                 my $drive = parse_drive($opt, $value);
4884                 if (drive_is_cloudinit($drive)) {
4885                     &$apply_pending_cloudinit($opt, $value);
4886                 }
4887                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4888                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4889             } elsif ($opt =~ m/^memory$/) { #dimms
4890                 die "skip\n" if !$hotplug_features->{memory};
4891                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4892             } elsif ($opt eq 'cpuunits') {
4893                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4894             } elsif ($opt eq 'cpulimit') {
4895                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4896                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4897             } else {
4898                 die "skip\n";  # skip non-hot-pluggable options
4899             }
4900         };
4901         if (my $err = $@) {
4902             &$add_error($opt, $err) if $err ne "skip\n";
4903         } else {
4904             # save new config if hotplug was successful
4905             $conf->{$opt} = $value;
4906             delete $conf->{pending}->{$opt};
4907             PVE::QemuConfig->write_config($vmid, $conf);
4908             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4909         }
4910     }
4911 }
4912
4913 sub try_deallocate_drive {
4914     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4915
4916     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4917         my $volid = $drive->{file};
4918         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4919             my $sid = PVE::Storage::parse_volume_id($volid);
4920             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4921
4922             # check if the disk is really unused
4923             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4924                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4925             PVE::Storage::vdisk_free($storecfg, $volid);
4926             return 1;
4927         } else {
4928             # If vm is not owner of this disk remove from config
4929             return 1;
4930         }
4931     }
4932
4933     return undef;
4934 }
4935
4936 sub vmconfig_delete_or_detach_drive {
4937     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4938
4939     my $drive = parse_drive($opt, $conf->{$opt});
4940
4941     my $rpcenv = PVE::RPCEnvironment::get();
4942     my $authuser = $rpcenv->get_user();
4943
4944     if ($force) {
4945         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4946         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4947     } else {
4948         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4949     }
4950 }
4951
4952
4953
4954 sub vmconfig_apply_pending {
4955     my ($vmid, $conf, $storecfg) = @_;
4956
4957     # cold plug
4958
4959     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4960     foreach my $opt (sort keys %$pending_delete_hash) {
4961         die "internal error" if $opt =~ m/^unused/;
4962         my $force = $pending_delete_hash->{$opt}->{force};
4963         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4964         if (!defined($conf->{$opt})) {
4965             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4966             PVE::QemuConfig->write_config($vmid, $conf);
4967         } elsif (is_valid_drivename($opt)) {
4968             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4969             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4970             delete $conf->{$opt};
4971             PVE::QemuConfig->write_config($vmid, $conf);
4972         } else {
4973             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4974             delete $conf->{$opt};
4975             PVE::QemuConfig->write_config($vmid, $conf);
4976         }
4977     }
4978
4979     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4980
4981     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4982         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4983
4984         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4985             # skip if nothing changed
4986         } elsif (is_valid_drivename($opt)) {
4987             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4988                 if defined($conf->{$opt});
4989             $conf->{$opt} = $conf->{pending}->{$opt};
4990         } else {
4991             $conf->{$opt} = $conf->{pending}->{$opt};
4992         }
4993
4994         delete $conf->{pending}->{$opt};
4995         PVE::QemuConfig->write_config($vmid, $conf);
4996     }
4997 }
4998
4999 my $safe_num_ne = sub {
5000     my ($a, $b) = @_;
5001
5002     return 0 if !defined($a) && !defined($b);
5003     return 1 if !defined($a);
5004     return 1 if !defined($b);
5005
5006     return $a != $b;
5007 };
5008
5009 my $safe_string_ne = sub {
5010     my ($a, $b) = @_;
5011
5012     return 0 if !defined($a) && !defined($b);
5013     return 1 if !defined($a);
5014     return 1 if !defined($b);
5015
5016     return $a ne $b;
5017 };
5018
5019 sub vmconfig_update_net {
5020     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5021
5022     my $newnet = parse_net($value);
5023
5024     if ($conf->{$opt}) {
5025         my $oldnet = parse_net($conf->{$opt});
5026
5027         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5028             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5029             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5030             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5031
5032             # for non online change, we try to hot-unplug
5033             die "skip\n" if !$hotplug;
5034             vm_deviceunplug($vmid, $conf, $opt);
5035         } else {
5036
5037             die "internal error" if $opt !~ m/net(\d+)/;
5038             my $iface = "tap${vmid}i$1";
5039
5040             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5041                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5042                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5043                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5044                 PVE::Network::tap_unplug($iface);
5045                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5046             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5047                 # Rate can be applied on its own but any change above needs to
5048                 # include the rate in tap_plug since OVS resets everything.
5049                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5050             }
5051
5052             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5053                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5054             }
5055
5056             return 1;
5057         }
5058     }
5059
5060     if ($hotplug) {
5061         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5062     } else {
5063         die "skip\n";
5064     }
5065 }
5066
5067 sub vmconfig_update_disk {
5068     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5069
5070     # fixme: do we need force?
5071
5072     my $drive = parse_drive($opt, $value);
5073
5074     if ($conf->{$opt}) {
5075
5076         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5077
5078             my $media = $drive->{media} || 'disk';
5079             my $oldmedia = $old_drive->{media} || 'disk';
5080             die "unable to change media type\n" if $media ne $oldmedia;
5081
5082             if (!drive_is_cdrom($old_drive)) {
5083
5084                 if ($drive->{file} ne $old_drive->{file}) {
5085
5086                     die "skip\n" if !$hotplug;
5087
5088                     # unplug and register as unused
5089                     vm_deviceunplug($vmid, $conf, $opt);
5090                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5091
5092                 } else {
5093                     # update existing disk
5094
5095                     # skip non hotpluggable value
5096                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5097                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5098                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5099                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5100                         die "skip\n";
5101                     }
5102
5103                     # apply throttle
5104                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5105                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5106                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5107                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5108                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5109                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5110                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5111                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5112                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5113                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5114                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5115                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5116                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5117                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5118                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5119                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5120                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5121                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5122
5123                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5124                                                    ($drive->{mbps} || 0)*1024*1024,
5125                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5126                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5127                                                    $drive->{iops} || 0,
5128                                                    $drive->{iops_rd} || 0,
5129                                                    $drive->{iops_wr} || 0,
5130                                                    ($drive->{mbps_max} || 0)*1024*1024,
5131                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5132                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5133                                                    $drive->{iops_max} || 0,
5134                                                    $drive->{iops_rd_max} || 0,
5135                                                    $drive->{iops_wr_max} || 0,
5136                                                    $drive->{bps_max_length} || 1,
5137                                                    $drive->{bps_rd_max_length} || 1,
5138                                                    $drive->{bps_wr_max_length} || 1,
5139                                                    $drive->{iops_max_length} || 1,
5140                                                    $drive->{iops_rd_max_length} || 1,
5141                                                    $drive->{iops_wr_max_length} || 1);
5142
5143                     }
5144
5145                     return 1;
5146                 }
5147
5148             } else { # cdrom
5149
5150                 if ($drive->{file} eq 'none') {
5151                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5152                     if (drive_is_cloudinit($old_drive)) {
5153                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5154                     }
5155                 } else {
5156                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5157                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5158                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5159                 }
5160
5161                 return 1;
5162             }
5163         }
5164     }
5165
5166     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5167     # hotplug new disks
5168     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5169     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5170 }
5171
5172 sub vm_start {
5173     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5174         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5175
5176     PVE::QemuConfig->lock_config($vmid, sub {
5177         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5178
5179         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5180
5181         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5182
5183         PVE::QemuConfig->check_lock($conf)
5184             if !($skiplock || $is_suspended);
5185
5186         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5187
5188         # clean up leftover reboot request files
5189         eval { clear_reboot_request($vmid); };
5190         warn $@ if $@;
5191
5192         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5193             vmconfig_apply_pending($vmid, $conf, $storecfg);
5194             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5195         }
5196
5197         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5198
5199         my $defaults = load_defaults();
5200
5201         # set environment variable useful inside network script
5202         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5203
5204         my $local_volumes = {};
5205
5206         if ($targetstorage) {
5207             foreach_drive($conf, sub {
5208                 my ($ds, $drive) = @_;
5209
5210                 return if drive_is_cdrom($drive);
5211
5212                 my $volid = $drive->{file};
5213
5214                 return if !$volid;
5215
5216                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5217
5218                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5219                 return if $scfg->{shared};
5220                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5221             });
5222
5223             my $format = undef;
5224
5225             foreach my $opt (sort keys %$local_volumes) {
5226
5227                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5228                 my $drive = parse_drive($opt, $conf->{$opt});
5229
5230                 #if remote storage is specified, use default format
5231                 if ($targetstorage && $targetstorage ne "1") {
5232                     $storeid = $targetstorage;
5233                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5234                     $format = $defFormat;
5235                 } else {
5236                     #else we use same format than original
5237                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5238                     $format = qemu_img_format($scfg, $volid);
5239                 }
5240
5241                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5242                 my $newdrive = $drive;
5243                 $newdrive->{format} = $format;
5244                 $newdrive->{file} = $newvolid;
5245                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5246                 $local_volumes->{$opt} = $drivestr;
5247                 #pass drive to conf for command line
5248                 $conf->{$opt} = $drivestr;
5249             }
5250         }
5251
5252         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5253
5254         if ($is_suspended) {
5255             # enforce machine type on suspended vm to ensure HW compatibility
5256             $forcemachine = $conf->{runningmachine};
5257             print "Resuming suspended VM\n";
5258         }
5259
5260         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5261
5262         my $migration_ip;
5263         my $get_migration_ip = sub {
5264             my ($cidr, $nodename) = @_;
5265
5266             return $migration_ip if defined($migration_ip);
5267
5268             if (!defined($cidr)) {
5269                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5270                 $cidr = $dc_conf->{migration}->{network};
5271             }
5272
5273             if (defined($cidr)) {
5274                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5275
5276                 die "could not get IP: no address configured on local " .
5277                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5278
5279                 die "could not get IP: multiple addresses configured on local " .
5280                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5281
5282                 $migration_ip = @$ips[0];
5283             }
5284
5285             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5286                 if !defined($migration_ip);
5287
5288             return $migration_ip;
5289         };
5290
5291         my $migrate_uri;
5292         if ($statefile) {
5293             if ($statefile eq 'tcp') {
5294                 my $localip = "localhost";
5295                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5296                 my $nodename = PVE::INotify::nodename();
5297
5298                 if (!defined($migration_type)) {
5299                     if (defined($datacenterconf->{migration}->{type})) {
5300                         $migration_type = $datacenterconf->{migration}->{type};
5301                     } else {
5302                         $migration_type = 'secure';
5303                     }
5304                 }
5305
5306                 if ($migration_type eq 'insecure') {
5307                     $localip = $get_migration_ip->($migration_network, $nodename);
5308                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5309                 }
5310
5311                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5312                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5313                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5314                 push @$cmd, '-incoming', $migrate_uri;
5315                 push @$cmd, '-S';
5316
5317             } elsif ($statefile eq 'unix') {
5318                 # should be default for secure migrations as a ssh TCP forward
5319                 # tunnel is not deterministic reliable ready and fails regurarly
5320                 # to set up in time, so use UNIX socket forwards
5321                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5322                 unlink $socket_addr;
5323
5324                 $migrate_uri = "unix:$socket_addr";
5325
5326                 push @$cmd, '-incoming', $migrate_uri;
5327                 push @$cmd, '-S';
5328
5329             } elsif (-e $statefile) {
5330                 push @$cmd, '-loadstate', $statefile;
5331             } else {
5332                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5333                 push @$vollist, $statefile;
5334                 push @$cmd, '-loadstate', $statepath;
5335             }
5336         } elsif ($paused) {
5337             push @$cmd, '-S';
5338         }
5339
5340         # host pci devices
5341         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5342           my $d = parse_hostpci($conf->{"hostpci$i"});
5343           next if !$d;
5344           my $pcidevices = $d->{pciid};
5345           foreach my $pcidevice (@$pcidevices) {
5346                 my $pciid = $pcidevice->{id};
5347
5348                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5349                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5350                 die "no pci device info for device '$pciid'\n" if !$info;
5351
5352                 if ($d->{mdev}) {
5353                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5354                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5355                 } else {
5356                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5357                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5358                     die "can't reset pci device '$pciid'\n"
5359                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5360                 }
5361           }
5362         }
5363
5364         PVE::Storage::activate_volumes($storecfg, $vollist);
5365
5366         eval {
5367             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5368                 outfunc => sub {}, errfunc => sub {});
5369         };
5370         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5371         # timeout should be more than enough here...
5372         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5373
5374         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5375                                                   : $defaults->{cpuunits};
5376
5377         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5378         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5379
5380         my %properties = (
5381             Slice => 'qemu.slice',
5382             KillMode => 'none',
5383             CPUShares => $cpuunits
5384         );
5385
5386         if (my $cpulimit = $conf->{cpulimit}) {
5387             $properties{CPUQuota} = int($cpulimit * 100);
5388         }
5389         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5390
5391         my $run_qemu = sub {
5392             PVE::Tools::run_fork sub {
5393                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5394                 run_command($cmd, %run_params);
5395             };
5396         };
5397
5398         if ($conf->{hugepages}) {
5399
5400             my $code = sub {
5401                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5402                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5403
5404                 PVE::QemuServer::Memory::hugepages_mount();
5405                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5406
5407                 eval { $run_qemu->() };
5408                 if (my $err = $@) {
5409                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5410                     die $err;
5411                 }
5412
5413                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5414             };
5415             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5416
5417         } else {
5418             eval { $run_qemu->() };
5419         }
5420
5421         if (my $err = $@) {
5422             # deactivate volumes if start fails
5423             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5424             die "start failed: $err";
5425         }
5426
5427         print "migration listens on $migrate_uri\n" if $migrate_uri;
5428
5429         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5430             eval { mon_cmd($vmid, "cont"); };
5431             warn $@ if $@;
5432         }
5433
5434         #start nbd server for storage migration
5435         if ($targetstorage) {
5436             my $nodename = PVE::INotify::nodename();
5437             my $localip = $get_migration_ip->($migration_network, $nodename);
5438             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5439             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5440
5441             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5442
5443             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5444
5445             foreach my $opt (sort keys %$local_volumes) {
5446                 my $volid = $local_volumes->{$opt};
5447                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5448                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5449                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5450             }
5451         }
5452
5453         if ($migratedfrom) {
5454             eval {
5455                 set_migration_caps($vmid);
5456             };
5457             warn $@ if $@;
5458
5459             if ($spice_port) {
5460                 print "spice listens on port $spice_port\n";
5461                 if ($spice_ticket) {
5462                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5463                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5464                 }
5465             }
5466
5467         } else {
5468             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5469                 if !$statefile && $conf->{balloon};
5470
5471             foreach my $opt (keys %$conf) {
5472                 next if $opt !~  m/^net\d+$/;
5473                 my $nicconf = parse_net($conf->{$opt});
5474                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5475             }
5476         }
5477
5478         mon_cmd($vmid, 'qom-set',
5479                     path => "machine/peripheral/balloon0",
5480                     property => "guest-stats-polling-interval",
5481                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5482
5483         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5484             print "Resumed VM, removing state\n";
5485             delete $conf->@{qw(lock vmstate runningmachine)};
5486             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5487             PVE::Storage::vdisk_free($storecfg, $vmstate);
5488             PVE::QemuConfig->write_config($vmid, $conf);
5489         }
5490
5491         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5492     });
5493 }
5494
5495 sub vm_commandline {
5496     my ($storecfg, $vmid, $snapname) = @_;
5497
5498     my $conf = PVE::QemuConfig->load_config($vmid);
5499     my $forcemachine;
5500
5501     if ($snapname) {
5502         my $snapshot = $conf->{snapshots}->{$snapname};
5503         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5504
5505         # check for a 'runningmachine' in snapshot
5506         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5507
5508         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5509
5510         $conf = $snapshot;
5511     }
5512
5513     my $defaults = load_defaults();
5514
5515     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5516
5517     return PVE::Tools::cmd2string($cmd);
5518 }
5519
5520 sub vm_reset {
5521     my ($vmid, $skiplock) = @_;
5522
5523     PVE::QemuConfig->lock_config($vmid, sub {
5524
5525         my $conf = PVE::QemuConfig->load_config($vmid);
5526
5527         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5528
5529         mon_cmd($vmid, "system_reset");
5530     });
5531 }
5532
5533 sub get_vm_volumes {
5534     my ($conf) = @_;
5535
5536     my $vollist = [];
5537     foreach_volid($conf, sub {
5538         my ($volid, $attr) = @_;
5539
5540         return if $volid =~ m|^/|;
5541
5542         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5543         return if !$sid;
5544
5545         push @$vollist, $volid;
5546     });
5547
5548     return $vollist;
5549 }
5550
5551 sub vm_stop_cleanup {
5552     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5553
5554     eval {
5555
5556         if (!$keepActive) {
5557             my $vollist = get_vm_volumes($conf);
5558             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5559         }
5560
5561         foreach my $ext (qw(mon qmp pid vnc qga)) {
5562             unlink "/var/run/qemu-server/${vmid}.$ext";
5563         }
5564
5565         if ($conf->{ivshmem}) {
5566             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5567             # just delete it for now, VMs which have this already open do not
5568             # are affected, but new VMs will get a separated one. If this
5569             # becomes an issue we either add some sort of ref-counting or just
5570             # add a "don't delete on stop" flag to the ivshmem format.
5571             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5572         }
5573
5574         foreach my $key (keys %$conf) {
5575             next if $key !~ m/^hostpci(\d+)$/;
5576             my $hostpciindex = $1;
5577             my $d = parse_hostpci($conf->{$key});
5578             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5579
5580             foreach my $pci (@{$d->{pciid}}) {
5581                 my $pciid = $pci->{id};
5582                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5583             }
5584         }
5585
5586         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5587     };
5588     warn $@ if $@; # avoid errors - just warn
5589 }
5590
5591 # call only in locked context
5592 sub _do_vm_stop {
5593     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5594
5595     my $pid = check_running($vmid, $nocheck);
5596     return if !$pid;
5597
5598     my $conf;
5599     if (!$nocheck) {
5600         $conf = PVE::QemuConfig->load_config($vmid);
5601         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5602         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5603             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5604             $timeout = $opts->{down} if $opts->{down};
5605         }
5606         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5607     }
5608
5609     eval {
5610         if ($shutdown) {
5611             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5612                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5613             } else {
5614                 mon_cmd($vmid, "system_powerdown");
5615             }
5616         } else {
5617             mon_cmd($vmid, "quit");
5618         }
5619     };
5620     my $err = $@;
5621
5622     if (!$err) {
5623         $timeout = 60 if !defined($timeout);
5624
5625         my $count = 0;
5626         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5627             $count++;
5628             sleep 1;
5629         }
5630
5631         if ($count >= $timeout) {
5632             if ($force) {
5633                 warn "VM still running - terminating now with SIGTERM\n";
5634                 kill 15, $pid;
5635             } else {
5636                 die "VM quit/powerdown failed - got timeout\n";
5637             }
5638         } else {
5639             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5640             return;
5641         }
5642     } else {
5643         if ($force) {
5644             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5645             kill 15, $pid;
5646         } else {
5647             die "VM quit/powerdown failed\n";
5648         }
5649     }
5650
5651     # wait again
5652     $timeout = 10;
5653
5654     my $count = 0;
5655     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5656         $count++;
5657         sleep 1;
5658     }
5659
5660     if ($count >= $timeout) {
5661         warn "VM still running - terminating now with SIGKILL\n";
5662         kill 9, $pid;
5663         sleep 1;
5664     }
5665
5666     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5667 }
5668
5669 # Note: use $nocheck to skip tests if VM configuration file exists.
5670 # We need that when migration VMs to other nodes (files already moved)
5671 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5672 sub vm_stop {
5673     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5674
5675     $force = 1 if !defined($force) && !$shutdown;
5676
5677     if ($migratedfrom){
5678         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5679         kill 15, $pid if $pid;
5680         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5681         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5682         return;
5683     }
5684
5685     PVE::QemuConfig->lock_config($vmid, sub {
5686         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5687    });
5688 }
5689
5690 sub vm_reboot {
5691     my ($vmid, $timeout) = @_;
5692
5693     PVE::QemuConfig->lock_config($vmid, sub {
5694         eval {
5695
5696             # only reboot if running, as qmeventd starts it again on a stop event
5697             return if !check_running($vmid);
5698
5699             create_reboot_request($vmid);
5700
5701             my $storecfg = PVE::Storage::config();
5702             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5703
5704         };
5705         if (my $err = $@) {
5706             # avoid that the next normal shutdown will be confused for a reboot
5707             clear_reboot_request($vmid);
5708             die $err;
5709         }
5710    });
5711 }
5712
5713 sub vm_suspend {
5714     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5715
5716     my $conf;
5717     my $path;
5718     my $storecfg;
5719     my $vmstate;
5720
5721     PVE::QemuConfig->lock_config($vmid, sub {
5722
5723         $conf = PVE::QemuConfig->load_config($vmid);
5724
5725         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5726         PVE::QemuConfig->check_lock($conf)
5727             if !($skiplock || $is_backing_up);
5728
5729         die "cannot suspend to disk during backup\n"
5730             if $is_backing_up && $includestate;
5731
5732         if ($includestate) {
5733             $conf->{lock} = 'suspending';
5734             my $date = strftime("%Y-%m-%d", localtime(time()));
5735             $storecfg = PVE::Storage::config();
5736             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5737             $path = PVE::Storage::path($storecfg, $vmstate);
5738             PVE::QemuConfig->write_config($vmid, $conf);
5739         } else {
5740             mon_cmd($vmid, "stop");
5741         }
5742     });
5743
5744     if ($includestate) {
5745         # save vm state
5746         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5747
5748         eval {
5749             mon_cmd($vmid, "savevm-start", statefile => $path);
5750             for(;;) {
5751                 my $state = mon_cmd($vmid, "query-savevm");
5752                 if (!$state->{status}) {
5753                     die "savevm not active\n";
5754                 } elsif ($state->{status} eq 'active') {
5755                     sleep(1);
5756                     next;
5757                 } elsif ($state->{status} eq 'completed') {
5758                     print "State saved, quitting\n";
5759                     last;
5760                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5761                     die "query-savevm failed with error '$state->{error}'\n"
5762                 } else {
5763                     die "query-savevm returned status '$state->{status}'\n";
5764                 }
5765             }
5766         };
5767         my $err = $@;
5768
5769         PVE::QemuConfig->lock_config($vmid, sub {
5770             $conf = PVE::QemuConfig->load_config($vmid);
5771             if ($err) {
5772                 # cleanup, but leave suspending lock, to indicate something went wrong
5773                 eval {
5774                     mon_cmd($vmid, "savevm-end");
5775                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5776                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5777                     delete $conf->@{qw(vmstate runningmachine)};
5778                     PVE::QemuConfig->write_config($vmid, $conf);
5779                 };
5780                 warn $@ if $@;
5781                 die $err;
5782             }
5783
5784             die "lock changed unexpectedly\n"
5785                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5786
5787             mon_cmd($vmid, "quit");
5788             $conf->{lock} = 'suspended';
5789             PVE::QemuConfig->write_config($vmid, $conf);
5790         });
5791     }
5792 }
5793
5794 sub vm_resume {
5795     my ($vmid, $skiplock, $nocheck) = @_;
5796
5797     PVE::QemuConfig->lock_config($vmid, sub {
5798         my $res = mon_cmd($vmid, 'query-status');
5799         my $resume_cmd = 'cont';
5800
5801         if ($res->{status} && $res->{status} eq 'suspended') {
5802             $resume_cmd = 'system_wakeup';
5803         }
5804
5805         if (!$nocheck) {
5806
5807             my $conf = PVE::QemuConfig->load_config($vmid);
5808
5809             PVE::QemuConfig->check_lock($conf)
5810                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5811         }
5812
5813         mon_cmd($vmid, $resume_cmd);
5814     });
5815 }
5816
5817 sub vm_sendkey {
5818     my ($vmid, $skiplock, $key) = @_;
5819
5820     PVE::QemuConfig->lock_config($vmid, sub {
5821
5822         my $conf = PVE::QemuConfig->load_config($vmid);
5823
5824         # there is no qmp command, so we use the human monitor command
5825         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5826         die $res if $res ne '';
5827     });
5828 }
5829
5830 # vzdump restore implementaion
5831
5832 sub tar_archive_read_firstfile {
5833     my $archive = shift;
5834
5835     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5836
5837     # try to detect archive type first
5838     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5839         die "unable to open file '$archive'\n";
5840     my $firstfile = <$fh>;
5841     kill 15, $pid;
5842     close $fh;
5843
5844     die "ERROR: archive contaions no data\n" if !$firstfile;
5845     chomp $firstfile;
5846
5847     return $firstfile;
5848 }
5849
5850 sub tar_restore_cleanup {
5851     my ($storecfg, $statfile) = @_;
5852
5853     print STDERR "starting cleanup\n";
5854
5855     if (my $fd = IO::File->new($statfile, "r")) {
5856         while (defined(my $line = <$fd>)) {
5857             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5858                 my $volid = $2;
5859                 eval {
5860                     if ($volid =~ m|^/|) {
5861                         unlink $volid || die 'unlink failed\n';
5862                     } else {
5863                         PVE::Storage::vdisk_free($storecfg, $volid);
5864                     }
5865                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5866                 };
5867                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5868             } else {
5869                 print STDERR "unable to parse line in statfile - $line";
5870             }
5871         }
5872         $fd->close();
5873     }
5874 }
5875
5876 sub restore_archive {
5877     my ($archive, $vmid, $user, $opts) = @_;
5878
5879     my $format = $opts->{format};
5880     my $comp;
5881
5882     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5883         $format = 'tar' if !$format;
5884         $comp = 'gzip';
5885     } elsif ($archive =~ m/\.tar$/) {
5886         $format = 'tar' if !$format;
5887     } elsif ($archive =~ m/.tar.lzo$/) {
5888         $format = 'tar' if !$format;
5889         $comp = 'lzop';
5890     } elsif ($archive =~ m/\.vma$/) {
5891         $format = 'vma' if !$format;
5892     } elsif ($archive =~ m/\.vma\.gz$/) {
5893         $format = 'vma' if !$format;
5894         $comp = 'gzip';
5895     } elsif ($archive =~ m/\.vma\.lzo$/) {
5896         $format = 'vma' if !$format;
5897         $comp = 'lzop';
5898     } else {
5899         $format = 'vma' if !$format; # default
5900     }
5901
5902     # try to detect archive format
5903     if ($format eq 'tar') {
5904         return restore_tar_archive($archive, $vmid, $user, $opts);
5905     } else {
5906         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5907     }
5908 }
5909
5910 sub restore_update_config_line {
5911     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5912
5913     return if $line =~ m/^\#qmdump\#/;
5914     return if $line =~ m/^\#vzdump\#/;
5915     return if $line =~ m/^lock:/;
5916     return if $line =~ m/^unused\d+:/;
5917     return if $line =~ m/^parent:/;
5918
5919     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5920     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5921         # try to convert old 1.X settings
5922         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5923         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5924             my ($model, $macaddr) = split(/\=/, $devconfig);
5925             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5926             my $net = {
5927                 model => $model,
5928                 bridge => "vmbr$ind",
5929                 macaddr => $macaddr,
5930             };
5931             my $netstr = print_net($net);
5932
5933             print $outfd "net$cookie->{netcount}: $netstr\n";
5934             $cookie->{netcount}++;
5935         }
5936     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5937         my ($id, $netstr) = ($1, $2);
5938         my $net = parse_net($netstr);
5939         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5940         $netstr = print_net($net);
5941         print $outfd "$id: $netstr\n";
5942     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5943         my $virtdev = $1;
5944         my $value = $3;
5945         my $di = parse_drive($virtdev, $value);
5946         if (defined($di->{backup}) && !$di->{backup}) {
5947             print $outfd "#$line";
5948         } elsif ($map->{$virtdev}) {
5949             delete $di->{format}; # format can change on restore
5950             $di->{file} = $map->{$virtdev};
5951             $value = print_drive($vmid, $di);
5952             print $outfd "$virtdev: $value\n";
5953         } else {
5954             print $outfd $line;
5955         }
5956     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5957         my $vmgenid = $1;
5958         if ($vmgenid ne '0') {
5959             # always generate a new vmgenid if there was a valid one setup
5960             $vmgenid = generate_uuid();
5961         }
5962         print $outfd "vmgenid: $vmgenid\n";
5963     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5964         my ($uuid, $uuid_str);
5965         UUID::generate($uuid);
5966         UUID::unparse($uuid, $uuid_str);
5967         my $smbios1 = parse_smbios1($2);
5968         $smbios1->{uuid} = $uuid_str;
5969         print $outfd $1.print_smbios1($smbios1)."\n";
5970     } else {
5971         print $outfd $line;
5972     }
5973 }
5974
5975 sub scan_volids {
5976     my ($cfg, $vmid) = @_;
5977
5978     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5979
5980     my $volid_hash = {};
5981     foreach my $storeid (keys %$info) {
5982         foreach my $item (@{$info->{$storeid}}) {
5983             next if !($item->{volid} && $item->{size});
5984             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5985             $volid_hash->{$item->{volid}} = $item;
5986         }
5987     }
5988
5989     return $volid_hash;
5990 }
5991
5992 sub is_volume_in_use {
5993     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5994
5995     my $path = PVE::Storage::path($storecfg, $volid);
5996
5997     my $scan_config = sub {
5998         my ($cref, $snapname) = @_;
5999
6000         foreach my $key (keys %$cref) {
6001             my $value = $cref->{$key};
6002             if (is_valid_drivename($key)) {
6003                 next if $skip_drive && $key eq $skip_drive;
6004                 my $drive = parse_drive($key, $value);
6005                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6006                 return 1 if $volid eq $drive->{file};
6007                 if ($drive->{file} =~ m!^/!) {
6008                     return 1 if $drive->{file} eq $path;
6009                 } else {
6010                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6011                     next if !$storeid;
6012                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6013                     next if !$scfg;
6014                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6015                 }
6016             }
6017         }
6018
6019         return 0;
6020     };
6021
6022     return 1 if &$scan_config($conf);
6023
6024     undef $skip_drive;
6025
6026     foreach my $snapname (keys %{$conf->{snapshots}}) {
6027         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6028     }
6029
6030     return 0;
6031 }
6032
6033 sub update_disksize {
6034     my ($vmid, $conf, $volid_hash) = @_;
6035
6036     my $changes;
6037     my $prefix = "VM $vmid:";
6038
6039     # used and unused disks
6040     my $referenced = {};
6041
6042     # Note: it is allowed to define multiple storages with same path (alias), so
6043     # we need to check both 'volid' and real 'path' (two different volid can point
6044     # to the same path).
6045
6046     my $referencedpath = {};
6047
6048     # update size info
6049     foreach my $opt (keys %$conf) {
6050         if (is_valid_drivename($opt)) {
6051             my $drive = parse_drive($opt, $conf->{$opt});
6052             my $volid = $drive->{file};
6053             next if !$volid;
6054
6055             $referenced->{$volid} = 1;
6056             if ($volid_hash->{$volid} &&
6057                 (my $path = $volid_hash->{$volid}->{path})) {
6058                 $referencedpath->{$path} = 1;
6059             }
6060
6061             next if drive_is_cdrom($drive);
6062             next if !$volid_hash->{$volid};
6063
6064             $drive->{size} = $volid_hash->{$volid}->{size};
6065             my $new = print_drive($vmid, $drive);
6066             if ($new ne $conf->{$opt}) {
6067                 $changes = 1;
6068                 $conf->{$opt} = $new;
6069                 print "$prefix update disk '$opt' information.\n";
6070             }
6071         }
6072     }
6073
6074     # remove 'unusedX' entry if volume is used
6075     foreach my $opt (keys %$conf) {
6076         next if $opt !~ m/^unused\d+$/;
6077         my $volid = $conf->{$opt};
6078         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6079         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6080             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6081             $changes = 1;
6082             delete $conf->{$opt};
6083         }
6084
6085         $referenced->{$volid} = 1;
6086         $referencedpath->{$path} = 1 if $path;
6087     }
6088
6089     foreach my $volid (sort keys %$volid_hash) {
6090         next if $volid =~ m/vm-$vmid-state-/;
6091         next if $referenced->{$volid};
6092         my $path = $volid_hash->{$volid}->{path};
6093         next if !$path; # just to be sure
6094         next if $referencedpath->{$path};
6095         $changes = 1;
6096         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6097         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6098         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6099     }
6100
6101     return $changes;
6102 }
6103
6104 sub rescan {
6105     my ($vmid, $nolock, $dryrun) = @_;
6106
6107     my $cfg = PVE::Storage::config();
6108
6109     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6110     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6111     foreach my $stor (keys %{$cfg->{ids}}) {
6112         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6113     }
6114
6115     print "rescan volumes...\n";
6116     my $volid_hash = scan_volids($cfg, $vmid);
6117
6118     my $updatefn =  sub {
6119         my ($vmid) = @_;
6120
6121         my $conf = PVE::QemuConfig->load_config($vmid);
6122
6123         PVE::QemuConfig->check_lock($conf);
6124
6125         my $vm_volids = {};
6126         foreach my $volid (keys %$volid_hash) {
6127             my $info = $volid_hash->{$volid};
6128             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6129         }
6130
6131         my $changes = update_disksize($vmid, $conf, $vm_volids);
6132
6133         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6134     };
6135
6136     if (defined($vmid)) {
6137         if ($nolock) {
6138             &$updatefn($vmid);
6139         } else {
6140             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6141         }
6142     } else {
6143         my $vmlist = config_list();
6144         foreach my $vmid (keys %$vmlist) {
6145             if ($nolock) {
6146                 &$updatefn($vmid);
6147             } else {
6148                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6149             }
6150         }
6151     }
6152 }
6153
6154 sub restore_vma_archive {
6155     my ($archive, $vmid, $user, $opts, $comp) = @_;
6156
6157     my $readfrom = $archive;
6158
6159     my $cfg = PVE::Storage::config();
6160     my $commands = [];
6161     my $bwlimit = $opts->{bwlimit};
6162
6163     my $dbg_cmdstring = '';
6164     my $add_pipe = sub {
6165         my ($cmd) = @_;
6166         push @$commands, $cmd;
6167         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6168         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6169         $readfrom = '-';
6170     };
6171
6172     my $input = undef;
6173     if ($archive eq '-') {
6174         $input = '<&STDIN';
6175     } else {
6176         # If we use a backup from a PVE defined storage we also consider that
6177         # storage's rate limit:
6178         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6179         if (defined($volid)) {
6180             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6181             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6182             if ($readlimit) {
6183                 print STDERR "applying read rate limit: $readlimit\n";
6184                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6185                 $add_pipe->($cstream);
6186             }
6187         }
6188     }
6189
6190     if ($comp) {
6191         my $cmd;
6192         if ($comp eq 'gzip') {
6193             $cmd = ['zcat', $readfrom];
6194         } elsif ($comp eq 'lzop') {
6195             $cmd = ['lzop', '-d', '-c', $readfrom];
6196         } else {
6197             die "unknown compression method '$comp'\n";
6198         }
6199         $add_pipe->($cmd);
6200     }
6201
6202     my $tmpdir = "/var/tmp/vzdumptmp$$";
6203     rmtree $tmpdir;
6204
6205     # disable interrupts (always do cleanups)
6206     local $SIG{INT} =
6207         local $SIG{TERM} =
6208         local $SIG{QUIT} =
6209         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6210
6211     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6212     POSIX::mkfifo($mapfifo, 0600);
6213     my $fifofh;
6214
6215     my $openfifo = sub {
6216         open($fifofh, '>', $mapfifo) || die $!;
6217     };
6218
6219     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6220
6221     my $oldtimeout;
6222     my $timeout = 5;
6223
6224     my $devinfo = {};
6225
6226     my $rpcenv = PVE::RPCEnvironment::get();
6227
6228     my $conffile = PVE::QemuConfig->config_file($vmid);
6229     my $tmpfn = "$conffile.$$.tmp";
6230
6231     # Note: $oldconf is undef if VM does not exists
6232     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6233     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6234
6235     my %storage_limits;
6236
6237     my $print_devmap = sub {
6238         my $virtdev_hash = {};
6239
6240         my $cfgfn = "$tmpdir/qemu-server.conf";
6241
6242         # we can read the config - that is already extracted
6243         my $fh = IO::File->new($cfgfn, "r") ||
6244             "unable to read qemu-server.conf - $!\n";
6245
6246         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6247         if (-f $fwcfgfn) {
6248             my $pve_firewall_dir = '/etc/pve/firewall';
6249             mkdir $pve_firewall_dir; # make sure the dir exists
6250             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6251         }
6252
6253         while (defined(my $line = <$fh>)) {
6254             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6255                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6256                 die "archive does not contain data for drive '$virtdev'\n"
6257                     if !$devinfo->{$devname};
6258                 if (defined($opts->{storage})) {
6259                     $storeid = $opts->{storage} || 'local';
6260                 } elsif (!$storeid) {
6261                     $storeid = 'local';
6262                 }
6263                 $format = 'raw' if !$format;
6264                 $devinfo->{$devname}->{devname} = $devname;
6265                 $devinfo->{$devname}->{virtdev} = $virtdev;
6266                 $devinfo->{$devname}->{format} = $format;
6267                 $devinfo->{$devname}->{storeid} = $storeid;
6268
6269                 # check permission on storage
6270                 my $pool = $opts->{pool}; # todo: do we need that?
6271                 if ($user ne 'root@pam') {
6272                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6273                 }
6274
6275                 $storage_limits{$storeid} = $bwlimit;
6276
6277                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6278             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6279                 my $virtdev = $1;
6280                 my $drive = parse_drive($virtdev, $2);
6281                 if (drive_is_cloudinit($drive)) {
6282                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6283                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6284                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6285
6286                     my $d = {
6287                         format => $format,
6288                         storeid => $opts->{storage} // $storeid,
6289                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6290                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6291                         name => "vm-$vmid-cloudinit",
6292                         is_cloudinit => 1,
6293                     };
6294                     $virtdev_hash->{$virtdev} = $d;
6295                 }
6296             }
6297         }
6298
6299         foreach my $key (keys %storage_limits) {
6300             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6301             next if !$limit;
6302             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6303             $storage_limits{$key} = $limit * 1024;
6304         }
6305
6306         foreach my $devname (keys %$devinfo) {
6307             die "found no device mapping information for device '$devname'\n"
6308                 if !$devinfo->{$devname}->{virtdev};
6309         }
6310
6311         # create empty/temp config
6312         if ($oldconf) {
6313             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6314             foreach_drive($oldconf, sub {
6315                 my ($ds, $drive) = @_;
6316
6317                 return if drive_is_cdrom($drive, 1);
6318
6319                 my $volid = $drive->{file};
6320                 return if !$volid || $volid =~ m|^/|;
6321
6322                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6323                 return if !$path || !$owner || ($owner != $vmid);
6324
6325                 # Note: only delete disk we want to restore
6326                 # other volumes will become unused
6327                 if ($virtdev_hash->{$ds}) {
6328                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6329                     if (my $err = $@) {
6330                         warn $err;
6331                     }
6332                 }
6333             });
6334
6335             # delete vmstate files, after the restore we have no snapshots anymore
6336             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6337                 my $snap = $oldconf->{snapshots}->{$snapname};
6338                 if ($snap->{vmstate}) {
6339                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6340                     if (my $err = $@) {
6341                         warn $err;
6342                     }
6343                 }
6344             }
6345         }
6346
6347         my $map = {};
6348         foreach my $virtdev (sort keys %$virtdev_hash) {
6349             my $d = $virtdev_hash->{$virtdev};
6350             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6351             my $storeid = $d->{storeid};
6352             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6353
6354             my $map_opts = '';
6355             if (my $limit = $storage_limits{$storeid}) {
6356                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6357             }
6358
6359             # test if requested format is supported
6360             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6361             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6362             $d->{format} = $defFormat if !$supported;
6363
6364             my $name;
6365             if ($d->{is_cloudinit}) {
6366                 $name = $d->{name};
6367                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6368             }
6369
6370             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6371             print STDERR "new volume ID is '$volid'\n";
6372             $d->{volid} = $volid;
6373
6374             PVE::Storage::activate_volumes($cfg, [$volid]);
6375
6376             my $write_zeros = 1;
6377             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6378                 $write_zeros = 0;
6379             }
6380
6381             if (!$d->{is_cloudinit}) {
6382                 my $path = PVE::Storage::path($cfg, $volid);
6383
6384                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6385
6386                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6387             }
6388             $map->{$virtdev} = $volid;
6389         }
6390
6391         $fh->seek(0, 0) || die "seek failed - $!\n";
6392
6393         my $outfd = new IO::File ($tmpfn, "w") ||
6394             die "unable to write config for VM $vmid\n";
6395
6396         my $cookie = { netcount => 0 };
6397         while (defined(my $line = <$fh>)) {
6398             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6399         }
6400
6401         $fh->close();
6402         $outfd->close();
6403     };
6404
6405     eval {
6406         # enable interrupts
6407         local $SIG{INT} =
6408             local $SIG{TERM} =
6409             local $SIG{QUIT} =
6410             local $SIG{HUP} =
6411             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6412         local $SIG{ALRM} = sub { die "got timeout\n"; };
6413
6414         $oldtimeout = alarm($timeout);
6415
6416         my $parser = sub {
6417             my $line = shift;
6418
6419             print "$line\n";
6420
6421             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6422                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6423                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6424             } elsif ($line =~ m/^CTIME: /) {
6425                 # we correctly received the vma config, so we can disable
6426                 # the timeout now for disk allocation (set to 10 minutes, so
6427                 # that we always timeout if something goes wrong)
6428                 alarm(600);
6429                 &$print_devmap();
6430                 print $fifofh "done\n";
6431                 my $tmp = $oldtimeout || 0;
6432                 $oldtimeout = undef;
6433                 alarm($tmp);
6434                 close($fifofh);
6435             }
6436         };
6437
6438         print "restore vma archive: $dbg_cmdstring\n";
6439         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6440     };
6441     my $err = $@;
6442
6443     alarm($oldtimeout) if $oldtimeout;
6444
6445     my $vollist = [];
6446     foreach my $devname (keys %$devinfo) {
6447         my $volid = $devinfo->{$devname}->{volid};
6448         push @$vollist, $volid if $volid;
6449     }
6450
6451     PVE::Storage::deactivate_volumes($cfg, $vollist);
6452
6453     unlink $mapfifo;
6454
6455     if ($err) {
6456         rmtree $tmpdir;
6457         unlink $tmpfn;
6458
6459         foreach my $devname (keys %$devinfo) {
6460             my $volid = $devinfo->{$devname}->{volid};
6461             next if !$volid;
6462             eval {
6463                 if ($volid =~ m|^/|) {
6464                     unlink $volid || die 'unlink failed\n';
6465                 } else {
6466                     PVE::Storage::vdisk_free($cfg, $volid);
6467                 }
6468                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6469             };
6470             print STDERR "unable to cleanup '$volid' - $@" if $@;
6471         }
6472         die $err;
6473     }
6474
6475     rmtree $tmpdir;
6476
6477     rename($tmpfn, $conffile) ||
6478         die "unable to commit configuration file '$conffile'\n";
6479
6480     PVE::Cluster::cfs_update(); # make sure we read new file
6481
6482     eval { rescan($vmid, 1); };
6483     warn $@ if $@;
6484 }
6485
6486 sub restore_tar_archive {
6487     my ($archive, $vmid, $user, $opts) = @_;
6488
6489     if ($archive ne '-') {
6490         my $firstfile = tar_archive_read_firstfile($archive);
6491         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6492             if $firstfile ne 'qemu-server.conf';
6493     }
6494
6495     my $storecfg = PVE::Storage::config();
6496
6497     # avoid zombie disks when restoring over an existing VM -> cleanup first
6498     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6499     # skiplock=1 because qmrestore has set the 'create' lock itself already
6500     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6501     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6502
6503     my $tocmd = "/usr/lib/qemu-server/qmextract";
6504
6505     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6506     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6507     $tocmd .= ' --prealloc' if $opts->{prealloc};
6508     $tocmd .= ' --info' if $opts->{info};
6509
6510     # tar option "xf" does not autodetect compression when read from STDIN,
6511     # so we pipe to zcat
6512     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6513         PVE::Tools::shellquote("--to-command=$tocmd");
6514
6515     my $tmpdir = "/var/tmp/vzdumptmp$$";
6516     mkpath $tmpdir;
6517
6518     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6519     local $ENV{VZDUMP_VMID} = $vmid;
6520     local $ENV{VZDUMP_USER} = $user;
6521
6522     my $conffile = PVE::QemuConfig->config_file($vmid);
6523     my $tmpfn = "$conffile.$$.tmp";
6524
6525     # disable interrupts (always do cleanups)
6526     local $SIG{INT} =
6527         local $SIG{TERM} =
6528         local $SIG{QUIT} =
6529         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6530
6531     eval {
6532         # enable interrupts
6533         local $SIG{INT} =
6534             local $SIG{TERM} =
6535             local $SIG{QUIT} =
6536             local $SIG{HUP} =
6537             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6538
6539         if ($archive eq '-') {
6540             print "extracting archive from STDIN\n";
6541             run_command($cmd, input => "<&STDIN");
6542         } else {
6543             print "extracting archive '$archive'\n";
6544             run_command($cmd);
6545         }
6546
6547         return if $opts->{info};
6548
6549         # read new mapping
6550         my $map = {};
6551         my $statfile = "$tmpdir/qmrestore.stat";
6552         if (my $fd = IO::File->new($statfile, "r")) {
6553             while (defined (my $line = <$fd>)) {
6554                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6555                     $map->{$1} = $2 if $1;
6556                 } else {
6557                     print STDERR "unable to parse line in statfile - $line\n";
6558                 }
6559             }
6560             $fd->close();
6561         }
6562
6563         my $confsrc = "$tmpdir/qemu-server.conf";
6564
6565         my $srcfd = new IO::File($confsrc, "r") ||
6566             die "unable to open file '$confsrc'\n";
6567
6568         my $outfd = new IO::File ($tmpfn, "w") ||
6569             die "unable to write config for VM $vmid\n";
6570
6571         my $cookie = { netcount => 0 };
6572         while (defined (my $line = <$srcfd>)) {
6573             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6574         }
6575
6576         $srcfd->close();
6577         $outfd->close();
6578     };
6579     if (my $err = $@) {
6580         unlink $tmpfn;
6581         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6582         die $err;
6583     }
6584
6585     rmtree $tmpdir;
6586
6587     rename $tmpfn, $conffile ||
6588         die "unable to commit configuration file '$conffile'\n";
6589
6590     PVE::Cluster::cfs_update(); # make sure we read new file
6591
6592     eval { rescan($vmid, 1); };
6593     warn $@ if $@;
6594 };
6595
6596 sub foreach_storage_used_by_vm {
6597     my ($conf, $func) = @_;
6598
6599     my $sidhash = {};
6600
6601     foreach_drive($conf, sub {
6602         my ($ds, $drive) = @_;
6603         return if drive_is_cdrom($drive);
6604
6605         my $volid = $drive->{file};
6606
6607         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6608         $sidhash->{$sid} = $sid if $sid;
6609     });
6610
6611     foreach my $sid (sort keys %$sidhash) {
6612         &$func($sid);
6613     }
6614 }
6615
6616 sub do_snapshots_with_qemu {
6617     my ($storecfg, $volid) = @_;
6618
6619     my $storage_name = PVE::Storage::parse_volume_id($volid);
6620     my $scfg = $storecfg->{ids}->{$storage_name};
6621
6622     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6623         return 1;
6624     }
6625
6626     if ($volid =~ m/\.(qcow2|qed)$/){
6627         return 1;
6628     }
6629
6630     return undef;
6631 }
6632
6633 sub qga_check_running {
6634     my ($vmid, $nowarn) = @_;
6635
6636     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6637     if ($@) {
6638         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6639         return 0;
6640     }
6641     return 1;
6642 }
6643
6644 sub template_create {
6645     my ($vmid, $conf, $disk) = @_;
6646
6647     my $storecfg = PVE::Storage::config();
6648
6649     foreach_drive($conf, sub {
6650         my ($ds, $drive) = @_;
6651
6652         return if drive_is_cdrom($drive);
6653         return if $disk && $ds ne $disk;
6654
6655         my $volid = $drive->{file};
6656         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6657
6658         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6659         $drive->{file} = $voliddst;
6660         $conf->{$ds} = print_drive($vmid, $drive);
6661         PVE::QemuConfig->write_config($vmid, $conf);
6662     });
6663 }
6664
6665 sub convert_iscsi_path {
6666     my ($path) = @_;
6667
6668     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6669         my $portal = $1;
6670         my $target = $2;
6671         my $lun = $3;
6672
6673         my $initiator_name = get_initiator_name();
6674
6675         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6676                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6677     }
6678
6679     die "cannot convert iscsi path '$path', unkown format\n";
6680 }
6681
6682 sub qemu_img_convert {
6683     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6684
6685     my $storecfg = PVE::Storage::config();
6686     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6687     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6688
6689     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6690
6691     my $cachemode;
6692     my $src_path;
6693     my $src_is_iscsi = 0;
6694     my $src_format = 'raw';
6695
6696     if ($src_storeid) {
6697         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6698         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6699         $src_format = qemu_img_format($src_scfg, $src_volname);
6700         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6701         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6702         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6703     } elsif (-f $src_volid) {
6704         $src_path = $src_volid;
6705         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6706             $src_format = $1;
6707         }
6708     }
6709
6710     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6711
6712     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6713     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6714     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6715     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6716
6717     my $cmd = [];
6718     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6719     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6720     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6721     push @$cmd, '-T', $cachemode if defined($cachemode);
6722
6723     if ($src_is_iscsi) {
6724         push @$cmd, '--image-opts';
6725         $src_path = convert_iscsi_path($src_path);
6726     } else {
6727         push @$cmd, '-f', $src_format;
6728     }
6729
6730     if ($dst_is_iscsi) {
6731         push @$cmd, '--target-image-opts';
6732         $dst_path = convert_iscsi_path($dst_path);
6733     } else {
6734         push @$cmd, '-O', $dst_format;
6735     }
6736
6737     push @$cmd, $src_path;
6738
6739     if (!$dst_is_iscsi && $is_zero_initialized) {
6740         push @$cmd, "zeroinit:$dst_path";
6741     } else {
6742         push @$cmd, $dst_path;
6743     }
6744
6745     my $parser = sub {
6746         my $line = shift;
6747         if($line =~ m/\((\S+)\/100\%\)/){
6748             my $percent = $1;
6749             my $transferred = int($size * $percent / 100);
6750             my $remaining = $size - $transferred;
6751
6752             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6753         }
6754
6755     };
6756
6757     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6758     my $err = $@;
6759     die "copy failed: $err" if $err;
6760 }
6761
6762 sub qemu_img_format {
6763     my ($scfg, $volname) = @_;
6764
6765     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6766         return $1;
6767     } else {
6768         return "raw";
6769     }
6770 }
6771
6772 sub qemu_drive_mirror {
6773     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6774
6775     $jobs = {} if !$jobs;
6776
6777     my $qemu_target;
6778     my $format;
6779     $jobs->{"drive-$drive"} = {};
6780
6781     if ($dst_volid =~ /^nbd:/) {
6782         $qemu_target = $dst_volid;
6783         $format = "nbd";
6784     } else {
6785         my $storecfg = PVE::Storage::config();
6786         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6787
6788         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6789
6790         $format = qemu_img_format($dst_scfg, $dst_volname);
6791
6792         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6793
6794         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6795     }
6796
6797     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6798     $opts->{format} = $format if $format;
6799
6800     if (defined($bwlimit)) {
6801         $opts->{speed} = $bwlimit * 1024;
6802         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6803     } else {
6804         print "drive mirror is starting for drive-$drive\n";
6805     }
6806
6807     # if a job already runs for this device we get an error, catch it for cleanup
6808     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6809     if (my $err = $@) {
6810         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6811         warn "$@\n" if $@;
6812         die "mirroring error: $err\n";
6813     }
6814
6815     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6816 }
6817
6818 sub qemu_drive_mirror_monitor {
6819     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6820
6821     eval {
6822         my $err_complete = 0;
6823
6824         while (1) {
6825             die "storage migration timed out\n" if $err_complete > 300;
6826
6827             my $stats = mon_cmd($vmid, "query-block-jobs");
6828
6829             my $running_mirror_jobs = {};
6830             foreach my $stat (@$stats) {
6831                 next if $stat->{type} ne 'mirror';
6832                 $running_mirror_jobs->{$stat->{device}} = $stat;
6833             }
6834
6835             my $readycounter = 0;
6836
6837             foreach my $job (keys %$jobs) {
6838
6839                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6840                     print "$job : finished\n";
6841                     delete $jobs->{$job};
6842                     next;
6843                 }
6844
6845                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6846
6847                 my $busy = $running_mirror_jobs->{$job}->{busy};
6848                 my $ready = $running_mirror_jobs->{$job}->{ready};
6849                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6850                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6851                     my $remaining = $total - $transferred;
6852                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6853
6854                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6855                 }
6856
6857                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6858             }
6859
6860             last if scalar(keys %$jobs) == 0;
6861
6862             if ($readycounter == scalar(keys %$jobs)) {
6863                 print "all mirroring jobs are ready \n";
6864                 last if $skipcomplete; #do the complete later
6865
6866                 if ($vmiddst && $vmiddst != $vmid) {
6867                     my $agent_running = $qga && qga_check_running($vmid);
6868                     if ($agent_running) {
6869                         print "freeze filesystem\n";
6870                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6871                     } else {
6872                         print "suspend vm\n";
6873                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6874                     }
6875
6876                     # if we clone a disk for a new target vm, we don't switch the disk
6877                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6878
6879                     if ($agent_running) {
6880                         print "unfreeze filesystem\n";
6881                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6882                     } else {
6883                         print "resume vm\n";
6884                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6885                     }
6886
6887                     last;
6888                 } else {
6889
6890                     foreach my $job (keys %$jobs) {
6891                         # try to switch the disk if source and destination are on the same guest
6892                         print "$job: Completing block job...\n";
6893
6894                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6895                         if ($@ =~ m/cannot be completed/) {
6896                             print "$job: Block job cannot be completed, try again.\n";
6897                             $err_complete++;
6898                         }else {
6899                             print "$job: Completed successfully.\n";
6900                             $jobs->{$job}->{complete} = 1;
6901                         }
6902                     }
6903                 }
6904             }
6905             sleep 1;
6906         }
6907     };
6908     my $err = $@;
6909
6910     if ($err) {
6911         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6912         die "mirroring error: $err";
6913     }
6914
6915 }
6916
6917 sub qemu_blockjobs_cancel {
6918     my ($vmid, $jobs) = @_;
6919
6920     foreach my $job (keys %$jobs) {
6921         print "$job: Cancelling block job\n";
6922         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6923         $jobs->{$job}->{cancel} = 1;
6924     }
6925
6926     while (1) {
6927         my $stats = mon_cmd($vmid, "query-block-jobs");
6928
6929         my $running_jobs = {};
6930         foreach my $stat (@$stats) {
6931             $running_jobs->{$stat->{device}} = $stat;
6932         }
6933
6934         foreach my $job (keys %$jobs) {
6935
6936             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6937                 print "$job: Done.\n";
6938                 delete $jobs->{$job};
6939             }
6940         }
6941
6942         last if scalar(keys %$jobs) == 0;
6943
6944         sleep 1;
6945     }
6946 }
6947
6948 sub clone_disk {
6949     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6950         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6951
6952     my $newvolid;
6953
6954     if (!$full) {
6955         print "create linked clone of drive $drivename ($drive->{file})\n";
6956         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6957         push @$newvollist, $newvolid;
6958     } else {
6959
6960         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6961         $storeid = $storage if $storage;
6962
6963         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6964         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6965
6966         print "create full clone of drive $drivename ($drive->{file})\n";
6967         my $name = undef;
6968         if (drive_is_cloudinit($drive)) {
6969             $name = "vm-$newvmid-cloudinit";
6970             $name .= ".$dst_format" if $dst_format ne 'raw';
6971             $snapname = undef;
6972             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6973         }
6974         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6975         push @$newvollist, $newvolid;
6976
6977         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6978
6979         if (drive_is_cloudinit($drive)) {
6980             goto no_data_clone;
6981         }
6982
6983         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6984         if (!$running || $snapname) {
6985             # TODO: handle bwlimits
6986             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6987         } else {
6988
6989             my $kvmver = get_running_qemu_version ($vmid);
6990             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6991                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6992                     if $drive->{iothread};
6993             }
6994
6995             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6996         }
6997     }
6998
6999 no_data_clone:
7000     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7001
7002     my $disk = $drive;
7003     $disk->{format} = undef;
7004     $disk->{file} = $newvolid;
7005     $disk->{size} = $size;
7006
7007     return $disk;
7008 }
7009
7010 # this only works if VM is running
7011 sub get_current_qemu_machine {
7012     my ($vmid) = @_;
7013
7014     my $res = mon_cmd($vmid, "query-machines");
7015
7016     my ($current, $default);
7017     foreach my $e (@$res) {
7018         $default = $e->{name} if $e->{'is-default'};
7019         $current = $e->{name} if $e->{'is-current'};
7020     }
7021
7022     # fallback to the default machine if current is not supported by qemu
7023     return $current || $default || 'pc';
7024 }
7025
7026 sub get_running_qemu_version {
7027     my ($vmid) = @_;
7028     my $res = mon_cmd($vmid, "query-version");
7029     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7030 }
7031
7032 sub qemu_machine_feature_enabled {
7033     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7034
7035     my $current_major;
7036     my $current_minor;
7037
7038     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7039
7040         $current_major = $3;
7041         $current_minor = $4;
7042
7043     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7044
7045         $current_major = $1;
7046         $current_minor = $2;
7047     }
7048
7049     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7050 }
7051
7052 # gets in pairs the versions you want to compares, i.e.:
7053 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7054 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7055 sub version_cmp {
7056     my @versions = @_;
7057
7058     my $size = scalar(@versions);
7059
7060     return 0 if $size == 0;
7061     die "cannot compare odd count of versions" if $size & 1;
7062
7063     for (my $i = 0; $i < $size; $i += 2) {
7064         my ($a, $b) = splice(@versions, 0, 2);
7065         $a //= 0;
7066         $b //= 0;
7067
7068         return 1 if $a > $b;
7069         return -1 if $a < $b;
7070     }
7071     return 0;
7072 }
7073
7074 # dies if a) VM not running or not exisiting b) Version query failed
7075 # So, any defined return value is valid, any invalid state can be caught by eval
7076 sub runs_at_least_qemu_version {
7077     my ($vmid, $major, $minor, $extra) = @_;
7078
7079     my $v = mon_cmd($vmid, "query-version");
7080     die "could not query currently running version for VM $vmid\n" if !defined($v);
7081     $v = $v->{qemu};
7082
7083     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7084 }
7085
7086 sub qemu_machine_pxe {
7087     my ($vmid, $conf) = @_;
7088
7089     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7090
7091     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7092         $machine .= '.pxe';
7093     }
7094
7095     return $machine;
7096 }
7097
7098 sub qemu_use_old_bios_files {
7099     my ($machine_type) = @_;
7100
7101     return if !$machine_type;
7102
7103     my $use_old_bios_files = undef;
7104
7105     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7106         $machine_type = $1;
7107         $use_old_bios_files = 1;
7108     } else {
7109         my $kvmver = kvm_user_version();
7110         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7111         # load new efi bios files on migration. So this hack is required to allow
7112         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7113         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7114         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7115     }
7116
7117     return ($use_old_bios_files, $machine_type);
7118 }
7119
7120 sub create_efidisk($$$$$) {
7121     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7122
7123     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7124     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7125
7126     my $vars_size_b = -s $ovmf_vars;
7127     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7128     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7129     PVE::Storage::activate_volumes($storecfg, [$volid]);
7130
7131     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7132
7133     return ($volid, $vars_size);
7134 }
7135
7136 sub vm_iothreads_list {
7137     my ($vmid) = @_;
7138
7139     my $res = mon_cmd($vmid, 'query-iothreads');
7140
7141     my $iothreads = {};
7142     foreach my $iothread (@$res) {
7143         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7144     }
7145
7146     return $iothreads;
7147 }
7148
7149 sub scsihw_infos {
7150     my ($conf, $drive) = @_;
7151
7152     my $maxdev = 0;
7153
7154     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7155         $maxdev = 7;
7156     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7157         $maxdev = 1;
7158     } else {
7159         $maxdev = 256;
7160     }
7161
7162     my $controller = int($drive->{index} / $maxdev);
7163     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7164
7165     return ($maxdev, $controller, $controller_prefix);
7166 }
7167
7168 sub add_hyperv_enlightenments {
7169     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7170
7171     return if $winversion < 6;
7172     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7173
7174     if ($gpu_passthrough || defined($hv_vendor_id)) {
7175         $hv_vendor_id //= 'proxmox';
7176         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7177     }
7178
7179     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7180         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7181         push @$cpuFlags , 'hv_vapic';
7182         push @$cpuFlags , 'hv_time';
7183     } else {
7184         push @$cpuFlags , 'hv_spinlocks=0xffff';
7185     }
7186
7187     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7188         push @$cpuFlags , 'hv_reset';
7189         push @$cpuFlags , 'hv_vpindex';
7190         push @$cpuFlags , 'hv_runtime';
7191     }
7192
7193     if ($winversion >= 7) {
7194         push @$cpuFlags , 'hv_relaxed';
7195
7196         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7197             push @$cpuFlags , 'hv_synic';
7198             push @$cpuFlags , 'hv_stimer';
7199         }
7200
7201         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7202             push @$cpuFlags , 'hv_ipi';
7203         }
7204     }
7205 }
7206
7207 sub windows_version {
7208     my ($ostype) = @_;
7209
7210     return 0 if !$ostype;
7211
7212     my $winversion = 0;
7213
7214     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7215         $winversion = 5;
7216     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7217         $winversion = 6;
7218     } elsif ($ostype =~ m/^win(\d+)$/) {
7219         $winversion = $1;
7220     }
7221
7222     return $winversion;
7223 }
7224
7225 sub resolve_dst_disk_format {
7226         my ($storecfg, $storeid, $src_volname, $format) = @_;
7227         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7228
7229         if (!$format) {
7230             # if no target format is specified, use the source disk format as hint
7231             if ($src_volname) {
7232                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7233                 $format = qemu_img_format($scfg, $src_volname);
7234             } else {
7235                 return $defFormat;
7236             }
7237         }
7238
7239         # test if requested format is supported - else use default
7240         my $supported = grep { $_ eq $format } @$validFormats;
7241         $format = $defFormat if !$supported;
7242         return $format;
7243 }
7244
7245 sub resolve_first_disk {
7246     my $conf = shift;
7247     my @disks = PVE::QemuServer::valid_drive_names();
7248     my $firstdisk;
7249     foreach my $ds (reverse @disks) {
7250         next if !$conf->{$ds};
7251         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7252         next if PVE::QemuServer::drive_is_cdrom($disk);
7253         $firstdisk = $ds;
7254     }
7255     return $firstdisk;
7256 }
7257
7258 sub generate_uuid {
7259     my ($uuid, $uuid_str);
7260     UUID::generate($uuid);
7261     UUID::unparse($uuid, $uuid_str);
7262     return $uuid_str;
7263 }
7264
7265 sub generate_smbios1_uuid {
7266     return "uuid=".generate_uuid();
7267 }
7268
7269 sub nbd_stop {
7270     my ($vmid) = @_;
7271
7272     mon_cmd($vmid, 'nbd-server-stop');
7273 }
7274
7275 sub create_reboot_request {
7276     my ($vmid) = @_;
7277     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7278         or die "failed to create reboot trigger file: $!\n";
7279     close($fh);
7280 }
7281
7282 sub clear_reboot_request {
7283     my ($vmid) = @_;
7284     my $path = "/run/qemu-server/$vmid.reboot";
7285     my $res = 0;
7286
7287     $res = unlink($path);
7288     die "could not remove reboot request for $vmid: $!"
7289         if !$res && $! != POSIX::ENOENT;
7290
7291     return $res;
7292 }
7293
7294 # bash completion helper
7295
7296 sub complete_backup_archives {
7297     my ($cmdname, $pname, $cvalue) = @_;
7298
7299     my $cfg = PVE::Storage::config();
7300
7301     my $storeid;
7302
7303     if ($cvalue =~ m/^([^:]+):/) {
7304         $storeid = $1;
7305     }
7306
7307     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7308
7309     my $res = [];
7310     foreach my $id (keys %$data) {
7311         foreach my $item (@{$data->{$id}}) {
7312             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7313             push @$res, $item->{volid} if defined($item->{volid});
7314         }
7315     }
7316
7317     return $res;
7318 }
7319
7320 my $complete_vmid_full = sub {
7321     my ($running) = @_;
7322
7323     my $idlist = vmstatus();
7324
7325     my $res = [];
7326
7327     foreach my $id (keys %$idlist) {
7328         my $d = $idlist->{$id};
7329         if (defined($running)) {
7330             next if $d->{template};
7331             next if $running && $d->{status} ne 'running';
7332             next if !$running && $d->{status} eq 'running';
7333         }
7334         push @$res, $id;
7335
7336     }
7337     return $res;
7338 };
7339
7340 sub complete_vmid {
7341     return &$complete_vmid_full();
7342 }
7343
7344 sub complete_vmid_stopped {
7345     return &$complete_vmid_full(0);
7346 }
7347
7348 sub complete_vmid_running {
7349     return &$complete_vmid_full(1);
7350 }
7351
7352 sub complete_storage {
7353
7354     my $cfg = PVE::Storage::config();
7355     my $ids = $cfg->{ids};
7356
7357     my $res = [];
7358     foreach my $sid (keys %$ids) {
7359         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7360         next if !$ids->{$sid}->{content}->{images};
7361         push @$res, $sid;
7362     }
7363
7364     return $res;
7365 }
7366
7367 sub complete_migration_storage {
7368     my ($cmd, $param, $current_value, $all_args) = @_;
7369
7370     my $targetnode = @$all_args[1];
7371
7372     my $cfg = PVE::Storage::config();
7373     my $ids = $cfg->{ids};
7374
7375     my $res = [];
7376     foreach my $sid (keys %$ids) {
7377         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7378         next if !$ids->{$sid}->{content}->{images};
7379         push @$res, $sid;
7380     }
7381
7382     return $res;
7383 }
7384
7385 1;