PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     }
 641 };
 642
 643 my $cicustom_fmt = {
 644     meta => {
 645         type => 'string',
 646         optional => 1,
 647         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 648         format => 'pve-volume-id',
 649         format_description => 'volume',
 650     },
 651     network => {
 652         type => 'string',
 653         optional => 1,
 654         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 655         format => 'pve-volume-id',
 656         format_description => 'volume',
 657     },
 658     user => {
 659         type => 'string',
 660         optional => 1,
 661         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 662         format => 'pve-volume-id',
 663         format_description => 'volume',
 664     },
 665 };
 666 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 667
 668 my $confdesc_cloudinit = {
 669     citype => {
 670         optional => 1,
 671         type => 'string',
 672         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 673         enum => ['configdrive2', 'nocloud'],
 674     },
 675     ciuser => {
 676         optional => 1,
 677         type => 'string',
 678         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 679     },
 680     cipassword => {
 681         optional => 1,
 682         type => 'string',
 683         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 684     },
 685     cicustom => {
 686         optional => 1,
 687         type => 'string',
 688         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 689         format => 'pve-qm-cicustom',
 690     },
 691     searchdomain => {
 692         optional => 1,
 693         type => 'string',
 694         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 695     },
 696     nameserver => {
 697         optional => 1,
 698         type => 'string', format => 'address-list',
 699         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 700     },
 701     sshkeys => {
 702         optional => 1,
 703         type => 'string',
 704         format => 'urlencoded',
 705         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 706     },
 707 };
 708
 709 # what about other qemu settings ?
 710 #cpu => 'string',
 711 #machine => 'string',
 712 #fda => 'file',
 713 #fdb => 'file',
 714 #mtdblock => 'file',
 715 #sd => 'file',
 716 #pflash => 'file',
 717 #snapshot => 'bool',
 718 #bootp => 'file',
 719 ##tftp => 'dir',
 720 ##smb => 'dir',
 721 #kernel => 'file',
 722 #append => 'string',
 723 #initrd => 'file',
 724 ##soundhw => 'string',
 725
 726 while (my ($k, $v) = each %$confdesc) {
 727     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 728 }
 729
 730 my $MAX_IDE_DISKS = 4;
 731 my $MAX_SCSI_DISKS = 14;
 732 my $MAX_VIRTIO_DISKS = 16;
 733 my $MAX_SATA_DISKS = 6;
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_UNUSED_DISKS = 256;
 737 my $MAX_HOSTPCI_DEVICES = 4;
 738 my $MAX_SERIAL_PORTS = 4;
 739 my $MAX_PARALLEL_PORTS = 3;
 740 my $MAX_NUMA = 8;
 741
 742 my $numa_fmt = {
 743     cpus => {
 744         type => "string",
 745         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 746         description => "CPUs accessing this NUMA node.",
 747         format_description => "id[-id];...",
 748     },
 749     memory => {
 750         type => "number",
 751         description => "Amount of memory this NUMA node provides.",
 752         optional => 1,
 753     },
 754     hostnodes => {
 755         type => "string",
 756         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 757         description => "Host NUMA nodes to use.",
 758         format_description => "id[-id];...",
 759         optional => 1,
 760     },
 761     policy => {
 762         type => 'string',
 763         enum => [qw(preferred bind interleave)],
 764         description => "NUMA allocation policy.",
 765         optional => 1,
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 769 my $numadesc = {
 770     optional => 1,
 771     type => 'string', format => $numa_fmt,
 772     description => "NUMA topology.",
 773 };
 774 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 775
 776 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 777     $confdesc->{"numa$i"} = $numadesc;
 778 }
 779
 780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 781                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 782                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 784
 785 my $net_fmt_bridge_descr = <<__EOD__;
 786 Bridge to attach the network device to. The Proxmox VE standard bridge
 787 is called 'vmbr0'.
 788
 789 If you do not specify a bridge, we create a kvm user (NATed) network
 790 device, which provides DHCP and DNS services. The following addresses
 791 are used:
 792
 793  10.0.2.2   Gateway
 794  10.0.2.3   DNS Server
 795  10.0.2.4   SMB Server
 796
 797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 798 __EOD__
 799
 800 my $net_fmt = {
 801     macaddr  => get_standard_option('mac-addr', {
 802         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 803     }),
 804     model => {
 805         type => 'string',
 806         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 807         enum => $nic_model_list,
 808         default_key => 1,
 809     },
 810     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 811     bridge => {
 812         type => 'string',
 813         description => $net_fmt_bridge_descr,
 814         format_description => 'bridge',
 815         optional => 1,
 816     },
 817     queues => {
 818         type => 'integer',
 819         minimum => 0, maximum => 16,
 820         description => 'Number of packet queues to be used on the device.',
 821         optional => 1,
 822     },
 823     rate => {
 824         type => 'number',
 825         minimum => 0,
 826         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 827         optional => 1,
 828     },
 829     tag => {
 830         type => 'integer',
 831         minimum => 1, maximum => 4094,
 832         description => 'VLAN tag to apply to packets on this interface.',
 833         optional => 1,
 834     },
 835     trunks => {
 836         type => 'string',
 837         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 838         description => 'VLAN trunks to pass through this interface.',
 839         format_description => 'vlanid[;vlanid...]',
 840         optional => 1,
 841     },
 842     firewall => {
 843         type => 'boolean',
 844         description => 'Whether this interface should be protected by the firewall.',
 845         optional => 1,
 846     },
 847     link_down => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be disconnected (like pulling the plug).',
 850         optional => 1,
 851     },
 852 };
 853
 854 my $netdesc = {
 855     optional => 1,
 856     type => 'string', format => $net_fmt,
 857     description => "Specify network devices.",
 858 };
 859
 860 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 861
 862 my $ipconfig_fmt = {
 863     ip => {
 864         type => 'string',
 865         format => 'pve-ipv4-config',
 866         format_description => 'IPv4Format/CIDR',
 867         description => 'IPv4 address in CIDR format.',
 868         optional => 1,
 869         default => 'dhcp',
 870     },
 871     gw => {
 872         type => 'string',
 873         format => 'ipv4',
 874         format_description => 'GatewayIPv4',
 875         description => 'Default gateway for IPv4 traffic.',
 876         optional => 1,
 877         requires => 'ip',
 878     },
 879     ip6 => {
 880         type => 'string',
 881         format => 'pve-ipv6-config',
 882         format_description => 'IPv6Format/CIDR',
 883         description => 'IPv6 address in CIDR format.',
 884         optional => 1,
 885         default => 'dhcp',
 886     },
 887     gw6 => {
 888         type => 'string',
 889         format => 'ipv6',
 890         format_description => 'GatewayIPv6',
 891         description => 'Default gateway for IPv6 traffic.',
 892         optional => 1,
 893         requires => 'ip6',
 894     },
 895 };
 896 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 897 my $ipconfigdesc = {
 898     optional => 1,
 899     type => 'string', format => 'pve-qm-ipconfig',
 900     description => <<'EODESCR',
 901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 902
 903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 904
 905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 907
 908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 909 EODESCR
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 912
 913 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 914     $confdesc->{"net$i"} = $netdesc;
 915     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 916 }
 917
 918 foreach my $key (keys %$confdesc_cloudinit) {
 919     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 920 }
 921
 922 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 923 sub verify_volume_id_or_qm_path {
 924     my ($volid, $noerr) = @_;
 925
 926     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 927         return $volid;
 928     }
 929
 930     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 931     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 932     if ($@) {
 933         return undef if $noerr;
 934         die $@;
 935     }
 936     return $volid;
 937 }
 938
 939 my $drivename_hash;
 940
 941 my %drivedesc_base = (
 942     volume => { alias => 'file' },
 943     file => {
 944         type => 'string',
 945         format => 'pve-volume-id-or-qm-path',
 946         default_key => 1,
 947         format_description => 'volume',
 948         description => "The drive's backing volume.",
 949     },
 950     media => {
 951         type => 'string',
 952         enum => [qw(cdrom disk)],
 953         description => "The drive's media type.",
 954         default => 'disk',
 955         optional => 1
 956     },
 957     cyls => {
 958         type => 'integer',
 959         description => "Force the drive's physical geometry to have a specific cylinder count.",
 960         optional => 1
 961     },
 962     heads => {
 963         type => 'integer',
 964         description => "Force the drive's physical geometry to have a specific head count.",
 965         optional => 1
 966     },
 967     secs => {
 968         type => 'integer',
 969         description => "Force the drive's physical geometry to have a specific sector count.",
 970         optional => 1
 971     },
 972     trans => {
 973         type => 'string',
 974         enum => [qw(none lba auto)],
 975         description => "Force disk geometry bios translation mode.",
 976         optional => 1,
 977     },
 978     snapshot => {
 979         type => 'boolean',
 980         description => "Controls qemu's snapshot mode feature."
 981             . " If activated, changes made to the disk are temporary and will"
 982             . " be discarded when the VM is shutdown.",
 983         optional => 1,
 984     },
 985     cache => {
 986         type => 'string',
 987         enum => [qw(none writethrough writeback unsafe directsync)],
 988         description => "The drive's cache mode",
 989         optional => 1,
 990     },
 991     format => get_standard_option('pve-qm-image-format'),
 992     size => {
 993         type => 'string',
 994         format => 'disk-size',
 995         format_description => 'DiskSize',
 996         description => "Disk size. This is purely informational and has no effect.",
 997         optional => 1,
 998     },
 999     backup => {
1000         type => 'boolean',
1001         description => "Whether the drive should be included when making backups.",
1002         optional => 1,
1003     },
1004     replicate => {
1005         type => 'boolean',
1006         description => 'Whether the drive should considered for replication jobs.',
1007         optional => 1,
1008         default => 1,
1009     },
1010     rerror => {
1011         type => 'string',
1012         enum => [qw(ignore report stop)],
1013         description => 'Read error action.',
1014         optional => 1,
1015     },
1016     werror => {
1017         type => 'string',
1018         enum => [qw(enospc ignore report stop)],
1019         description => 'Write error action.',
1020         optional => 1,
1021     },
1022     aio => {
1023         type => 'string',
1024         enum => [qw(native threads)],
1025         description => 'AIO type to use.',
1026         optional => 1,
1027     },
1028     discard => {
1029         type => 'string',
1030         enum => [qw(ignore on)],
1031         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1032         optional => 1,
1033     },
1034     detect_zeroes => {
1035         type => 'boolean',
1036         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1037         optional => 1,
1038     },
1039     serial => {
1040         type => 'string',
1041         format => 'urlencoded',
1042         format_description => 'serial',
1043         maxLength => 20*3, # *3 since it's %xx url enoded
1044         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1045         optional => 1,
1046     },
1047     shared => {
1048         type => 'boolean',
1049         description => 'Mark this locally-managed volume as available on all nodes',
1050         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1051         optional => 1,
1052         default => 0,
1053     }
1054 );
1055
1056 my %iothread_fmt = ( iothread => {
1057         type => 'boolean',
1058         description => "Whether to use iothreads for this drive",
1059         optional => 1,
1060 });
1061
1062 my %model_fmt = (
1063     model => {
1064         type => 'string',
1065         format => 'urlencoded',
1066         format_description => 'model',
1067         maxLength => 40*3, # *3 since it's %xx url enoded
1068         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1069         optional => 1,
1070     },
1071 );
1072
1073 my %queues_fmt = (
1074     queues => {
1075         type => 'integer',
1076         description => "Number of queues.",
1077         minimum => 2,
1078         optional => 1
1079     }
1080 );
1081
1082 my %scsiblock_fmt = (
1083     scsiblock => {
1084         type => 'boolean',
1085         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1086         optional => 1,
1087         default => 0,
1088     },
1089 );
1090
1091 my %ssd_fmt = (
1092     ssd => {
1093         type => 'boolean',
1094         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1095         optional => 1,
1096     },
1097 );
1098
1099 my %wwn_fmt = (
1100     wwn => {
1101         type => 'string',
1102         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1103         format_description => 'wwn',
1104         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1105         optional => 1,
1106     },
1107 );
1108
1109 my $add_throttle_desc = sub {
1110     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1111     my $d = {
1112         type => $type,
1113         format_description => $unit,
1114         description => "Maximum $what in $longunit.",
1115         optional => 1,
1116     };
1117     $d->{minimum} = $minimum if defined($minimum);
1118     $drivedesc_base{$key} = $d;
1119 };
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1122 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1123 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1124 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1130
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1138
1139 # burst lengths
1140 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1146
1147 # legacy support
1148 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1152
1153 my $ide_fmt = {
1154     %drivedesc_base,
1155     %model_fmt,
1156     %ssd_fmt,
1157     %wwn_fmt,
1158 };
1159 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1160
1161 my $idedesc = {
1162     optional => 1,
1163     type => 'string', format => $ide_fmt,
1164     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1165 };
1166 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1167
1168 my $scsi_fmt = {
1169     %drivedesc_base,
1170     %iothread_fmt,
1171     %queues_fmt,
1172     %scsiblock_fmt,
1173     %ssd_fmt,
1174     %wwn_fmt,
1175 };
1176 my $scsidesc = {
1177     optional => 1,
1178     type => 'string', format => $scsi_fmt,
1179     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1182
1183 my $sata_fmt = {
1184     %drivedesc_base,
1185     %ssd_fmt,
1186     %wwn_fmt,
1187 };
1188 my $satadesc = {
1189     optional => 1,
1190     type => 'string', format => $sata_fmt,
1191     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1192 };
1193 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1194
1195 my $virtio_fmt = {
1196     %drivedesc_base,
1197     %iothread_fmt,
1198 };
1199 my $virtiodesc = {
1200     optional => 1,
1201     type => 'string', format => $virtio_fmt,
1202     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1205
1206 my $alldrive_fmt = {
1207     %drivedesc_base,
1208     %iothread_fmt,
1209     %model_fmt,
1210     %queues_fmt,
1211     %scsiblock_fmt,
1212     %ssd_fmt,
1213     %wwn_fmt,
1214 };
1215
1216 my $efidisk_fmt = {
1217     volume => { alias => 'file' },
1218     file => {
1219         type => 'string',
1220         format => 'pve-volume-id-or-qm-path',
1221         default_key => 1,
1222         format_description => 'volume',
1223         description => "The drive's backing volume.",
1224     },
1225     format => get_standard_option('pve-qm-image-format'),
1226     size => {
1227         type => 'string',
1228         format => 'disk-size',
1229         format_description => 'DiskSize',
1230         description => "Disk size. This is purely informational and has no effect.",
1231         optional => 1,
1232     },
1233 };
1234
1235 my $efidisk_desc = {
1236     optional => 1,
1237     type => 'string', format => $efidisk_fmt,
1238     description => "Configure a Disk for storing EFI vars",
1239 };
1240
1241 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1242
1243 my $usb_fmt = {
1244     host => {
1245         default_key => 1,
1246         type => 'string', format => 'pve-qm-usb-device',
1247         format_description => 'HOSTUSBDEVICE|spice',
1248         description => <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1250
1251  'bus-port(.port)*' (decimal numbers) or
1252  'vendor_id:product_id' (hexadeciaml numbers) or
1253  'spice'
1254
1255 You can use the 'lsusb -t' command to list existing usb devices.
1256
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1258
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1260 EODESCR
1261     },
1262     usb3 => {
1263         optional => 1,
1264         type => 'boolean',
1265         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1266         default => 0,
1267     },
1268 };
1269
1270 my $usbdesc = {
1271     optional => 1,
1272     type => 'string', format => $usb_fmt,
1273     description => "Configure an USB device (n is 0 to 4).",
1274 };
1275 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1276
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1278 my $hostpci_fmt = {
1279     host => {
1280         default_key => 1,
1281         type => 'string',
1282         pattern => qr/$PCIRE(;$PCIRE)*/,
1283         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1284         description => <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1287
1288 'bus:dev.func' (hexadecimal numbers)
1289
1290 You can us the 'lspci' command to list existing PCI devices.
1291 EODESCR
1292     },
1293     rombar => {
1294         type => 'boolean',
1295         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1296         optional => 1,
1297         default => 1,
1298     },
1299     romfile => {
1300         type => 'string',
1301         pattern => '[^,;]+',
1302         format_description => 'string',
1303         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1304         optional => 1,
1305     },
1306     pcie => {
1307         type => 'boolean',
1308         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1309         optional => 1,
1310         default => 0,
1311     },
1312     'x-vga' => {
1313         type => 'boolean',
1314         description =>  "Enable vfio-vga device support.",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'mdev' => {
1319         type => 'string',
1320         format_description => 'string',
1321         pattern => '[^/\.:]+',
1322         optional => 1,
1323         description => <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1327 EODESCR
1328     }
1329 };
1330 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1331
1332 my $hostpcidesc = {
1333         optional => 1,
1334         type => 'string', format => 'pve-qm-hostpci',
1335         description => "Map host PCI devices into guest.",
1336         verbose_description =>  <<EODESCR,
1337 Map host PCI devices into guest.
1338
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1346
1347 my $serialdesc = {
1348         optional => 1,
1349         type => 'string',
1350         pattern => '(/dev/.+|socket)',
1351         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1352         verbose_description =>  <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1356
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1358
1359 CAUTION: Experimental! User reported problems with this option.
1360 EODESCR
1361 };
1362
1363 my $paralleldesc= {
1364         optional => 1,
1365         type => 'string',
1366         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1367         description =>  "Map host parallel devices (n is 0 to 2).",
1368         verbose_description =>  <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1370
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1372
1373 CAUTION: Experimental! User reported problems with this option.
1374 EODESCR
1375 };
1376
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1378     $confdesc->{"parallel$i"} = $paralleldesc;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1382     $confdesc->{"serial$i"} = $serialdesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1386     $confdesc->{"hostpci$i"} = $hostpcidesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1390     $drivename_hash->{"ide$i"} = 1;
1391     $confdesc->{"ide$i"} = $idedesc;
1392 }
1393
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1395     $drivename_hash->{"sata$i"} = 1;
1396     $confdesc->{"sata$i"} = $satadesc;
1397 }
1398
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1400     $drivename_hash->{"scsi$i"} = 1;
1401     $confdesc->{"scsi$i"} = $scsidesc ;
1402 }
1403
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1405     $drivename_hash->{"virtio$i"} = 1;
1406     $confdesc->{"virtio$i"} = $virtiodesc;
1407 }
1408
1409 $drivename_hash->{efidisk0} = 1;
1410 $confdesc->{efidisk0} = $efidisk_desc;
1411
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1413     $confdesc->{"usb$i"} = $usbdesc;
1414 }
1415
1416 my $unuseddesc = {
1417     optional => 1,
1418     type => 'string', format => 'pve-volume-id',
1419     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1420 };
1421
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1423     $confdesc->{"unused$i"} = $unuseddesc;
1424 }
1425
1426 my $kvm_api_version = 0;
1427
1428 sub kvm_version {
1429     return $kvm_api_version if $kvm_api_version;
1430
1431     open my $fh, '<', '/dev/kvm'
1432         or return undef;
1433
1434     # 0xae00 => KVM_GET_API_VERSION
1435     $kvm_api_version = ioctl($fh, 0xae00, 0);
1436
1437     return $kvm_api_version;
1438 }
1439
1440 my $kvm_user_version;
1441
1442 sub kvm_user_version {
1443
1444     return $kvm_user_version if $kvm_user_version;
1445
1446     $kvm_user_version = 'unknown';
1447
1448     my $code = sub {
1449         my $line = shift;
1450         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451             $kvm_user_version = $2;
1452         }
1453     };
1454
1455     eval { run_command("kvm -version", outfunc => $code); };
1456     warn $@ if $@;
1457
1458     return $kvm_user_version;
1459
1460 }
1461
1462 sub kernel_has_vhost_net {
1463     return -c '/dev/vhost-net';
1464 }
1465
1466 sub valid_drive_names {
1467     # order is important - used to autoselect boot disk
1468     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1472             'efidisk0');
1473 }
1474
1475 sub is_valid_drivename {
1476     my $dev = shift;
1477
1478     return defined($drivename_hash->{$dev});
1479 }
1480
1481 sub option_exists {
1482     my $key = shift;
1483     return defined($confdesc->{$key});
1484 }
1485
1486 sub nic_models {
1487     return $nic_model_list;
1488 }
1489
1490 sub os_list_description {
1491
1492     return {
1493         other => 'Other',
1494         wxp => 'Windows XP',
1495         w2k => 'Windows 2000',
1496         w2k3 =>, 'Windows 2003',
1497         w2k8 => 'Windows 2008',
1498         wvista => 'Windows Vista',
1499         win7 => 'Windows 7',
1500         win8 => 'Windows 8/2012',
1501         win10 => 'Windows 10/2016',
1502         l24 => 'Linux 2.4',
1503         l26 => 'Linux 2.6',
1504     };
1505 }
1506
1507 my $cdrom_path;
1508
1509 sub get_cdrom_path {
1510
1511     return  $cdrom_path if $cdrom_path;
1512
1513     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1514     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1515     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1516 }
1517
1518 sub get_iso_path {
1519     my ($storecfg, $vmid, $cdrom) = @_;
1520
1521     if ($cdrom eq 'cdrom') {
1522         return get_cdrom_path();
1523     } elsif ($cdrom eq 'none') {
1524         return '';
1525     } elsif ($cdrom =~ m|^/|) {
1526         return $cdrom;
1527     } else {
1528         return PVE::Storage::path($storecfg, $cdrom);
1529     }
1530 }
1531
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id {
1534     my ($vmid, $file, $media) = @_;
1535
1536      if (!($file eq 'none' || $file eq 'cdrom' ||
1537           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1538
1539         return undef if $file =~ m|/|;
1540
1541         if ($media && $media eq 'cdrom') {
1542             $file = "local:iso/$file";
1543         } else {
1544             $file = "local:$vmid/$file";
1545         }
1546     }
1547
1548     return $file;
1549 }
1550
1551 sub verify_media_type {
1552     my ($opt, $vtype, $media) = @_;
1553
1554     return if !$media;
1555
1556     my $etype;
1557     if ($media eq 'disk') {
1558         $etype = 'images';
1559     } elsif ($media eq 'cdrom') {
1560         $etype = 'iso';
1561     } else {
1562         die "internal error";
1563     }
1564
1565     return if ($vtype eq $etype);
1566
1567     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1568 }
1569
1570 sub cleanup_drive_path {
1571     my ($opt, $storecfg, $drive) = @_;
1572
1573     # try to convert filesystem paths to volume IDs
1574
1575     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1576         ($drive->{file} !~ m|^/dev/.+|) &&
1577         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1578         ($drive->{file} !~ m/^\d+$/)) {
1579         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1580         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1582         verify_media_type($opt, $vtype, $drive->{media});
1583         $drive->{file} = $volid;
1584     }
1585
1586     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1587 }
1588
1589 sub parse_hotplug_features {
1590     my ($data) = @_;
1591
1592     my $res = {};
1593
1594     return $res if $data eq '0';
1595
1596     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1597
1598     foreach my $feature (PVE::Tools::split_list($data)) {
1599         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1600             $res->{$1} = 1;
1601         } else {
1602             die "invalid hotplug feature '$feature'\n";
1603         }
1604     }
1605     return $res;
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1609 sub pve_verify_hotplug_features {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_hotplug_features($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse hotplug option\n";
1617 }
1618
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 #        [,iothread=on][,serial=serial][,model=model]
1624
1625 sub parse_drive {
1626     my ($key, $data) = @_;
1627
1628     my ($interface, $index);
1629
1630     if ($key =~ m/^([^\d]+)(\d+)$/) {
1631         $interface = $1;
1632         $index = $2;
1633     } else {
1634         return undef;
1635     }
1636
1637     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1638                                      : $confdesc->{$key}->{format};
1639     if (!$desc) {
1640         warn "invalid drive key: $key\n";
1641         return undef;
1642     }
1643     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1644     return undef if !$res;
1645     $res->{interface} = $interface;
1646     $res->{index} = $index;
1647
1648     my $error = 0;
1649     foreach my $opt (qw(bps bps_rd bps_wr)) {
1650         if (my $bps = defined(delete $res->{$opt})) {
1651             if (defined($res->{"m$opt"})) {
1652                 warn "both $opt and m$opt specified\n";
1653                 ++$error;
1654                 next;
1655             }
1656             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1657         }
1658     }
1659
1660     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661     for my $requirement (
1662         [mbps_max => 'mbps'],
1663         [mbps_rd_max => 'mbps_rd'],
1664         [mbps_wr_max => 'mbps_wr'],
1665         [miops_max => 'miops'],
1666         [miops_rd_max => 'miops_rd'],
1667         [miops_wr_max => 'miops_wr'],
1668         [bps_max_length => 'mbps_max'],
1669         [bps_rd_max_length => 'mbps_rd_max'],
1670         [bps_wr_max_length => 'mbps_wr_max'],
1671         [iops_max_length => 'iops_max'],
1672         [iops_rd_max_length => 'iops_rd_max'],
1673         [iops_wr_max_length => 'iops_wr_max']) {
1674         my ($option, $requires) = @$requirement;
1675         if ($res->{$option} && !$res->{$requires}) {
1676             warn "$option requires $requires\n";
1677             ++$error;
1678         }
1679     }
1680
1681     return undef if $error;
1682
1683     return undef if $res->{mbps_rd} && $res->{mbps};
1684     return undef if $res->{mbps_wr} && $res->{mbps};
1685     return undef if $res->{iops_rd} && $res->{iops};
1686     return undef if $res->{iops_wr} && $res->{iops};
1687
1688     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1689         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1690         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1691         return undef if $res->{interface} eq 'virtio';
1692     }
1693
1694     if (my $size = $res->{size}) {
1695         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1696     }
1697
1698     return $res;
1699 }
1700
1701 sub print_drive {
1702     my ($vmid, $drive) = @_;
1703     my $data = { %$drive };
1704     delete $data->{$_} for qw(index interface);
1705     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1706 }
1707
1708 sub scsi_inquiry {
1709     my($fh, $noerr) = @_;
1710
1711     my $SG_IO = 0x2285;
1712     my $SG_GET_VERSION_NUM = 0x2282;
1713
1714     my $versionbuf = "\x00" x 8;
1715     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1716     if (!$ret) {
1717         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1718         return undef;
1719     }
1720     my $version = unpack("I", $versionbuf);
1721     if ($version < 30000) {
1722         die "scsi generic interface too old\n"  if !$noerr;
1723         return undef;
1724     }
1725
1726     my $buf = "\x00" x 36;
1727     my $sensebuf = "\x00" x 8;
1728     my $cmd = pack("C x3 C x1", 0x12, 36);
1729
1730     # see /usr/include/scsi/sg.h
1731     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1732
1733     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734                       length($sensebuf), 0, length($buf), $buf,
1735                       $cmd, $sensebuf, 6000);
1736
1737     $ret = ioctl($fh, $SG_IO, $packet);
1738     if (!$ret) {
1739         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1740         return undef;
1741     }
1742
1743     my @res = unpack($sg_io_hdr_t, $packet);
1744     if ($res[17] || $res[18]) {
1745         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my $res = {};
1750     (my $byte0, my $byte1, $res->{vendor},
1751      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1752
1753     $res->{removable} = $byte1 & 128 ? 1 : 0;
1754     $res->{type} = $byte0 & 31;
1755
1756     return $res;
1757 }
1758
1759 sub path_is_scsi {
1760     my ($path) = @_;
1761
1762     my $fh = IO::File->new("+<$path") || return undef;
1763     my $res = scsi_inquiry($fh, 1);
1764     close($fh);
1765
1766     return $res;
1767 }
1768
1769 sub machine_type_is_q35 {
1770     my ($conf) = @_;
1771
1772     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1773 }
1774
1775 sub print_tabletdevice_full {
1776     my ($conf, $arch) = @_;
1777
1778     my $q35 = machine_type_is_q35($conf);
1779
1780     # we use uhci for old VMs because tablet driver was buggy in older qemu
1781     my $usbbus;
1782     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1783         $usbbus = 'ehci';
1784     } else {
1785         $usbbus = 'uhci';
1786     }
1787
1788     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1789 }
1790
1791 sub print_keyboarddevice_full {
1792     my ($conf, $arch, $machine) = @_;
1793
1794     return undef if $arch ne 'aarch64';
1795
1796     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1797 }
1798
1799 sub print_drivedevice_full {
1800     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1801
1802     my $device = '';
1803     my $maxdev = 0;
1804
1805     if ($drive->{interface} eq 'virtio') {
1806         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1809     } elsif ($drive->{interface} eq 'scsi') {
1810
1811         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1812         my $unit = $drive->{index} % $maxdev;
1813         my $devicetype = 'hd';
1814         my $path = '';
1815         if (drive_is_cdrom($drive)) {
1816             $devicetype = 'cd';
1817         } else {
1818             if ($drive->{file} =~ m|^/|) {
1819                 $path = $drive->{file};
1820                 if (my $info = path_is_scsi($path)) {
1821                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1822                         $devicetype = 'block';
1823                     } elsif ($info->{type} == 1) { # tape
1824                         $devicetype = 'generic';
1825                     }
1826                 }
1827             } else {
1828                  $path = PVE::Storage::path($storecfg, $drive->{file});
1829             }
1830
1831             if($path =~ m/^iscsi\:\/\//){
1832                 $devicetype = 'generic';
1833             }
1834         }
1835
1836         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1837             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1838         } else {
1839             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1840         }
1841
1842         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843             $device .= ",rotation_rate=1";
1844         }
1845         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1846
1847     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1848         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1849         my $controller = int($drive->{index} / $maxdev);
1850         my $unit = $drive->{index} % $maxdev;
1851         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1852
1853         $device = "ide-$devicetype";
1854         if ($drive->{interface} eq 'ide') {
1855             $device .= ",bus=ide.$controller,unit=$unit";
1856         } else {
1857             $device .= ",bus=ahci$controller.$unit";
1858         }
1859         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1860
1861         if ($devicetype eq 'hd') {
1862             if (my $model = $drive->{model}) {
1863                 $model = URI::Escape::uri_unescape($model);
1864                 $device .= ",model=$model";
1865             }
1866             if ($drive->{ssd}) {
1867                 $device .= ",rotation_rate=1";
1868             }
1869         }
1870         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1871     } elsif ($drive->{interface} eq 'usb') {
1872         die "implement me";
1873         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1874     } else {
1875         die "unsupported interface type";
1876     }
1877
1878     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1879
1880     if (my $serial = $drive->{serial}) {
1881         $serial = URI::Escape::uri_unescape($serial);
1882         $device .= ",serial=$serial";
1883     }
1884
1885
1886     return $device;
1887 }
1888
1889 sub get_initiator_name {
1890     my $initiator;
1891
1892     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893     while (defined(my $line = <$fh>)) {
1894         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1895         $initiator = $1;
1896         last;
1897     }
1898     $fh->close();
1899
1900     return $initiator;
1901 }
1902
1903 sub print_drive_full {
1904     my ($storecfg, $vmid, $drive) = @_;
1905
1906     my $path;
1907     my $volid = $drive->{file};
1908     my $format;
1909
1910     if (drive_is_cdrom($drive)) {
1911         $path = get_iso_path($storecfg, $vmid, $volid);
1912     } else {
1913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1914         if ($storeid) {
1915             $path = PVE::Storage::path($storecfg, $volid);
1916             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1917             $format = qemu_img_format($scfg, $volname);
1918         } else {
1919             $path = $volid;
1920             $format = "raw";
1921         }
1922    }
1923
1924     my $opts = '';
1925     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926     foreach my $o (@qemu_drive_options) {
1927         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1928     }
1929
1930     # snapshot only accepts on|off
1931     if (defined($drive->{snapshot})) {
1932         my $v = $drive->{snapshot} ? 'on' : 'off';
1933         $opts .= ",snapshot=$v";
1934     }
1935
1936     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1937         my ($dir, $qmpname) = @$type;
1938         if (my $v = $drive->{"mbps$dir"}) {
1939             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1940         }
1941         if (my $v = $drive->{"mbps${dir}_max"}) {
1942             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1943         }
1944         if (my $v = $drive->{"bps${dir}_max_length"}) {
1945             $opts .= ",throttling.bps$qmpname-max-length=$v";
1946         }
1947         if (my $v = $drive->{"iops${dir}"}) {
1948             $opts .= ",throttling.iops$qmpname=$v";
1949         }
1950         if (my $v = $drive->{"iops${dir}_max"}) {
1951             $opts .= ",throttling.iops$qmpname-max=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}_max_length"}) {
1954             $opts .= ",throttling.iops$qmpname-max-length=$v";
1955         }
1956     }
1957
1958     $opts .= ",format=$format" if $format && !$drive->{format};
1959
1960     my $cache_direct = 0;
1961
1962     if (my $cache = $drive->{cache}) {
1963         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964     } elsif (!drive_is_cdrom($drive)) {
1965         $opts .= ",cache=none";
1966         $cache_direct = 1;
1967     }
1968
1969     # aio native works only with O_DIRECT
1970     if (!$drive->{aio}) {
1971         if($cache_direct) {
1972             $opts .= ",aio=native";
1973         } else {
1974             $opts .= ",aio=threads";
1975         }
1976     }
1977
1978     if (!drive_is_cdrom($drive)) {
1979         my $detectzeroes;
1980         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1981             $detectzeroes = 'off';
1982         } elsif ($drive->{discard}) {
1983             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1984         } else {
1985             # This used to be our default with discard not being specified:
1986             $detectzeroes = 'on';
1987         }
1988         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1989     }
1990
1991     my $pathinfo = $path ? "file=$path," : '';
1992
1993     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1994 }
1995
1996 sub print_netdevice_full {
1997     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1998
1999     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2000
2001     my $device = $net->{model};
2002     if ($net->{model} eq 'virtio') {
2003          $device = 'virtio-net-pci';
2004      };
2005
2006     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2007     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2009         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010         my $vectors = $net->{queues} * 2 + 2;
2011         $tmpstr .= ",vectors=$vectors,mq=on";
2012     }
2013     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2014
2015     if ($use_old_bios_files) {
2016         my $romfile;
2017         if ($device eq 'virtio-net-pci') {
2018             $romfile = 'pxe-virtio.rom';
2019         } elsif ($device eq 'e1000') {
2020             $romfile = 'pxe-e1000.rom';
2021         } elsif ($device eq 'ne2k') {
2022             $romfile = 'pxe-ne2k_pci.rom';
2023         } elsif ($device eq 'pcnet') {
2024             $romfile = 'pxe-pcnet.rom';
2025         } elsif ($device eq 'rtl8139') {
2026             $romfile = 'pxe-rtl8139.rom';
2027         }
2028         $tmpstr .= ",romfile=$romfile" if $romfile;
2029     }
2030
2031     return $tmpstr;
2032 }
2033
2034 sub print_netdev_full {
2035     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2036
2037     my $i = '';
2038     if ($netid =~ m/^net(\d+)$/) {
2039         $i = int($1);
2040     }
2041
2042     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2043
2044     my $ifname = "tap${vmid}i$i";
2045
2046     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047     die "interface name '$ifname' is too long (max 15 character)\n"
2048         if length($ifname) >= 16;
2049
2050     my $vhostparam = '';
2051     if (is_native($arch)) {
2052         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2053     }
2054
2055     my $vmname = $conf->{name} || "vm$vmid";
2056
2057     my $netdev = "";
2058     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2059
2060     if ($net->{bridge}) {
2061         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2062     } else {
2063         $netdev = "type=user,id=$netid,hostname=$vmname";
2064     }
2065
2066     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2067
2068     return $netdev;
2069 }
2070
2071
2072 sub print_cpu_device {
2073     my ($conf, $id) = @_;
2074
2075     my $kvm = $conf->{kvm} // 1;
2076     my $cpu = $kvm ? "kvm64" : "qemu64";
2077     if (my $cputype = $conf->{cpu}) {
2078         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2079             or die "Cannot parse cpu description: $cputype\n";
2080         $cpu = $cpuconf->{cputype};
2081     }
2082
2083     my $cores = $conf->{cores} || 1;
2084
2085     my $current_core = ($id - 1) % $cores;
2086     my $current_socket = int(($id - 1 - $current_core)/$cores);
2087
2088     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2089 }
2090
2091 my $vga_map = {
2092     'cirrus' => 'cirrus-vga',
2093     'std' => 'VGA',
2094     'vmware' => 'vmware-svga',
2095     'virtio' => 'virtio-vga',
2096 };
2097
2098 sub print_vga_device {
2099     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2100
2101     my $type = $vga_map->{$vga->{type}};
2102     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103         $type = 'virtio-gpu';
2104     }
2105     my $vgamem_mb = $vga->{memory};
2106     if ($qxlnum) {
2107         $type = $id ? 'qxl' : 'qxl-vga';
2108     }
2109     die "no devicetype for $vga->{type}\n" if !$type;
2110
2111     my $memory = "";
2112     if ($vgamem_mb) {
2113         if ($vga->{type} eq 'virtio') {
2114             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2115             $memory = ",max_hostmem=$bytes";
2116         } elsif ($qxlnum) {
2117             # from https://www.spice-space.org/multiple-monitors.html
2118             $memory = ",vgamem_mb=$vga->{memory}";
2119             my $ram = $vgamem_mb * 4;
2120             my $vram = $vgamem_mb * 2;
2121             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2122         } else {
2123             $memory = ",vgamem_mb=$vga->{memory}";
2124         }
2125     } elsif ($qxlnum && $id) {
2126         $memory = ",ram_size=67108864,vram_size=33554432";
2127     }
2128
2129     my $q35 = machine_type_is_q35($conf);
2130     my $vgaid = "vga" . ($id // '');
2131     my $pciaddr;
2132
2133     if ($q35 && $vgaid eq 'vga') {
2134         # the first display uses pcie.0 bus on q35 machines
2135         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2136     } else {
2137         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2138     }
2139
2140     return "$type,id=${vgaid}${memory}${pciaddr}";
2141 }
2142
2143 sub drive_is_cloudinit {
2144     my ($drive) = @_;
2145     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2146 }
2147
2148 sub drive_is_cdrom {
2149     my ($drive, $exclude_cloudinit) = @_;
2150
2151     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2152
2153     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2154
2155 }
2156
2157 sub parse_number_sets {
2158     my ($set) = @_;
2159     my $res = [];
2160     foreach my $part (split(/;/, $set)) {
2161         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163             push @$res, [ $1, $2 ];
2164         } else {
2165             die "invalid range: $part\n";
2166         }
2167     }
2168     return $res;
2169 }
2170
2171 sub parse_numa {
2172     my ($data) = @_;
2173
2174     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2175     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2176     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2177     return $res;
2178 }
2179
2180 sub parse_hostpci {
2181     my ($value) = @_;
2182
2183     return undef if !$value;
2184
2185     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2186
2187     my @idlist = split(/;/, $res->{host});
2188     delete $res->{host};
2189     foreach my $id (@idlist) {
2190         if ($id =~ m/\./) { # full id 00:00.1
2191             push @{$res->{pciid}}, {
2192                 id => $id,
2193             };
2194         } else { # partial id 00:00
2195             $res->{pciid} = PVE::SysFSTools::lspci($id);
2196         }
2197     }
2198     return $res;
2199 }
2200
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2202 sub parse_net {
2203     my ($data) = @_;
2204
2205     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2206     if ($@) {
2207         warn $@;
2208         return undef;
2209     }
2210     if (!defined($res->{macaddr})) {
2211         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2212         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2213     }
2214     return $res;
2215 }
2216
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig {
2219     my ($data) = @_;
2220
2221     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2222     if ($@) {
2223         warn $@;
2224         return undef;
2225     }
2226
2227     if ($res->{gw} && !$res->{ip}) {
2228         warn 'gateway specified without specifying an IP address';
2229         return undef;
2230     }
2231     if ($res->{gw6} && !$res->{ip6}) {
2232         warn 'IPv6 gateway specified without specifying an IPv6 address';
2233         return undef;
2234     }
2235     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2236         warn 'gateway specified together with DHCP';
2237         return undef;
2238     }
2239     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2240         # gw6 + auto/dhcp
2241         warn "IPv6 gateway specified together with $res->{ip6} address";
2242         return undef;
2243     }
2244
2245     if (!$res->{ip} && !$res->{ip6}) {
2246         return { ip => 'dhcp', ip6 => 'dhcp' };
2247     }
2248
2249     return $res;
2250 }
2251
2252 sub print_net {
2253     my $net = shift;
2254
2255     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2256 }
2257
2258 sub add_random_macs {
2259     my ($settings) = @_;
2260
2261     foreach my $opt (keys %$settings) {
2262         next if $opt !~ m/^net(\d+)$/;
2263         my $net = parse_net($settings->{$opt});
2264         next if !$net;
2265         $settings->{$opt} = print_net($net);
2266     }
2267 }
2268
2269 sub vm_is_volid_owner {
2270     my ($storecfg, $vmid, $volid) = @_;
2271
2272     if ($volid !~  m|^/|) {
2273         my ($path, $owner);
2274         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2275         if ($owner && ($owner == $vmid)) {
2276             return 1;
2277         }
2278     }
2279
2280     return undef;
2281 }
2282
2283 sub split_flagged_list {
2284     my $text = shift || '';
2285     $text =~ s/[,;]/ /g;
2286     $text =~ s/^\s+//;
2287     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2288 }
2289
2290 sub join_flagged_list {
2291     my ($how, $lst) = @_;
2292     join $how, map { $lst->{$_} . $_ } keys %$lst;
2293 }
2294
2295 sub vmconfig_delete_pending_option {
2296     my ($conf, $key, $force) = @_;
2297
2298     delete $conf->{pending}->{$key};
2299     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2300     $pending_delete_hash->{$key} = $force ? '!' : '';
2301     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2302 }
2303
2304 sub vmconfig_undelete_pending_option {
2305     my ($conf, $key) = @_;
2306
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     delete $pending_delete_hash->{$key};
2309
2310     if (%$pending_delete_hash) {
2311         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2312     } else {
2313         delete $conf->{pending}->{delete};
2314     }
2315 }
2316
2317 sub vmconfig_register_unused_drive {
2318     my ($storecfg, $vmid, $conf, $drive) = @_;
2319
2320     if (drive_is_cloudinit($drive)) {
2321         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2322         warn $@ if $@;
2323     } elsif (!drive_is_cdrom($drive)) {
2324         my $volid = $drive->{file};
2325         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2326             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2327         }
2328     }
2329 }
2330
2331 sub vmconfig_cleanup_pending {
2332     my ($conf) = @_;
2333
2334     # remove pending changes when nothing changed
2335     my $changes;
2336     foreach my $opt (keys %{$conf->{pending}}) {
2337         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2338             $changes = 1;
2339             delete $conf->{pending}->{$opt};
2340         }
2341     }
2342
2343     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2344     my $pending_delete_hash = {};
2345     while (my ($opt, $force) = each %$current_delete_hash) {
2346         if (defined($conf->{$opt})) {
2347             $pending_delete_hash->{$opt} = $force;
2348         } else {
2349             $changes = 1;
2350         }
2351     }
2352
2353     if (%$pending_delete_hash) {
2354         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2355     } else {
2356         delete $conf->{pending}->{delete};
2357     }
2358
2359     return $changes;
2360 }
2361
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2363 my $smbios1_fmt = {
2364     uuid => {
2365         type => 'string',
2366         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367         format_description => 'UUID',
2368         description => "Set SMBIOS1 UUID.",
2369         optional => 1,
2370     },
2371     version => {
2372         type => 'string',
2373         pattern => '[A-Za-z0-9+\/]+={0,2}',
2374         format_description => 'Base64 encoded string',
2375         description => "Set SMBIOS1 version.",
2376         optional => 1,
2377     },
2378     serial => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 serial number.",
2383         optional => 1,
2384     },
2385     manufacturer => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 manufacturer.",
2390         optional => 1,
2391     },
2392     product => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 product ID.",
2397         optional => 1,
2398     },
2399     sku => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 SKU string.",
2404         optional => 1,
2405     },
2406     family => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 family string.",
2411         optional => 1,
2412     },
2413     base64 => {
2414         type => 'boolean',
2415         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2416         optional => 1,
2417     },
2418 };
2419
2420 sub parse_smbios1 {
2421     my ($data) = @_;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub print_smbios1 {
2429     my ($smbios1) = @_;
2430     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2431 }
2432
2433 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2434
2435 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2436 sub verify_bootdisk {
2437     my ($value, $noerr) = @_;
2438
2439     return $value if is_valid_drivename($value);
2440
2441     return undef if $noerr;
2442
2443     die "invalid boot disk '$value'\n";
2444 }
2445
2446 sub parse_watchdog {
2447     my ($value) = @_;
2448
2449     return undef if !$value;
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2452     warn $@ if $@;
2453     return $res;
2454 }
2455
2456 sub parse_guest_agent {
2457     my ($value) = @_;
2458
2459     return {} if !defined($value->{agent});
2460
2461     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2462     warn $@ if $@;
2463
2464     # if the agent is disabled ignore the other potentially set properties
2465     return {} if !$res->{enabled};
2466     return $res;
2467 }
2468
2469 sub parse_vga {
2470     my ($value) = @_;
2471
2472     return {} if !$value;
2473     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2474     warn $@ if $@;
2475     return $res;
2476 }
2477
2478 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2479 sub verify_usb_device {
2480     my ($value, $noerr) = @_;
2481
2482     return $value if parse_usb_device($value);
2483
2484     return undef if $noerr;
2485
2486     die "unable to parse usb device\n";
2487 }
2488
2489 # add JSON properties for create and set function
2490 sub json_config_properties {
2491     my $prop = shift;
2492
2493     foreach my $opt (keys %$confdesc) {
2494         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495         $prop->{$opt} = $confdesc->{$opt};
2496     }
2497
2498     return $prop;
2499 }
2500
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties {
2503
2504     return dclone($confdesc_cloudinit);
2505 }
2506
2507 sub check_type {
2508     my ($key, $value) = @_;
2509
2510     die "unknown setting '$key'\n" if !$confdesc->{$key};
2511
2512     my $type = $confdesc->{$key}->{type};
2513
2514     if (!defined($value)) {
2515         die "got undefined value\n";
2516     }
2517
2518     if ($value =~ m/[\n\r]/) {
2519         die "property contains a line feed\n";
2520     }
2521
2522     if ($type eq 'boolean') {
2523         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525         die "type check ('boolean') failed - got '$value'\n";
2526     } elsif ($type eq 'integer') {
2527         return int($1) if $value =~ m/^(\d+)$/;
2528         die "type check ('integer') failed - got '$value'\n";
2529     } elsif ($type eq 'number') {
2530         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531         die "type check ('number') failed - got '$value'\n";
2532     } elsif ($type eq 'string') {
2533         if (my $fmt = $confdesc->{$key}->{format}) {
2534             PVE::JSONSchema::check_format($fmt, $value);
2535             return $value;
2536         }
2537         $value =~ s/^\"(.*)\"$/$1/;
2538         return $value;
2539     } else {
2540         die "internal error"
2541     }
2542 }
2543
2544 sub touch_config {
2545     my ($vmid) = @_;
2546
2547     my $conf = PVE::QemuConfig->config_file($vmid);
2548     utime undef, undef, $conf;
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conffile = PVE::QemuConfig->config_file($vmid);
2555
2556     my $conf = PVE::QemuConfig->load_config($vmid);
2557
2558     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2559
2560     if ($conf->{template}) {
2561         # check if any base image is still used by a linked clone
2562         foreach_drive($conf, sub {
2563                 my ($ds, $drive) = @_;
2564
2565                 return if drive_is_cdrom($drive);
2566
2567                 my $volid = $drive->{file};
2568
2569                 return if !$volid || $volid =~ m|^/|;
2570
2571                 die "base volume '$volid' is still in use by linked cloned\n"
2572                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2573
2574         });
2575     }
2576
2577     # only remove disks owned by this VM
2578     foreach_drive($conf, sub {
2579         my ($ds, $drive) = @_;
2580
2581         return if drive_is_cdrom($drive, 1);
2582
2583         my $volid = $drive->{file};
2584
2585         return if !$volid || $volid =~ m|^/|;
2586
2587         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2588         return if !$path || !$owner || ($owner != $vmid);
2589
2590         eval {
2591             PVE::Storage::vdisk_free($storecfg, $volid);
2592         };
2593         warn "Could not remove disk '$volid', check manually: $@" if $@;
2594
2595     });
2596
2597     if ($keep_empty_config) {
2598         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2599     } else {
2600         unlink $conffile;
2601     }
2602
2603     # also remove unused disk
2604     eval {
2605         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2606
2607         eval {
2608             PVE::Storage::foreach_volid($dl, sub {
2609                 my ($volid, $sid, $volname, $d) = @_;
2610                 PVE::Storage::vdisk_free($storecfg, $volid);
2611             });
2612         };
2613         warn $@ if $@;
2614
2615     };
2616     warn $@ if $@;
2617 }
2618
2619 sub parse_vm_config {
2620     my ($filename, $raw) = @_;
2621
2622     return undef if !defined($raw);
2623
2624     my $res = {
2625         digest => Digest::SHA::sha1_hex($raw),
2626         snapshots => {},
2627         pending => {},
2628     };
2629
2630     $filename =~ m|/qemu-server/(\d+)\.conf$|
2631         || die "got strange filename '$filename'";
2632
2633     my $vmid = $1;
2634
2635     my $conf = $res;
2636     my $descr;
2637     my $section = '';
2638
2639     my @lines = split(/\n/, $raw);
2640     foreach my $line (@lines) {
2641         next if $line =~ m/^\s*$/;
2642
2643         if ($line =~ m/^\[PENDING\]\s*$/i) {
2644             $section = 'pending';
2645             if (defined($descr)) {
2646                 $descr =~ s/\s+$//;
2647                 $conf->{description} = $descr;
2648             }
2649             $descr = undef;
2650             $conf = $res->{$section} = {};
2651             next;
2652
2653         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2654             $section = $1;
2655             if (defined($descr)) {
2656                 $descr =~ s/\s+$//;
2657                 $conf->{description} = $descr;
2658             }
2659             $descr = undef;
2660             $conf = $res->{snapshots}->{$section} = {};
2661             next;
2662         }
2663
2664         if ($line =~ m/^\#(.*)\s*$/) {
2665             $descr = '' if !defined($descr);
2666             $descr .= PVE::Tools::decode_text($1) . "\n";
2667             next;
2668         }
2669
2670         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($2);
2673         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674             $conf->{snapstate} = $1;
2675         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2676             my $key = $1;
2677             my $value = $2;
2678             $conf->{$key} = $value;
2679         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2680             my $value = $1;
2681             if ($section eq 'pending') {
2682                 $conf->{delete} = $value; # we parse this later
2683             } else {
2684                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2685             }
2686         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2687             my $key = $1;
2688             my $value = $2;
2689             eval { $value = check_type($key, $value); };
2690             if ($@) {
2691                 warn "vm $vmid - unable to parse value of '$key' - $@";
2692             } else {
2693                 $key = 'ide2' if $key eq 'cdrom';
2694                 my $fmt = $confdesc->{$key}->{format};
2695                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696                     my $v = parse_drive($key, $value);
2697                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2698                         $v->{file} = $volid;
2699                         $value = print_drive($vmid, $v);
2700                     } else {
2701                         warn "vm $vmid - unable to parse value of '$key'\n";
2702                         next;
2703                     }
2704                 }
2705
2706                 $conf->{$key} = $value;
2707             }
2708         }
2709     }
2710
2711     if (defined($descr)) {
2712         $descr =~ s/\s+$//;
2713         $conf->{description} = $descr;
2714     }
2715     delete $res->{snapstate}; # just to be sure
2716
2717     return $res;
2718 }
2719
2720 sub write_vm_config {
2721     my ($filename, $conf) = @_;
2722
2723     delete $conf->{snapstate}; # just to be sure
2724
2725     if ($conf->{cdrom}) {
2726         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2727         $conf->{ide2} = $conf->{cdrom};
2728         delete $conf->{cdrom};
2729     }
2730
2731     # we do not use 'smp' any longer
2732     if ($conf->{sockets}) {
2733         delete $conf->{smp};
2734     } elsif ($conf->{smp}) {
2735         $conf->{sockets} = $conf->{smp};
2736         delete $conf->{cores};
2737         delete $conf->{smp};
2738     }
2739
2740     my $used_volids = {};
2741
2742     my $cleanup_config = sub {
2743         my ($cref, $pending, $snapname) = @_;
2744
2745         foreach my $key (keys %$cref) {
2746             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747                 $key eq 'snapstate' || $key eq 'pending';
2748             my $value = $cref->{$key};
2749             if ($key eq 'delete') {
2750                 die "propertry 'delete' is only allowed in [PENDING]\n"
2751                     if !$pending;
2752                 # fixme: check syntax?
2753                 next;
2754             }
2755             eval { $value = check_type($key, $value); };
2756             die "unable to parse value of '$key' - $@" if $@;
2757
2758             $cref->{$key} = $value;
2759
2760             if (!$snapname && is_valid_drivename($key)) {
2761                 my $drive = parse_drive($key, $value);
2762                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2763             }
2764         }
2765     };
2766
2767     &$cleanup_config($conf);
2768
2769     &$cleanup_config($conf->{pending}, 1);
2770
2771     foreach my $snapname (keys %{$conf->{snapshots}}) {
2772         die "internal error" if $snapname eq 'pending';
2773         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2774     }
2775
2776     # remove 'unusedX' settings if we re-add a volume
2777     foreach my $key (keys %$conf) {
2778         my $value = $conf->{$key};
2779         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780             delete $conf->{$key};
2781         }
2782     }
2783
2784     my $generate_raw_config = sub {
2785         my ($conf, $pending) = @_;
2786
2787         my $raw = '';
2788
2789         # add description as comment to top of file
2790         if (defined(my $descr = $conf->{description})) {
2791             if ($descr) {
2792                 foreach my $cl (split(/\n/, $descr)) {
2793                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2794                 }
2795             } else {
2796                 $raw .= "#\n" if $pending;
2797             }
2798         }
2799
2800         foreach my $key (sort keys %$conf) {
2801             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802             $raw .= "$key: $conf->{$key}\n";
2803         }
2804         return $raw;
2805     };
2806
2807     my $raw = &$generate_raw_config($conf);
2808
2809     if (scalar(keys %{$conf->{pending}})){
2810         $raw .= "\n[PENDING]\n";
2811         $raw .= &$generate_raw_config($conf->{pending}, 1);
2812     }
2813
2814     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2815         $raw .= "\n[$snapname]\n";
2816         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2817     }
2818
2819     return $raw;
2820 }
2821
2822 sub load_defaults {
2823
2824     my $res = {};
2825
2826     # we use static defaults from our JSON schema configuration
2827     foreach my $key (keys %$confdesc) {
2828         if (defined(my $default = $confdesc->{$key}->{default})) {
2829             $res->{$key} = $default;
2830         }
2831     }
2832
2833     return $res;
2834 }
2835
2836 sub config_list {
2837     my $vmlist = PVE::Cluster::get_vmlist();
2838     my $res = {};
2839     return $res if !$vmlist || !$vmlist->{ids};
2840     my $ids = $vmlist->{ids};
2841
2842     foreach my $vmid (keys %$ids) {
2843         my $d = $ids->{$vmid};
2844         next if !$d->{node} || $d->{node} ne $nodename;
2845         next if !$d->{type} || $d->{type} ne 'qemu';
2846         $res->{$vmid}->{exists} = 1;
2847     }
2848     return $res;
2849 }
2850
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources {
2853     my ($conf, $noerr) = @_;
2854
2855     my @loc_res = ();
2856
2857     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2858     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2859
2860     push @loc_res, "ivshmem" if $conf->{ivshmem};
2861
2862     foreach my $k (keys %$conf) {
2863         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864         # sockets are safe: they will recreated be on the target side post-migrate
2865         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2867     }
2868
2869     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2870
2871     return \@loc_res;
2872 }
2873
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability {
2876     my ($storecfg, $conf, $node) = @_;
2877
2878     foreach_drive($conf, sub {
2879         my ($ds, $drive) = @_;
2880
2881         my $volid = $drive->{file};
2882         return if !$volid;
2883
2884         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2885         return if !$sid;
2886
2887         # check if storage is available on both nodes
2888         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2889         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2890    });
2891 }
2892
2893 # list nodes where all VM images are available (used by has_feature API)
2894 sub shared_nodes {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => 1 } @$nodelist };
2899     my $nodename = PVE::INotify::nodename();
2900
2901     foreach_drive($conf, sub {
2902         my ($ds, $drive) = @_;
2903
2904         my $volid = $drive->{file};
2905         return if !$volid;
2906
2907         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2908         if ($storeid) {
2909             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2910             if ($scfg->{disable}) {
2911                 $nodehash = {};
2912             } elsif (my $avail = $scfg->{nodes}) {
2913                 foreach my $node (keys %$nodehash) {
2914                     delete $nodehash->{$node} if !$avail->{$node};
2915                 }
2916             } elsif (!$scfg->{shared}) {
2917                 foreach my $node (keys %$nodehash) {
2918                     delete $nodehash->{$node} if $node ne $nodename
2919                 }
2920             }
2921         }
2922     });
2923
2924     return $nodehash
2925 }
2926
2927 sub check_cmdline {
2928     my ($pidfile, $pid) = @_;
2929
2930     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2931     if (defined($fh)) {
2932         my $line = <$fh>;
2933         $fh->close;
2934         return undef if !$line;
2935         my @param = split(/\0/, $line);
2936
2937         my $cmd = $param[0];
2938         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2939
2940         for (my $i = 0; $i < scalar (@param); $i++) {
2941             my $p = $param[$i];
2942             next if !$p;
2943             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2944                 my $p = $param[$i+1];
2945                 return 1 if $p && ($p eq $pidfile);
2946                 return undef;
2947             }
2948         }
2949     }
2950     return undef;
2951 }
2952
2953 sub check_running {
2954     my ($vmid, $nocheck, $node) = @_;
2955
2956     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2957
2958     die "unable to find configuration file for VM $vmid - no such machine\n"
2959         if !$nocheck && ! -f $filename;
2960
2961     my $pidfile = pidfile_name($vmid);
2962
2963     if (my $fd = IO::File->new("<$pidfile")) {
2964         my $st = stat($fd);
2965         my $line = <$fd>;
2966         close($fd);
2967
2968         my $mtime = $st->mtime;
2969         if ($mtime > time()) {
2970             warn "file '$filename' modified in future\n";
2971         }
2972
2973         if ($line =~ m/^(\d+)$/) {
2974             my $pid = $1;
2975             if (check_cmdline($pidfile, $pid)) {
2976                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2977                     return $pid;
2978                 }
2979             }
2980         }
2981     }
2982
2983     return undef;
2984 }
2985
2986 sub vzlist {
2987
2988     my $vzlist = config_list();
2989
2990     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2991
2992     while (defined(my $de = $fd->read)) {
2993         next if $de !~ m/^(\d+)\.pid$/;
2994         my $vmid = $1;
2995         next if !defined($vzlist->{$vmid});
2996         if (my $pid = check_running($vmid)) {
2997             $vzlist->{$vmid}->{pid} = $pid;
2998         }
2999     }
3000
3001     return $vzlist;
3002 }
3003
3004 sub disksize {
3005     my ($storecfg, $conf) = @_;
3006
3007     my $bootdisk = $conf->{bootdisk};
3008     return undef if !$bootdisk;
3009     return undef if !is_valid_drivename($bootdisk);
3010
3011     return undef if !$conf->{$bootdisk};
3012
3013     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3014     return undef if !defined($drive);
3015
3016     return undef if drive_is_cdrom($drive);
3017
3018     my $volid = $drive->{file};
3019     return undef if !$volid;
3020
3021     return $drive->{size};
3022 }
3023
3024 our $vmstatus_return_properties = {
3025     vmid => get_standard_option('pve-vmid'),
3026     status => {
3027         description => "Qemu process status.",
3028         type => 'string',
3029         enum => ['stopped', 'running'],
3030     },
3031     maxmem => {
3032         description => "Maximum memory in bytes.",
3033         type => 'integer',
3034         optional => 1,
3035         renderer => 'bytes',
3036     },
3037     maxdisk => {
3038         description => "Root disk size in bytes.",
3039         type => 'integer',
3040         optional => 1,
3041         renderer => 'bytes',
3042     },
3043     name => {
3044         description => "VM name.",
3045         type => 'string',
3046         optional => 1,
3047     },
3048     qmpstatus => {
3049         description => "Qemu QMP agent status.",
3050         type => 'string',
3051         optional => 1,
3052     },
3053     pid => {
3054         description => "PID of running qemu process.",
3055         type => 'integer',
3056         optional => 1,
3057     },
3058     uptime => {
3059         description => "Uptime.",
3060         type => 'integer',
3061         optional => 1,
3062         renderer => 'duration',
3063     },
3064     cpus => {
3065         description => "Maximum usable CPUs.",
3066         type => 'number',
3067         optional => 1,
3068     },
3069     lock => {
3070         description => "The current config lock, if any.",
3071         type => 'string',
3072         optional => 1,
3073     }
3074 };
3075
3076 my $last_proc_pid_stat;
3077
3078 # get VM status information
3079 # This must be fast and should not block ($full == false)
3080 # We only query KVM using QMP if $full == true (this can be slow)
3081 sub vmstatus {
3082     my ($opt_vmid, $full) = @_;
3083
3084     my $res = {};
3085
3086     my $storecfg = PVE::Storage::config();
3087
3088     my $list = vzlist();
3089     my $defaults = load_defaults();
3090
3091     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3092
3093     my $cpucount = $cpuinfo->{cpus} || 1;
3094
3095     foreach my $vmid (keys %$list) {
3096         next if $opt_vmid && ($vmid ne $opt_vmid);
3097
3098         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3099         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3100
3101         my $d = { vmid => $vmid };
3102         $d->{pid} = $list->{$vmid}->{pid};
3103
3104         # fixme: better status?
3105         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3106
3107         my $size = disksize($storecfg, $conf);
3108         if (defined($size)) {
3109             $d->{disk} = 0; # no info available
3110             $d->{maxdisk} = $size;
3111         } else {
3112             $d->{disk} = 0;
3113             $d->{maxdisk} = 0;
3114         }
3115
3116         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3117             * ($conf->{cores} || $defaults->{cores});
3118         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3119         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3120
3121         $d->{name} = $conf->{name} || "VM $vmid";
3122         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3123             : $defaults->{memory}*(1024*1024);
3124
3125         if ($conf->{balloon}) {
3126             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3127             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3128                 : $defaults->{shares};
3129         }
3130
3131         $d->{uptime} = 0;
3132         $d->{cpu} = 0;
3133         $d->{mem} = 0;
3134
3135         $d->{netout} = 0;
3136         $d->{netin} = 0;
3137
3138         $d->{diskread} = 0;
3139         $d->{diskwrite} = 0;
3140
3141         $d->{template} = PVE::QemuConfig->is_template($conf);
3142
3143         $d->{serial} = 1 if conf_has_serial($conf);
3144         $d->{lock} = $conf->{lock} if $conf->{lock};
3145
3146         $res->{$vmid} = $d;
3147     }
3148
3149     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3150     foreach my $dev (keys %$netdev) {
3151         next if $dev !~ m/^tap([1-9]\d*)i/;
3152         my $vmid = $1;
3153         my $d = $res->{$vmid};
3154         next if !$d;
3155
3156         $d->{netout} += $netdev->{$dev}->{receive};
3157         $d->{netin} += $netdev->{$dev}->{transmit};
3158
3159         if ($full) {
3160             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3161             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3162         }
3163
3164     }
3165
3166     my $ctime = gettimeofday;
3167
3168     foreach my $vmid (keys %$list) {
3169
3170         my $d = $res->{$vmid};
3171         my $pid = $d->{pid};
3172         next if !$pid;
3173
3174         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3175         next if !$pstat; # not running
3176
3177         my $used = $pstat->{utime} + $pstat->{stime};
3178
3179         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3180
3181         if ($pstat->{vsize}) {
3182             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3183         }
3184
3185         my $old = $last_proc_pid_stat->{$pid};
3186         if (!$old) {
3187             $last_proc_pid_stat->{$pid} = {
3188                 time => $ctime,
3189                 used => $used,
3190                 cpu => 0,
3191             };
3192             next;
3193         }
3194
3195         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3196
3197         if ($dtime > 1000) {
3198             my $dutime = $used -  $old->{used};
3199
3200             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3201             $last_proc_pid_stat->{$pid} = {
3202                 time => $ctime,
3203                 used => $used,
3204                 cpu => $d->{cpu},
3205             };
3206         } else {
3207             $d->{cpu} = $old->{cpu};
3208         }
3209     }
3210
3211     return $res if !$full;
3212
3213     my $qmpclient = PVE::QMPClient->new();
3214
3215     my $ballooncb = sub {
3216         my ($vmid, $resp) = @_;
3217
3218         my $info = $resp->{'return'};
3219         return if !$info->{max_mem};
3220
3221         my $d = $res->{$vmid};
3222
3223         # use memory assigned to VM
3224         $d->{maxmem} = $info->{max_mem};
3225         $d->{balloon} = $info->{actual};
3226
3227         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3228             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3229             $d->{freemem} = $info->{free_mem};
3230         }
3231
3232         $d->{ballooninfo} = $info;
3233     };
3234
3235     my $blockstatscb = sub {
3236         my ($vmid, $resp) = @_;
3237         my $data = $resp->{'return'} || [];
3238         my $totalrdbytes = 0;
3239         my $totalwrbytes = 0;
3240
3241         for my $blockstat (@$data) {
3242             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3243             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3244
3245             $blockstat->{device} =~ s/drive-//;
3246             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3247         }
3248         $res->{$vmid}->{diskread} = $totalrdbytes;
3249         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3250     };
3251
3252     my $statuscb = sub {
3253         my ($vmid, $resp) = @_;
3254
3255         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3256         # this fails if ballon driver is not loaded, so this must be
3257         # the last commnand (following command are aborted if this fails).
3258         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3259
3260         my $status = 'unknown';
3261         if (!defined($status = $resp->{'return'}->{status})) {
3262             warn "unable to get VM status\n";
3263             return;
3264         }
3265
3266         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3267     };
3268
3269     foreach my $vmid (keys %$list) {
3270         next if $opt_vmid && ($vmid ne $opt_vmid);
3271         next if !$res->{$vmid}->{pid}; # not running
3272         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3273     }
3274
3275     $qmpclient->queue_execute(undef, 2);
3276
3277     foreach my $vmid (keys %$list) {
3278         next if $opt_vmid && ($vmid ne $opt_vmid);
3279         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3280     }
3281
3282     return $res;
3283 }
3284
3285 sub foreach_drive {
3286     my ($conf, $func, @param) = @_;
3287
3288     foreach my $ds (valid_drive_names()) {
3289         next if !defined($conf->{$ds});
3290
3291         my $drive = parse_drive($ds, $conf->{$ds});
3292         next if !$drive;
3293
3294         &$func($ds, $drive, @param);
3295     }
3296 }
3297
3298 sub foreach_volid {
3299     my ($conf, $func, @param) = @_;
3300
3301     my $volhash = {};
3302
3303     my $test_volid = sub {
3304         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3305
3306         return if !$volid;
3307
3308         $volhash->{$volid}->{cdrom} //= 1;
3309         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3310
3311         $volhash->{$volid}->{replicate} //= 0;
3312         $volhash->{$volid}->{replicate} = 1 if $replicate;
3313
3314         $volhash->{$volid}->{shared} //= 0;
3315         $volhash->{$volid}->{shared} = 1 if $shared;
3316
3317         $volhash->{$volid}->{referenced_in_config} //= 0;
3318         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3319
3320         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3321             if defined($snapname);
3322     };
3323
3324     foreach_drive($conf, sub {
3325         my ($ds, $drive) = @_;
3326         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3327     });
3328
3329     foreach my $snapname (keys %{$conf->{snapshots}}) {
3330         my $snap = $conf->{snapshots}->{$snapname};
3331         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3332         foreach_drive($snap, sub {
3333             my ($ds, $drive) = @_;
3334             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3335         });
3336     }
3337
3338     foreach my $volid (keys %$volhash) {
3339         &$func($volid, $volhash->{$volid}, @param);
3340     }
3341 }
3342
3343 sub conf_has_serial {
3344     my ($conf) = @_;
3345
3346     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3347         if ($conf->{"serial$i"}) {
3348             return 1;
3349         }
3350     }
3351
3352     return 0;
3353 }
3354
3355 sub vga_conf_has_spice {
3356     my ($vga) = @_;
3357
3358     my $vgaconf = parse_vga($vga);
3359     my $vgatype = $vgaconf->{type};
3360     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3361
3362     return $1 || 1;
3363 }
3364
3365 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3366 sub get_host_arch() {
3367     $host_arch = (POSIX::uname())[4] if !$host_arch;
3368     return $host_arch;
3369 }
3370
3371 sub is_native($) {
3372     my ($arch) = @_;
3373     return get_host_arch() eq $arch;
3374 }
3375
3376 my $default_machines = {
3377     x86_64 => 'pc',
3378     aarch64 => 'virt',
3379 };
3380
3381 sub get_basic_machine_info {
3382     my ($conf, $forcemachine) = @_;
3383
3384     my $arch = $conf->{arch} // get_host_arch();
3385     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3386     return ($arch, $machine);
3387 }
3388
3389 sub get_ovmf_files($) {
3390     my ($arch) = @_;
3391
3392     my $ovmf = $OVMF->{$arch}
3393         or die "no OVMF images known for architecture '$arch'\n";
3394
3395     return @$ovmf;
3396 }
3397
3398 my $Arch2Qemu = {
3399     aarch64 => '/usr/bin/qemu-system-aarch64',
3400     x86_64 => '/usr/bin/qemu-system-x86_64',
3401 };
3402 sub get_command_for_arch($) {
3403     my ($arch) = @_;
3404     return '/usr/bin/kvm' if is_native($arch);
3405
3406     my $cmd = $Arch2Qemu->{$arch}
3407         or die "don't know how to emulate architecture '$arch'\n";
3408     return $cmd;
3409 }
3410
3411 sub get_cpu_options {
3412     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3413
3414     my $cpuFlags = [];
3415     my $ostype = $conf->{ostype};
3416
3417     my $cpu = $kvm ? "kvm64" : "qemu64";
3418     if ($arch eq 'aarch64') {
3419         $cpu = 'cortex-a57';
3420     }
3421     my $hv_vendor_id;
3422     if (my $cputype = $conf->{cpu}) {
3423         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3424             or die "Cannot parse cpu description: $cputype\n";
3425         $cpu = $cpuconf->{cputype};
3426         $kvm_off = 1 if $cpuconf->{hidden};
3427         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3428
3429         if (defined(my $flags = $cpuconf->{flags})) {
3430             push @$cpuFlags, split(";", $flags);
3431         }
3432     }
3433
3434     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3435
3436     push @$cpuFlags , '-x2apic'
3437         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3438
3439     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3440
3441     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3442
3443     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3444
3445         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3446         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3447     }
3448
3449     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3450
3451     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3452
3453     push @$cpuFlags, 'kvm=off' if $kvm_off;
3454
3455     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3456         push @$cpuFlags, "vendor=${cpu_vendor}"
3457             if $cpu_vendor ne 'default';
3458     } elsif ($arch ne 'aarch64') {
3459         die "internal error"; # should not happen
3460     }
3461
3462     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3463
3464     return ('-cpu', $cpu);
3465 }
3466
3467 sub config_to_command {
3468     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3469
3470     my $cmd = [];
3471     my $globalFlags = [];
3472     my $machineFlags = [];
3473     my $rtcFlags = [];
3474     my $devices = [];
3475     my $pciaddr = '';
3476     my $bridges = {};
3477     my $kvmver = kvm_user_version();
3478     my $vernum = 0; # unknown
3479     my $ostype = $conf->{ostype};
3480     my $winversion = windows_version($ostype);
3481     my $kvm = $conf->{kvm};
3482
3483     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3484     $kvm //= 1 if is_native($arch);
3485
3486     if ($kvm) {
3487         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3488             if !defined kvm_version();
3489     }
3490
3491     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3492         $vernum = $1*1000000+$2*1000;
3493     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3494         $vernum = $1*1000000+$2*1000+$3;
3495     }
3496
3497     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3498
3499     my $have_ovz = -f '/proc/vz/vestat';
3500
3501     my $q35 = machine_type_is_q35($conf);
3502     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3503     my $use_old_bios_files = undef;
3504     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3505
3506     my $cpuunits = defined($conf->{cpuunits}) ?
3507             $conf->{cpuunits} : $defaults->{cpuunits};
3508
3509     push @$cmd, get_command_for_arch($arch);
3510
3511     push @$cmd, '-id', $vmid;
3512
3513     my $vmname = $conf->{name} || "vm$vmid";
3514
3515     push @$cmd, '-name', $vmname;
3516
3517     my $use_virtio = 0;
3518
3519     my $qmpsocket = qmp_socket($vmid);
3520     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3521     push @$cmd, '-mon', "chardev=qmp,mode=control";
3522
3523     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3524         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3525         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3526     }
3527
3528     push @$cmd, '-pidfile' , pidfile_name($vmid);
3529
3530     push @$cmd, '-daemonize';
3531
3532     if ($conf->{smbios1}) {
3533         my $smbios_conf = parse_smbios1($conf->{smbios1});
3534         if ($smbios_conf->{base64}) {
3535             # Do not pass base64 flag to qemu
3536             delete $smbios_conf->{base64};
3537             my $smbios_string = "";
3538             foreach my $key (keys %$smbios_conf) {
3539                 my $value;
3540                 if ($key eq "uuid") {
3541                     $value = $smbios_conf->{uuid}
3542                 } else {
3543                     $value = decode_base64($smbios_conf->{$key});
3544                 }
3545                 # qemu accepts any binary data, only commas need escaping by double comma
3546                 $value =~ s/,/,,/g;
3547                 $smbios_string .= "," . $key . "=" . $value if $value;
3548             }
3549             push @$cmd, '-smbios', "type=1" . $smbios_string;
3550         } else {
3551             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3552         }
3553     }
3554
3555     if ($conf->{vmgenid}) {
3556         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3557     }
3558
3559     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3560     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3561         die "uefi base image not found\n" if ! -f $ovmf_code;
3562
3563         my $path;
3564         my $format;
3565         if (my $efidisk = $conf->{efidisk0}) {
3566             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3567             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3568             $format = $d->{format};
3569             if ($storeid) {
3570                 $path = PVE::Storage::path($storecfg, $d->{file});
3571                 if (!defined($format)) {
3572                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3573                     $format = qemu_img_format($scfg, $volname);
3574                 }
3575             } else {
3576                 $path = $d->{file};
3577                 die "efidisk format must be specified\n"
3578                     if !defined($format);
3579             }
3580         } else {
3581             warn "no efidisk configured! Using temporary efivars disk.\n";
3582             $path = "/tmp/$vmid-ovmf.fd";
3583             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3584             $format = 'raw';
3585         }
3586
3587         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3588         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3589     }
3590
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3594     push @$devices, @usbcontrollers if @usbcontrollers;
3595     my $vga = parse_vga($conf->{vga});
3596
3597     my $qxlnum = vga_conf_has_spice($conf->{vga});
3598     $vga->{type} = 'qxl' if $qxlnum;
3599
3600     if (!$vga->{type}) {
3601         if ($arch eq 'aarch64') {
3602             $vga->{type} = 'virtio';
3603         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3604             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3605         } else {
3606             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3607         }
3608     }
3609
3610     # enable absolute mouse coordinates (needed by vnc)
3611     my $tablet;
3612     if (defined($conf->{tablet})) {
3613         $tablet = $conf->{tablet};
3614     } else {
3615         $tablet = $defaults->{tablet};
3616         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3618     }
3619
3620     if ($tablet) {
3621         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3622         my $kbd = print_keyboarddevice_full($conf, $arch);
3623         push @$devices, '-device', $kbd if defined($kbd);
3624     }
3625
3626     my $kvm_off = 0;
3627     my $gpu_passthrough;
3628
3629     # host pci devices
3630     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3631         my $d = parse_hostpci($conf->{"hostpci$i"});
3632         next if !$d;
3633
3634         my $pcie = $d->{pcie};
3635         if ($pcie) {
3636             die "q35 machine model is not enabled" if !$q35;
3637             # win7 wants to have the pcie devices directly on the pcie bus
3638             # instead of in the root port
3639             if ($winversion == 7) {
3640                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3641             } else {
3642                 $pciaddr = print_pcie_addr("hostpci$i");
3643             }
3644         } else {
3645             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3646         }
3647
3648         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3649         my $romfile = $d->{romfile};
3650
3651         my $xvga = '';
3652         if ($d->{'x-vga'}) {
3653             $xvga = ',x-vga=on';
3654             $kvm_off = 1;
3655             $vga->{type} = 'none' if !defined($conf->{vga});
3656             $gpu_passthrough = 1;
3657
3658             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3659                 $xvga = "";
3660             }
3661         }
3662         my $pcidevices = $d->{pciid};
3663         my $multifunction = 1 if @$pcidevices > 1;
3664         my $sysfspath;
3665         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3666             my $id = $pcidevices->[0]->{id};
3667             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3668             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3669         } elsif ($d->{mdev}) {
3670             warn "ignoring mediated device with multifunction device\n";
3671         }
3672
3673         my $j=0;
3674         foreach my $pcidevice (@$pcidevices) {
3675
3676             my $id = "hostpci$i";
3677             $id .= ".$j" if $multifunction;
3678             my $addr = $pciaddr;
3679             $addr .= ".$j" if $multifunction;
3680             my $devicestr = "vfio-pci";
3681             if ($sysfspath) {
3682                 $devicestr .= ",sysfsdev=$sysfspath";
3683             } else {
3684                 $devicestr .= ",host=$pcidevice->{id}";
3685             }
3686             $devicestr .= ",id=$id$addr";
3687
3688             if($j == 0){
3689                 $devicestr .= "$rombar$xvga";
3690                 $devicestr .= ",multifunction=on" if $multifunction;
3691                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3692             }
3693
3694             push @$devices, '-device', $devicestr;
3695             $j++;
3696         }
3697     }
3698
3699     # usb devices
3700     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3701     push @$devices, @usbdevices if @usbdevices;
3702     # serial devices
3703     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3704         if (my $path = $conf->{"serial$i"}) {
3705             if ($path eq 'socket') {
3706                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3708                 # On aarch64, serial0 is the UART device. Qemu only allows
3709                 # connecting UART devices via the '-serial' command line, as
3710                 # the device has a fixed slot on the hardware...
3711                 if ($arch eq 'aarch64' && $i == 0) {
3712                     push @$devices, '-serial', "chardev:serial$i";
3713                 } else {
3714                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3715                 }
3716             } else {
3717                 die "no such serial device\n" if ! -c $path;
3718                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3719                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3720             }
3721         }
3722     }
3723
3724     # parallel devices
3725     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3726         if (my $path = $conf->{"parallel$i"}) {
3727             die "no such parallel device\n" if ! -c $path;
3728             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3729             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3730             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3731         }
3732     }
3733
3734
3735     my $sockets = 1;
3736     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3737     $sockets = $conf->{sockets} if  $conf->{sockets};
3738
3739     my $cores = $conf->{cores} || 1;
3740
3741     my $maxcpus = $sockets * $cores;
3742
3743     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3744
3745     my $allowed_vcpus = $cpuinfo->{cpus};
3746
3747     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3748         if ($allowed_vcpus < $maxcpus);
3749
3750     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3751
3752         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753         for (my $i = 2; $i <= $vcpus; $i++)  {
3754             my $cpustr = print_cpu_device($conf,$i);
3755             push @$cmd, '-device', $cpustr;
3756         }
3757
3758     } else {
3759
3760         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3761     }
3762     push @$cmd, '-nodefaults';
3763
3764     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3765
3766     my $bootindex_hash = {};
3767     my $i = 1;
3768     foreach my $o (split(//, $bootorder)) {
3769         $bootindex_hash->{$o} = $i*100;
3770         $i++;
3771     }
3772
3773     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3774
3775     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3776
3777     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3778
3779     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3780         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3781         my $socket = vnc_socket($vmid);
3782         push @$cmd,  '-vnc', "unix:$socket,password";
3783     } else {
3784         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3785         push @$cmd, '-nographic';
3786     }
3787
3788     # time drift fix
3789     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3790
3791     my $useLocaltime = $conf->{localtime};
3792
3793     if ($winversion >= 5) { # windows
3794         $useLocaltime = 1 if !defined($conf->{localtime});
3795
3796         # use time drift fix when acpi is enabled
3797         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3798             $tdf = 1 if !defined($conf->{tdf});
3799         }
3800     }
3801
3802     if ($winversion >= 6) {
3803         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3804         push @$cmd, '-no-hpet';
3805     }
3806
3807     push @$rtcFlags, 'driftfix=slew' if $tdf;
3808
3809     if (!$kvm) {
3810         push @$machineFlags, 'accel=tcg';
3811     }
3812
3813     if ($machine_type) {
3814         push @$machineFlags, "type=${machine_type}";
3815     }
3816
3817     if ($conf->{startdate}) {
3818         push @$rtcFlags, "base=$conf->{startdate}";
3819     } elsif ($useLocaltime) {
3820         push @$rtcFlags, 'base=localtime';
3821     }
3822
3823     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3824
3825     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3826
3827     push @$cmd, '-S' if $conf->{freeze};
3828
3829     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3830
3831     # enable sound
3832     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3833     #push @$cmd, '-soundhw', 'es1370';
3834     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3835
3836     if (parse_guest_agent($conf)->{enabled}) {
3837         my $qgasocket = qmp_socket($vmid, 1);
3838         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3839         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3840         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3841         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3842     }
3843
3844     my $spice_port;
3845
3846     if ($qxlnum) {
3847         if ($qxlnum > 1) {
3848             if ($winversion){
3849                 for(my $i = 1; $i < $qxlnum; $i++){
3850                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3851                 }
3852             } else {
3853                 # assume other OS works like Linux
3854                 my ($ram, $vram) = ("134217728", "67108864");
3855                 if ($vga->{memory}) {
3856                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3857                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3858                 }
3859                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3860                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3861             }
3862         }
3863
3864         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3865
3866         my $nodename = PVE::INotify::nodename();
3867         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3868         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3869         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3870         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3871         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3872
3873         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3874
3875         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3876         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3877         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3878     }
3879
3880     # enable balloon by default, unless explicitly disabled
3881     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3882         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3883         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3884     }
3885
3886     if ($conf->{watchdog}) {
3887         my $wdopts = parse_watchdog($conf->{watchdog});
3888         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3889         my $watchdog = $wdopts->{model} || 'i6300esb';
3890         push @$devices, '-device', "$watchdog$pciaddr";
3891         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3892     }
3893
3894     my $vollist = [];
3895     my $scsicontroller = {};
3896     my $ahcicontroller = {};
3897     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3898
3899     # Add iscsi initiator name if available
3900     if (my $initiator = get_initiator_name()) {
3901         push @$devices, '-iscsi', "initiator-name=$initiator";
3902     }
3903
3904     foreach_drive($conf, sub {
3905         my ($ds, $drive) = @_;
3906
3907         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3908             push @$vollist, $drive->{file};
3909         }
3910
3911         # ignore efidisk here, already added in bios/fw handling code above
3912         return if $drive->{interface} eq 'efidisk';
3913
3914         $use_virtio = 1 if $ds =~ m/^virtio/;
3915
3916         if (drive_is_cdrom ($drive)) {
3917             if ($bootindex_hash->{d}) {
3918                 $drive->{bootindex} = $bootindex_hash->{d};
3919                 $bootindex_hash->{d} += 1;
3920             }
3921         } else {
3922             if ($bootindex_hash->{c}) {
3923                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3924                 $bootindex_hash->{c} += 1;
3925             }
3926         }
3927
3928         if($drive->{interface} eq 'virtio'){
3929            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3930         }
3931
3932         if ($drive->{interface} eq 'scsi') {
3933
3934             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3935
3936             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3937             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3938
3939             my $iothread = '';
3940             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3941                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3942                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3943             } elsif ($drive->{iothread}) {
3944                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3945             }
3946
3947             my $queues = '';
3948             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3949                 $queues = ",num_queues=$drive->{queues}";
3950             }
3951
3952             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3953             $scsicontroller->{$controller}=1;
3954         }
3955
3956         if ($drive->{interface} eq 'sata') {
3957            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3958            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3959            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3960            $ahcicontroller->{$controller}=1;
3961         }
3962
3963         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3964         push @$devices, '-drive',$drive_cmd;
3965         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3966     });
3967
3968     for (my $i = 0; $i < $MAX_NETS; $i++) {
3969          next if !$conf->{"net$i"};
3970          my $d = parse_net($conf->{"net$i"});
3971          next if !$d;
3972
3973          $use_virtio = 1 if $d->{model} eq 'virtio';
3974
3975          if ($bootindex_hash->{n}) {
3976             $d->{bootindex} = $bootindex_hash->{n};
3977             $bootindex_hash->{n} += 1;
3978          }
3979
3980          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3981          push @$devices, '-netdev', $netdevfull;
3982
3983          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3984          push @$devices, '-device', $netdevicefull;
3985     }
3986
3987     if ($conf->{ivshmem}) {
3988         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3989
3990         my $bus;
3991         if ($q35) {
3992             $bus = print_pcie_addr("ivshmem");
3993         } else {
3994             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3995         }
3996
3997         my $ivshmem_name = $ivshmem->{name} // $vmid;
3998         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3999
4000         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4001         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4002     }
4003
4004     if (!$q35) {
4005         # add pci bridges
4006         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4007            $bridges->{1} = 1;
4008            $bridges->{2} = 1;
4009         }
4010
4011         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4012
4013         while (my ($k, $v) = each %$bridges) {
4014             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4015             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4016         }
4017     }
4018
4019     push @$cmd, @$devices;
4020     push @$cmd, '-rtc', join(',', @$rtcFlags)
4021         if scalar(@$rtcFlags);
4022     push @$cmd, '-machine', join(',', @$machineFlags)
4023         if scalar(@$machineFlags);
4024     push @$cmd, '-global', join(',', @$globalFlags)
4025         if scalar(@$globalFlags);
4026
4027     if (my $vmstate = $conf->{vmstate}) {
4028         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4029         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4030         push @$cmd, '-loadstate', $statepath;
4031     }
4032
4033     # add custom args
4034     if ($conf->{args}) {
4035         my $aa = PVE::Tools::split_args($conf->{args});
4036         push @$cmd, @$aa;
4037     }
4038
4039     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4040 }
4041
4042 sub vnc_socket {
4043     my ($vmid) = @_;
4044     return "${var_run_tmpdir}/$vmid.vnc";
4045 }
4046
4047 sub spice_port {
4048     my ($vmid) = @_;
4049
4050     my $res = vm_mon_cmd($vmid, 'query-spice');
4051
4052     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4053 }
4054
4055 sub qmp_socket {
4056     my ($vmid, $qga, $name) = @_;
4057     my $sockettype = $qga ? 'qga' : 'qmp';
4058     my $ext = $name ? '-'.$name : '';
4059     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4060 }
4061
4062 sub pidfile_name {
4063     my ($vmid) = @_;
4064     return "${var_run_tmpdir}/$vmid.pid";
4065 }
4066
4067 sub vm_devices_list {
4068     my ($vmid) = @_;
4069
4070     my $res = vm_mon_cmd($vmid, 'query-pci');
4071     my $devices_to_check = [];
4072     my $devices = {};
4073     foreach my $pcibus (@$res) {
4074         push @$devices_to_check, @{$pcibus->{devices}},
4075     }
4076
4077     while (@$devices_to_check) {
4078         my $to_check = [];
4079         for my $d (@$devices_to_check) {
4080             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4081             next if !$d->{'pci_bridge'};
4082
4083             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4084             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4085         }
4086         $devices_to_check = $to_check;
4087     }
4088
4089     my $resblock = vm_mon_cmd($vmid, 'query-block');
4090     foreach my $block (@$resblock) {
4091         if($block->{device} =~ m/^drive-(\S+)/){
4092                 $devices->{$1} = 1;
4093         }
4094     }
4095
4096     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4097     foreach my $mice (@$resmice) {
4098         if ($mice->{name} eq 'QEMU HID Tablet') {
4099             $devices->{tablet} = 1;
4100             last;
4101         }
4102     }
4103
4104     # for usb devices there is no query-usb
4105     # but we can iterate over the entries in
4106     # qom-list path=/machine/peripheral
4107     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4108     foreach my $per (@$resperipheral) {
4109         if ($per->{name} =~ m/^usb\d+$/) {
4110             $devices->{$per->{name}} = 1;
4111         }
4112     }
4113
4114     return $devices;
4115 }
4116
4117 sub vm_deviceplug {
4118     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4119
4120     my $q35 = machine_type_is_q35($conf);
4121
4122     my $devices_list = vm_devices_list($vmid);
4123     return 1 if defined($devices_list->{$deviceid});
4124
4125     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4126
4127     if ($deviceid eq 'tablet') {
4128
4129         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4130
4131     } elsif ($deviceid eq 'keyboard') {
4132
4133         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4134
4135     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4136
4137         die "usb hotplug currently not reliable\n";
4138         # since we can't reliably hot unplug all added usb devices
4139         # and usb passthrough disables live migration
4140         # we disable usb hotplugging for now
4141         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4142
4143     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4144
4145         qemu_iothread_add($vmid, $deviceid, $device);
4146
4147         qemu_driveadd($storecfg, $vmid, $device);
4148         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4149
4150         qemu_deviceadd($vmid, $devicefull);
4151         eval { qemu_deviceaddverify($vmid, $deviceid); };
4152         if (my $err = $@) {
4153             eval { qemu_drivedel($vmid, $deviceid); };
4154             warn $@ if $@;
4155             die $err;
4156         }
4157
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159
4160
4161         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4162         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4163         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4164
4165         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4166
4167         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4168             qemu_iothread_add($vmid, $deviceid, $device);
4169             $devicefull .= ",iothread=iothread-$deviceid";
4170         }
4171
4172         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4173             $devicefull .= ",num_queues=$device->{queues}";
4174         }
4175
4176         qemu_deviceadd($vmid, $devicefull);
4177         qemu_deviceaddverify($vmid, $deviceid);
4178
4179     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4180
4181         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4182         qemu_driveadd($storecfg, $vmid, $device);
4183
4184         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4185         eval { qemu_deviceadd($vmid, $devicefull); };
4186         if (my $err = $@) {
4187             eval { qemu_drivedel($vmid, $deviceid); };
4188             warn $@ if $@;
4189             die $err;
4190         }
4191
4192     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4193
4194         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4195
4196         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4197         my $use_old_bios_files = undef;
4198         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4199
4200         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4201         qemu_deviceadd($vmid, $netdevicefull);
4202         eval {
4203             qemu_deviceaddverify($vmid, $deviceid);
4204             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4205         };
4206         if (my $err = $@) {
4207             eval { qemu_netdevdel($vmid, $deviceid); };
4208             warn $@ if $@;
4209             die $err;
4210         }
4211
4212     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4213
4214         my $bridgeid = $2;
4215         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4216         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4217
4218         qemu_deviceadd($vmid, $devicefull);
4219         qemu_deviceaddverify($vmid, $deviceid);
4220
4221     } else {
4222         die "can't hotplug device '$deviceid'\n";
4223     }
4224
4225     return 1;
4226 }
4227
4228 # fixme: this should raise exceptions on error!
4229 sub vm_deviceunplug {
4230     my ($vmid, $conf, $deviceid) = @_;
4231
4232     my $devices_list = vm_devices_list($vmid);
4233     return 1 if !defined($devices_list->{$deviceid});
4234
4235     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4236
4237     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240
4241     } elsif ($deviceid =~ m/^usb\d+$/) {
4242
4243         die "usb hotplug currently not reliable\n";
4244         # when unplugging usb devices this way,
4245         # there may be remaining usb controllers/hubs
4246         # so we disable it for now
4247         qemu_devicedel($vmid, $deviceid);
4248         qemu_devicedelverify($vmid, $deviceid);
4249
4250     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4251
4252         qemu_devicedel($vmid, $deviceid);
4253         qemu_devicedelverify($vmid, $deviceid);
4254         qemu_drivedel($vmid, $deviceid);
4255         qemu_iothread_del($conf, $vmid, $deviceid);
4256
4257     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4258
4259         qemu_devicedel($vmid, $deviceid);
4260         qemu_devicedelverify($vmid, $deviceid);
4261         qemu_iothread_del($conf, $vmid, $deviceid);
4262
4263     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4264
4265         qemu_devicedel($vmid, $deviceid);
4266         qemu_drivedel($vmid, $deviceid);
4267         qemu_deletescsihw($conf, $vmid, $deviceid);
4268
4269     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4270
4271         qemu_devicedel($vmid, $deviceid);
4272         qemu_devicedelverify($vmid, $deviceid);
4273         qemu_netdevdel($vmid, $deviceid);
4274
4275     } else {
4276         die "can't unplug device '$deviceid'\n";
4277     }
4278
4279     return 1;
4280 }
4281
4282 sub qemu_deviceadd {
4283     my ($vmid, $devicefull) = @_;
4284
4285     $devicefull = "driver=".$devicefull;
4286     my %options =  split(/[=,]/, $devicefull);
4287
4288     vm_mon_cmd($vmid, "device_add" , %options);
4289 }
4290
4291 sub qemu_devicedel {
4292     my ($vmid, $deviceid) = @_;
4293
4294     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4295 }
4296
4297 sub qemu_iothread_add {
4298     my($vmid, $deviceid, $device) = @_;
4299
4300     if ($device->{iothread}) {
4301         my $iothreads = vm_iothreads_list($vmid);
4302         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4303     }
4304 }
4305
4306 sub qemu_iothread_del {
4307     my($conf, $vmid, $deviceid) = @_;
4308
4309     my $confid = $deviceid;
4310     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4311         $confid = 'scsi' . $1;
4312     }
4313     my $device = parse_drive($confid, $conf->{$confid});
4314     if ($device->{iothread}) {
4315         my $iothreads = vm_iothreads_list($vmid);
4316         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4317     }
4318 }
4319
4320 sub qemu_objectadd {
4321     my($vmid, $objectid, $qomtype) = @_;
4322
4323     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4324
4325     return 1;
4326 }
4327
4328 sub qemu_objectdel {
4329     my($vmid, $objectid) = @_;
4330
4331     vm_mon_cmd($vmid, "object-del", id => $objectid);
4332
4333     return 1;
4334 }
4335
4336 sub qemu_driveadd {
4337     my ($storecfg, $vmid, $device) = @_;
4338
4339     my $drive = print_drive_full($storecfg, $vmid, $device);
4340     $drive =~ s/\\/\\\\/g;
4341     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4342
4343     # If the command succeeds qemu prints: "OK"
4344     return 1 if $ret =~ m/OK/s;
4345
4346     die "adding drive failed: $ret\n";
4347 }
4348
4349 sub qemu_drivedel {
4350     my($vmid, $deviceid) = @_;
4351
4352     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4353     $ret =~ s/^\s+//;
4354
4355     return 1 if $ret eq "";
4356
4357     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4358     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4359
4360     die "deleting drive $deviceid failed : $ret\n";
4361 }
4362
4363 sub qemu_deviceaddverify {
4364     my ($vmid, $deviceid) = @_;
4365
4366     for (my $i = 0; $i <= 5; $i++) {
4367          my $devices_list = vm_devices_list($vmid);
4368          return 1 if defined($devices_list->{$deviceid});
4369          sleep 1;
4370     }
4371
4372     die "error on hotplug device '$deviceid'\n";
4373 }
4374
4375
4376 sub qemu_devicedelverify {
4377     my ($vmid, $deviceid) = @_;
4378
4379     # need to verify that the device is correctly removed as device_del
4380     # is async and empty return is not reliable
4381
4382     for (my $i = 0; $i <= 5; $i++) {
4383          my $devices_list = vm_devices_list($vmid);
4384          return 1 if !defined($devices_list->{$deviceid});
4385          sleep 1;
4386     }
4387
4388     die "error on hot-unplugging device '$deviceid'\n";
4389 }
4390
4391 sub qemu_findorcreatescsihw {
4392     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4393
4394     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4395
4396     my $scsihwid="$controller_prefix$controller";
4397     my $devices_list = vm_devices_list($vmid);
4398
4399     if(!defined($devices_list->{$scsihwid})) {
4400         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4401     }
4402
4403     return 1;
4404 }
4405
4406 sub qemu_deletescsihw {
4407     my ($conf, $vmid, $opt) = @_;
4408
4409     my $device = parse_drive($opt, $conf->{$opt});
4410
4411     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4412         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4413         return 1;
4414     }
4415
4416     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4417
4418     my $devices_list = vm_devices_list($vmid);
4419     foreach my $opt (keys %{$devices_list}) {
4420         if (PVE::QemuServer::is_valid_drivename($opt)) {
4421             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4422             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4423                 return 1;
4424             }
4425         }
4426     }
4427
4428     my $scsihwid="scsihw$controller";
4429
4430     vm_deviceunplug($vmid, $conf, $scsihwid);
4431
4432     return 1;
4433 }
4434
4435 sub qemu_add_pci_bridge {
4436     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4437
4438     my $bridges = {};
4439
4440     my $bridgeid;
4441
4442     print_pci_addr($device, $bridges, $arch, $machine_type);
4443
4444     while (my ($k, $v) = each %$bridges) {
4445         $bridgeid = $k;
4446     }
4447     return 1 if !defined($bridgeid) || $bridgeid < 1;
4448
4449     my $bridge = "pci.$bridgeid";
4450     my $devices_list = vm_devices_list($vmid);
4451
4452     if (!defined($devices_list->{$bridge})) {
4453         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4454     }
4455
4456     return 1;
4457 }
4458
4459 sub qemu_set_link_status {
4460     my ($vmid, $device, $up) = @_;
4461
4462     vm_mon_cmd($vmid, "set_link", name => $device,
4463                up => $up ? JSON::true : JSON::false);
4464 }
4465
4466 sub qemu_netdevadd {
4467     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4468
4469     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4470     my %options =  split(/[=,]/, $netdev);
4471
4472     vm_mon_cmd($vmid, "netdev_add",  %options);
4473     return 1;
4474 }
4475
4476 sub qemu_netdevdel {
4477     my ($vmid, $deviceid) = @_;
4478
4479     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4480 }
4481
4482 sub qemu_usb_hotplug {
4483     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4484
4485     return if !$device;
4486
4487     # remove the old one first
4488     vm_deviceunplug($vmid, $conf, $deviceid);
4489
4490     # check if xhci controller is necessary and available
4491     if ($device->{usb3}) {
4492
4493         my $devicelist = vm_devices_list($vmid);
4494
4495         if (!$devicelist->{xhci}) {
4496             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4497             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4498         }
4499     }
4500     my $d = parse_usb_device($device->{host});
4501     $d->{usb3} = $device->{usb3};
4502
4503     # add the new one
4504     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4505 }
4506
4507 sub qemu_cpu_hotplug {
4508     my ($vmid, $conf, $vcpus) = @_;
4509
4510     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4511
4512     my $sockets = 1;
4513     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4514     $sockets = $conf->{sockets} if  $conf->{sockets};
4515     my $cores = $conf->{cores} || 1;
4516     my $maxcpus = $sockets * $cores;
4517
4518     $vcpus = $maxcpus if !$vcpus;
4519
4520     die "you can't add more vcpus than maxcpus\n"
4521         if $vcpus > $maxcpus;
4522
4523     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4524
4525     if ($vcpus < $currentvcpus) {
4526
4527         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4528
4529             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4530                 qemu_devicedel($vmid, "cpu$i");
4531                 my $retry = 0;
4532                 my $currentrunningvcpus = undef;
4533                 while (1) {
4534                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4535                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4536                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4537                     $retry++;
4538                     sleep 1;
4539                 }
4540                 #update conf after each succesfull cpu unplug
4541                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4542                 PVE::QemuConfig->write_config($vmid, $conf);
4543             }
4544         } else {
4545             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4546         }
4547
4548         return;
4549     }
4550
4551     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4552     die "vcpus in running vm does not match its configuration\n"
4553         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4554
4555     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4556
4557         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4558             my $cpustr = print_cpu_device($conf, $i);
4559             qemu_deviceadd($vmid, $cpustr);
4560
4561             my $retry = 0;
4562             my $currentrunningvcpus = undef;
4563             while (1) {
4564                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4565                 last if scalar(@{$currentrunningvcpus}) == $i;
4566                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4567                 sleep 1;
4568                 $retry++;
4569             }
4570             #update conf after each succesfull cpu hotplug
4571             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4572             PVE::QemuConfig->write_config($vmid, $conf);
4573         }
4574     } else {
4575
4576         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4577             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4578         }
4579     }
4580 }
4581
4582 sub qemu_block_set_io_throttle {
4583     my ($vmid, $deviceid,
4584         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4585         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4586         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4587         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4588
4589     return if !check_running($vmid) ;
4590
4591     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4592         bps => int($bps),
4593         bps_rd => int($bps_rd),
4594         bps_wr => int($bps_wr),
4595         iops => int($iops),
4596         iops_rd => int($iops_rd),
4597         iops_wr => int($iops_wr),
4598         bps_max => int($bps_max),
4599         bps_rd_max => int($bps_rd_max),
4600         bps_wr_max => int($bps_wr_max),
4601         iops_max => int($iops_max),
4602         iops_rd_max => int($iops_rd_max),
4603         iops_wr_max => int($iops_wr_max),
4604         bps_max_length => int($bps_max_length),
4605         bps_rd_max_length => int($bps_rd_max_length),
4606         bps_wr_max_length => int($bps_wr_max_length),
4607         iops_max_length => int($iops_max_length),
4608         iops_rd_max_length => int($iops_rd_max_length),
4609         iops_wr_max_length => int($iops_wr_max_length),
4610     );
4611
4612 }
4613
4614 # old code, only used to shutdown old VM after update
4615 sub __read_avail {
4616     my ($fh, $timeout) = @_;
4617
4618     my $sel = new IO::Select;
4619     $sel->add($fh);
4620
4621     my $res = '';
4622     my $buf;
4623
4624     my @ready;
4625     while (scalar (@ready = $sel->can_read($timeout))) {
4626         my $count;
4627         if ($count = $fh->sysread($buf, 8192)) {
4628             if ($buf =~ /^(.*)\(qemu\) $/s) {
4629                 $res .= $1;
4630                 last;
4631             } else {
4632                 $res .= $buf;
4633             }
4634         } else {
4635             if (!defined($count)) {
4636                 die "$!\n";
4637             }
4638             last;
4639         }
4640     }
4641
4642     die "monitor read timeout\n" if !scalar(@ready);
4643
4644     return $res;
4645 }
4646
4647 sub qemu_block_resize {
4648     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4649
4650     my $running = check_running($vmid);
4651
4652     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4653
4654     return if !$running;
4655
4656     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4657
4658 }
4659
4660 sub qemu_volume_snapshot {
4661     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4666         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4667     } else {
4668         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4669     }
4670 }
4671
4672 sub qemu_volume_snapshot_delete {
4673     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4674
4675     my $running = check_running($vmid);
4676
4677     if($running) {
4678
4679         $running = undef;
4680         my $conf = PVE::QemuConfig->load_config($vmid);
4681         foreach_drive($conf, sub {
4682             my ($ds, $drive) = @_;
4683             $running = 1 if $drive->{file} eq $volid;
4684         });
4685     }
4686
4687     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4688         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4689     } else {
4690         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4691     }
4692 }
4693
4694 sub set_migration_caps {
4695     my ($vmid) = @_;
4696
4697     my $cap_ref = [];
4698
4699     my $enabled_cap = {
4700         "auto-converge" => 1,
4701         "xbzrle" => 1,
4702         "x-rdma-pin-all" => 0,
4703         "zero-blocks" => 0,
4704         "compress" => 0
4705     };
4706
4707     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4708
4709     for my $supported_capability (@$supported_capabilities) {
4710         push @$cap_ref, {
4711             capability => $supported_capability->{capability},
4712             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4713         };
4714     }
4715
4716     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4717 }
4718
4719 my $fast_plug_option = {
4720     'lock' => 1,
4721     'name' => 1,
4722     'onboot' => 1,
4723     'shares' => 1,
4724     'startup' => 1,
4725     'description' => 1,
4726     'protection' => 1,
4727     'vmstatestorage' => 1,
4728     'hookscript' => 1,
4729 };
4730
4731 # hotplug changes in [PENDING]
4732 # $selection hash can be used to only apply specified options, for
4733 # example: { cores => 1 } (only apply changed 'cores')
4734 # $errors ref is used to return error messages
4735 sub vmconfig_hotplug_pending {
4736     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4737
4738     my $defaults = load_defaults();
4739     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4740
4741     # commit values which do not have any impact on running VM first
4742     # Note: those option cannot raise errors, we we do not care about
4743     # $selection and always apply them.
4744
4745     my $add_error = sub {
4746         my ($opt, $msg) = @_;
4747         $errors->{$opt} = "hotplug problem - $msg";
4748     };
4749
4750     my $changes = 0;
4751     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4752         if ($fast_plug_option->{$opt}) {
4753             $conf->{$opt} = $conf->{pending}->{$opt};
4754             delete $conf->{pending}->{$opt};
4755             $changes = 1;
4756         }
4757     }
4758
4759     if ($changes) {
4760         PVE::QemuConfig->write_config($vmid, $conf);
4761         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4762     }
4763
4764     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4765
4766     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4767     while (my ($opt, $force) = each %$pending_delete_hash) {
4768         next if $selection && !$selection->{$opt};
4769         eval {
4770             if ($opt eq 'hotplug') {
4771                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4772             } elsif ($opt eq 'tablet') {
4773                 die "skip\n" if !$hotplug_features->{usb};
4774                 if ($defaults->{tablet}) {
4775                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4776                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4777                         if $arch eq 'aarch64';
4778                 } else {
4779                     vm_deviceunplug($vmid, $conf, 'tablet');
4780                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4781                 }
4782             } elsif ($opt =~ m/^usb\d+/) {
4783                 die "skip\n";
4784                 # since we cannot reliably hot unplug usb devices
4785                 # we are disabling it
4786                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4787                 vm_deviceunplug($vmid, $conf, $opt);
4788             } elsif ($opt eq 'vcpus') {
4789                 die "skip\n" if !$hotplug_features->{cpu};
4790                 qemu_cpu_hotplug($vmid, $conf, undef);
4791             } elsif ($opt eq 'balloon') {
4792                 # enable balloon device is not hotpluggable
4793                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4794                 # here we reset the ballooning value to memory
4795                 my $balloon = $conf->{memory} || $defaults->{memory};
4796                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4797             } elsif ($fast_plug_option->{$opt}) {
4798                 # do nothing
4799             } elsif ($opt =~ m/^net(\d+)$/) {
4800                 die "skip\n" if !$hotplug_features->{network};
4801                 vm_deviceunplug($vmid, $conf, $opt);
4802             } elsif (is_valid_drivename($opt)) {
4803                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4804                 vm_deviceunplug($vmid, $conf, $opt);
4805                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4806             } elsif ($opt =~ m/^memory$/) {
4807                 die "skip\n" if !$hotplug_features->{memory};
4808                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4809             } elsif ($opt eq 'cpuunits') {
4810                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4811             } elsif ($opt eq 'cpulimit') {
4812                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4813             } else {
4814                 die "skip\n";
4815             }
4816         };
4817         if (my $err = $@) {
4818             &$add_error($opt, $err) if $err ne "skip\n";
4819         } else {
4820             # save new config if hotplug was successful
4821             delete $conf->{$opt};
4822             vmconfig_undelete_pending_option($conf, $opt);
4823             PVE::QemuConfig->write_config($vmid, $conf);
4824             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825         }
4826     }
4827
4828     my $apply_pending_cloudinit;
4829     $apply_pending_cloudinit = sub {
4830         my ($key, $value) = @_;
4831         $apply_pending_cloudinit = sub {}; # once is enough
4832
4833         my @cloudinit_opts = keys %$confdesc_cloudinit;
4834         foreach my $opt (keys %{$conf->{pending}}) {
4835             next if !grep { $_ eq $opt } @cloudinit_opts;
4836             $conf->{$opt} = delete $conf->{pending}->{$opt};
4837         }
4838
4839         my $new_conf = { %$conf };
4840         $new_conf->{$key} = $value;
4841         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4842     };
4843
4844     foreach my $opt (keys %{$conf->{pending}}) {
4845         next if $selection && !$selection->{$opt};
4846         my $value = $conf->{pending}->{$opt};
4847         eval {
4848             if ($opt eq 'hotplug') {
4849                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4850             } elsif ($opt eq 'tablet') {
4851                 die "skip\n" if !$hotplug_features->{usb};
4852                 if ($value == 1) {
4853                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4854                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4855                         if $arch eq 'aarch64';
4856                 } elsif ($value == 0) {
4857                     vm_deviceunplug($vmid, $conf, 'tablet');
4858                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4859                 }
4860             } elsif ($opt =~ m/^usb\d+$/) {
4861                 die "skip\n";
4862                 # since we cannot reliably hot unplug usb devices
4863                 # we are disabling it
4864                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4865                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4866                 die "skip\n" if !$d;
4867                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4868             } elsif ($opt eq 'vcpus') {
4869                 die "skip\n" if !$hotplug_features->{cpu};
4870                 qemu_cpu_hotplug($vmid, $conf, $value);
4871             } elsif ($opt eq 'balloon') {
4872                 # enable/disable balloning device is not hotpluggable
4873                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4874                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4875                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4876
4877                 # allow manual ballooning if shares is set to zero
4878                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4879                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4880                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4881                 }
4882             } elsif ($opt =~ m/^net(\d+)$/) {
4883                 # some changes can be done without hotplug
4884                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4885                                     $vmid, $opt, $value, $arch, $machine_type);
4886             } elsif (is_valid_drivename($opt)) {
4887                 # some changes can be done without hotplug
4888                 my $drive = parse_drive($opt, $value);
4889                 if (drive_is_cloudinit($drive)) {
4890                     &$apply_pending_cloudinit($opt, $value);
4891                 }
4892                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4893                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4894             } elsif ($opt =~ m/^memory$/) { #dimms
4895                 die "skip\n" if !$hotplug_features->{memory};
4896                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4897             } elsif ($opt eq 'cpuunits') {
4898                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4899             } elsif ($opt eq 'cpulimit') {
4900                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4901                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4902             } else {
4903                 die "skip\n";  # skip non-hot-pluggable options
4904             }
4905         };
4906         if (my $err = $@) {
4907             &$add_error($opt, $err) if $err ne "skip\n";
4908         } else {
4909             # save new config if hotplug was successful
4910             $conf->{$opt} = $value;
4911             delete $conf->{pending}->{$opt};
4912             PVE::QemuConfig->write_config($vmid, $conf);
4913             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4914         }
4915     }
4916 }
4917
4918 sub try_deallocate_drive {
4919     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4920
4921     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4922         my $volid = $drive->{file};
4923         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4924             my $sid = PVE::Storage::parse_volume_id($volid);
4925             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4926
4927             # check if the disk is really unused
4928             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4929                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4930             PVE::Storage::vdisk_free($storecfg, $volid);
4931             return 1;
4932         } else {
4933             # If vm is not owner of this disk remove from config
4934             return 1;
4935         }
4936     }
4937
4938     return undef;
4939 }
4940
4941 sub vmconfig_delete_or_detach_drive {
4942     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4943
4944     my $drive = parse_drive($opt, $conf->{$opt});
4945
4946     my $rpcenv = PVE::RPCEnvironment::get();
4947     my $authuser = $rpcenv->get_user();
4948
4949     if ($force) {
4950         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4951         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4952     } else {
4953         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4954     }
4955 }
4956
4957 sub vmconfig_apply_pending {
4958     my ($vmid, $conf, $storecfg) = @_;
4959
4960     # cold plug
4961
4962     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4963     while (my ($opt, $force) = each %$pending_delete_hash) {
4964         die "internal error" if $opt =~ m/^unused/;
4965         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4966         if (!defined($conf->{$opt})) {
4967             vmconfig_undelete_pending_option($conf, $opt);
4968             PVE::QemuConfig->write_config($vmid, $conf);
4969         } elsif (is_valid_drivename($opt)) {
4970             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4971             vmconfig_undelete_pending_option($conf, $opt);
4972             delete $conf->{$opt};
4973             PVE::QemuConfig->write_config($vmid, $conf);
4974         } else {
4975             vmconfig_undelete_pending_option($conf, $opt);
4976             delete $conf->{$opt};
4977             PVE::QemuConfig->write_config($vmid, $conf);
4978         }
4979     }
4980
4981     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4982
4983     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4984         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4985
4986         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4987             # skip if nothing changed
4988         } elsif (is_valid_drivename($opt)) {
4989             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4990                 if defined($conf->{$opt});
4991             $conf->{$opt} = $conf->{pending}->{$opt};
4992         } else {
4993             $conf->{$opt} = $conf->{pending}->{$opt};
4994         }
4995
4996         delete $conf->{pending}->{$opt};
4997         PVE::QemuConfig->write_config($vmid, $conf);
4998     }
4999 }
5000
5001 my $safe_num_ne = sub {
5002     my ($a, $b) = @_;
5003
5004     return 0 if !defined($a) && !defined($b);
5005     return 1 if !defined($a);
5006     return 1 if !defined($b);
5007
5008     return $a != $b;
5009 };
5010
5011 my $safe_string_ne = sub {
5012     my ($a, $b) = @_;
5013
5014     return 0 if !defined($a) && !defined($b);
5015     return 1 if !defined($a);
5016     return 1 if !defined($b);
5017
5018     return $a ne $b;
5019 };
5020
5021 sub vmconfig_update_net {
5022     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5023
5024     my $newnet = parse_net($value);
5025
5026     if ($conf->{$opt}) {
5027         my $oldnet = parse_net($conf->{$opt});
5028
5029         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5030             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5031             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5032             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5033
5034             # for non online change, we try to hot-unplug
5035             die "skip\n" if !$hotplug;
5036             vm_deviceunplug($vmid, $conf, $opt);
5037         } else {
5038
5039             die "internal error" if $opt !~ m/net(\d+)/;
5040             my $iface = "tap${vmid}i$1";
5041
5042             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5043                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5044                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5045                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5046                 PVE::Network::tap_unplug($iface);
5047                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5048             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5049                 # Rate can be applied on its own but any change above needs to
5050                 # include the rate in tap_plug since OVS resets everything.
5051                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5052             }
5053
5054             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5055                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5056             }
5057
5058             return 1;
5059         }
5060     }
5061
5062     if ($hotplug) {
5063         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5064     } else {
5065         die "skip\n";
5066     }
5067 }
5068
5069 sub vmconfig_update_disk {
5070     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5071
5072     # fixme: do we need force?
5073
5074     my $drive = parse_drive($opt, $value);
5075
5076     if ($conf->{$opt}) {
5077
5078         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5079
5080             my $media = $drive->{media} || 'disk';
5081             my $oldmedia = $old_drive->{media} || 'disk';
5082             die "unable to change media type\n" if $media ne $oldmedia;
5083
5084             if (!drive_is_cdrom($old_drive)) {
5085
5086                 if ($drive->{file} ne $old_drive->{file}) {
5087
5088                     die "skip\n" if !$hotplug;
5089
5090                     # unplug and register as unused
5091                     vm_deviceunplug($vmid, $conf, $opt);
5092                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5093
5094                 } else {
5095                     # update existing disk
5096
5097                     # skip non hotpluggable value
5098                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5099                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5100                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5101                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5102                         die "skip\n";
5103                     }
5104
5105                     # apply throttle
5106                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5107                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5108                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5109                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5110                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5111                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5112                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5113                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5114                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5115                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5116                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5117                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5118                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5119                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5120                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5121                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5122                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5123                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5124
5125                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5126                                                    ($drive->{mbps} || 0)*1024*1024,
5127                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5128                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5129                                                    $drive->{iops} || 0,
5130                                                    $drive->{iops_rd} || 0,
5131                                                    $drive->{iops_wr} || 0,
5132                                                    ($drive->{mbps_max} || 0)*1024*1024,
5133                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5134                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5135                                                    $drive->{iops_max} || 0,
5136                                                    $drive->{iops_rd_max} || 0,
5137                                                    $drive->{iops_wr_max} || 0,
5138                                                    $drive->{bps_max_length} || 1,
5139                                                    $drive->{bps_rd_max_length} || 1,
5140                                                    $drive->{bps_wr_max_length} || 1,
5141                                                    $drive->{iops_max_length} || 1,
5142                                                    $drive->{iops_rd_max_length} || 1,
5143                                                    $drive->{iops_wr_max_length} || 1);
5144
5145                     }
5146
5147                     return 1;
5148                 }
5149
5150             } else { # cdrom
5151
5152                 if ($drive->{file} eq 'none') {
5153                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5154                     if (drive_is_cloudinit($old_drive)) {
5155                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5156                     }
5157                 } else {
5158                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5159                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5160                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5161                 }
5162
5163                 return 1;
5164             }
5165         }
5166     }
5167
5168     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5169     # hotplug new disks
5170     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5171     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5172 }
5173
5174 sub vm_start {
5175     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5176         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5177
5178     PVE::QemuConfig->lock_config($vmid, sub {
5179         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5180
5181         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5182
5183         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5184
5185         PVE::QemuConfig->check_lock($conf)
5186             if !($skiplock || $is_suspended);
5187
5188         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5189
5190         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5191             vmconfig_apply_pending($vmid, $conf, $storecfg);
5192             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5193         }
5194
5195         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5196
5197         my $defaults = load_defaults();
5198
5199         # set environment variable useful inside network script
5200         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5201
5202         my $local_volumes = {};
5203
5204         if ($targetstorage) {
5205             foreach_drive($conf, sub {
5206                 my ($ds, $drive) = @_;
5207
5208                 return if drive_is_cdrom($drive);
5209
5210                 my $volid = $drive->{file};
5211
5212                 return if !$volid;
5213
5214                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5215
5216                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5217                 return if $scfg->{shared};
5218                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5219             });
5220
5221             my $format = undef;
5222
5223             foreach my $opt (sort keys %$local_volumes) {
5224
5225                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5226                 my $drive = parse_drive($opt, $conf->{$opt});
5227
5228                 #if remote storage is specified, use default format
5229                 if ($targetstorage && $targetstorage ne "1") {
5230                     $storeid = $targetstorage;
5231                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5232                     $format = $defFormat;
5233                 } else {
5234                     #else we use same format than original
5235                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5236                     $format = qemu_img_format($scfg, $volid);
5237                 }
5238
5239                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5240                 my $newdrive = $drive;
5241                 $newdrive->{format} = $format;
5242                 $newdrive->{file} = $newvolid;
5243                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5244                 $local_volumes->{$opt} = $drivestr;
5245                 #pass drive to conf for command line
5246                 $conf->{$opt} = $drivestr;
5247             }
5248         }
5249
5250         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5251
5252         if ($is_suspended) {
5253             # enforce machine type on suspended vm to ensure HW compatibility
5254             $forcemachine = $conf->{runningmachine};
5255             print "Resuming suspended VM\n";
5256         }
5257
5258         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5259
5260         my $migrate_port = 0;
5261         my $migrate_uri;
5262         if ($statefile) {
5263             if ($statefile eq 'tcp') {
5264                 my $localip = "localhost";
5265                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5266                 my $nodename = PVE::INotify::nodename();
5267
5268                 if (!defined($migration_type)) {
5269                     if (defined($datacenterconf->{migration}->{type})) {
5270                         $migration_type = $datacenterconf->{migration}->{type};
5271                     } else {
5272                         $migration_type = 'secure';
5273                     }
5274                 }
5275
5276                 if ($migration_type eq 'insecure') {
5277                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5278                     if ($migrate_network_addr) {
5279                         $localip = $migrate_network_addr;
5280                     } else {
5281                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5282                     }
5283
5284                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5285                 }
5286
5287                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5288                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5289                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5290                 push @$cmd, '-incoming', $migrate_uri;
5291                 push @$cmd, '-S';
5292
5293             } elsif ($statefile eq 'unix') {
5294                 # should be default for secure migrations as a ssh TCP forward
5295                 # tunnel is not deterministic reliable ready and fails regurarly
5296                 # to set up in time, so use UNIX socket forwards
5297                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5298                 unlink $socket_addr;
5299
5300                 $migrate_uri = "unix:$socket_addr";
5301
5302                 push @$cmd, '-incoming', $migrate_uri;
5303                 push @$cmd, '-S';
5304
5305             } else {
5306                 push @$cmd, '-loadstate', $statefile;
5307             }
5308         } elsif ($paused) {
5309             push @$cmd, '-S';
5310         }
5311
5312         # host pci devices
5313         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5314           my $d = parse_hostpci($conf->{"hostpci$i"});
5315           next if !$d;
5316           my $pcidevices = $d->{pciid};
5317           foreach my $pcidevice (@$pcidevices) {
5318                 my $pciid = $pcidevice->{id};
5319
5320                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5321                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5322                 die "no pci device info for device '$pciid'\n" if !$info;
5323
5324                 if ($d->{mdev}) {
5325                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5326                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5327                 } else {
5328                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5329                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5330                     die "can't reset pci device '$pciid'\n"
5331                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5332                 }
5333           }
5334         }
5335
5336         PVE::Storage::activate_volumes($storecfg, $vollist);
5337
5338         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5339             eval {
5340                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5341                     outfunc => sub {}, errfunc => sub {});
5342             };
5343         }
5344
5345         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5346                                                   : $defaults->{cpuunits};
5347
5348         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5349         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5350
5351         my %properties = (
5352             Slice => 'qemu.slice',
5353             KillMode => 'none',
5354             CPUShares => $cpuunits
5355         );
5356
5357         if (my $cpulimit = $conf->{cpulimit}) {
5358             $properties{CPUQuota} = int($cpulimit * 100);
5359         }
5360         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5361
5362         my $run_qemu = sub {
5363             PVE::Tools::run_fork sub {
5364                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5365                 run_command($cmd, %run_params);
5366             };
5367         };
5368
5369         if ($conf->{hugepages}) {
5370
5371             my $code = sub {
5372                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5373                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5374
5375                 PVE::QemuServer::Memory::hugepages_mount();
5376                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5377
5378                 eval { $run_qemu->() };
5379                 if (my $err = $@) {
5380                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5381                     die $err;
5382                 }
5383
5384                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5385             };
5386             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5387
5388         } else {
5389             eval { $run_qemu->() };
5390         }
5391
5392         if (my $err = $@) {
5393             # deactivate volumes if start fails
5394             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5395             die "start failed: $err";
5396         }
5397
5398         print "migration listens on $migrate_uri\n" if $migrate_uri;
5399
5400         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5401             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5402             warn $@ if $@;
5403         }
5404
5405         #start nbd server for storage migration
5406         if ($targetstorage) {
5407             my $nodename = PVE::INotify::nodename();
5408             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5409             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5410             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5411             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5412
5413             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5414
5415             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5416
5417             foreach my $opt (sort keys %$local_volumes) {
5418                 my $volid = $local_volumes->{$opt};
5419                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5420                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5421                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5422             }
5423         }
5424
5425         if ($migratedfrom) {
5426             eval {
5427                 set_migration_caps($vmid);
5428             };
5429             warn $@ if $@;
5430
5431             if ($spice_port) {
5432                 print "spice listens on port $spice_port\n";
5433                 if ($spice_ticket) {
5434                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5435                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5436                 }
5437             }
5438
5439         } else {
5440             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5441                 if !$statefile && $conf->{balloon};
5442
5443             foreach my $opt (keys %$conf) {
5444                 next if $opt !~  m/^net\d+$/;
5445                 my $nicconf = parse_net($conf->{$opt});
5446                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5447             }
5448         }
5449
5450         vm_mon_cmd_nocheck($vmid, 'qom-set',
5451                     path => "machine/peripheral/balloon0",
5452                     property => "guest-stats-polling-interval",
5453                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5454
5455         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5456             print "Resumed VM, removing state\n";
5457             delete $conf->@{qw(lock vmstate runningmachine)};
5458             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5459             PVE::Storage::vdisk_free($storecfg, $vmstate);
5460             PVE::QemuConfig->write_config($vmid, $conf);
5461         }
5462
5463         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5464     });
5465 }
5466
5467 sub vm_mon_cmd {
5468     my ($vmid, $execute, %params) = @_;
5469
5470     my $cmd = { execute => $execute, arguments => \%params };
5471     vm_qmp_command($vmid, $cmd);
5472 }
5473
5474 sub vm_mon_cmd_nocheck {
5475     my ($vmid, $execute, %params) = @_;
5476
5477     my $cmd = { execute => $execute, arguments => \%params };
5478     vm_qmp_command($vmid, $cmd, 1);
5479 }
5480
5481 sub vm_qmp_command {
5482     my ($vmid, $cmd, $nocheck) = @_;
5483
5484     my $res;
5485
5486     my $timeout;
5487     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5488         $timeout = $cmd->{arguments}->{timeout};
5489         delete $cmd->{arguments}->{timeout};
5490     }
5491
5492     eval {
5493         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5494         my $sname = qmp_socket($vmid);
5495         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5496             my $qmpclient = PVE::QMPClient->new();
5497
5498             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5499         } else {
5500             die "unable to open monitor socket\n";
5501         }
5502     };
5503     if (my $err = $@) {
5504         syslog("err", "VM $vmid qmp command failed - $err");
5505         die $err;
5506     }
5507
5508     return $res;
5509 }
5510
5511 sub vm_human_monitor_command {
5512     my ($vmid, $cmdline) = @_;
5513
5514     my $res;
5515
5516     my $cmd = {
5517         execute => 'human-monitor-command',
5518         arguments => { 'command-line' => $cmdline},
5519     };
5520
5521     return vm_qmp_command($vmid, $cmd);
5522 }
5523
5524 sub vm_commandline {
5525     my ($storecfg, $vmid, $snapname) = @_;
5526
5527     my $conf = PVE::QemuConfig->load_config($vmid);
5528
5529     if ($snapname) {
5530         my $snapshot = $conf->{snapshots}->{$snapname};
5531         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5532
5533         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5534
5535         $conf = $snapshot;
5536     }
5537
5538     my $defaults = load_defaults();
5539
5540     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5541
5542     return PVE::Tools::cmd2string($cmd);
5543 }
5544
5545 sub vm_reset {
5546     my ($vmid, $skiplock) = @_;
5547
5548     PVE::QemuConfig->lock_config($vmid, sub {
5549
5550         my $conf = PVE::QemuConfig->load_config($vmid);
5551
5552         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5553
5554         vm_mon_cmd($vmid, "system_reset");
5555     });
5556 }
5557
5558 sub get_vm_volumes {
5559     my ($conf) = @_;
5560
5561     my $vollist = [];
5562     foreach_volid($conf, sub {
5563         my ($volid, $attr) = @_;
5564
5565         return if $volid =~ m|^/|;
5566
5567         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5568         return if !$sid;
5569
5570         push @$vollist, $volid;
5571     });
5572
5573     return $vollist;
5574 }
5575
5576 sub vm_stop_cleanup {
5577     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5578
5579     eval {
5580
5581         if (!$keepActive) {
5582             my $vollist = get_vm_volumes($conf);
5583             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5584         }
5585
5586         foreach my $ext (qw(mon qmp pid vnc qga)) {
5587             unlink "/var/run/qemu-server/${vmid}.$ext";
5588         }
5589
5590         if ($conf->{ivshmem}) {
5591             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5592             # just delete it for now, VMs which have this already open do not
5593             # are affected, but new VMs will get a separated one. If this
5594             # becomes an issue we either add some sort of ref-counting or just
5595             # add a "don't delete on stop" flag to the ivshmem format.
5596             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5597         }
5598
5599         foreach my $key (keys %$conf) {
5600             next if $key !~ m/^hostpci(\d+)$/;
5601             my $hostpciindex = $1;
5602             my $d = parse_hostpci($conf->{$key});
5603             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5604
5605             foreach my $pci (@{$d->{pciid}}) {
5606                 my $pciid = $pci->{id};
5607                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5608             }
5609         }
5610
5611         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5612     };
5613     warn $@ if $@; # avoid errors - just warn
5614 }
5615
5616 # Note: use $nockeck to skip tests if VM configuration file exists.
5617 # We need that when migration VMs to other nodes (files already moved)
5618 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5619 sub vm_stop {
5620     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5621
5622     $force = 1 if !defined($force) && !$shutdown;
5623
5624     if ($migratedfrom){
5625         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5626         kill 15, $pid if $pid;
5627         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5628         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5629         return;
5630     }
5631
5632     PVE::QemuConfig->lock_config($vmid, sub {
5633
5634         my $pid = check_running($vmid, $nocheck);
5635         return if !$pid;
5636
5637         my $conf;
5638         if (!$nocheck) {
5639             $conf = PVE::QemuConfig->load_config($vmid);
5640             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5641             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5642                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5643                 $timeout = $opts->{down} if $opts->{down};
5644             }
5645             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5646         }
5647
5648         $timeout = 60 if !defined($timeout);
5649
5650         eval {
5651             if ($shutdown) {
5652                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5653                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5654                 } else {
5655                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5656                 }
5657             } else {
5658                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5659             }
5660         };
5661         my $err = $@;
5662
5663         if (!$err) {
5664             my $count = 0;
5665             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5666                 $count++;
5667                 sleep 1;
5668             }
5669
5670             if ($count >= $timeout) {
5671                 if ($force) {
5672                     warn "VM still running - terminating now with SIGTERM\n";
5673                     kill 15, $pid;
5674                 } else {
5675                     die "VM quit/powerdown failed - got timeout\n";
5676                 }
5677             } else {
5678                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5679                 return;
5680             }
5681         } else {
5682             if ($force) {
5683                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5684                 kill 15, $pid;
5685             } else {
5686                 die "VM quit/powerdown failed\n";
5687             }
5688         }
5689
5690         # wait again
5691         $timeout = 10;
5692
5693         my $count = 0;
5694         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5695             $count++;
5696             sleep 1;
5697         }
5698
5699         if ($count >= $timeout) {
5700             warn "VM still running - terminating now with SIGKILL\n";
5701             kill 9, $pid;
5702             sleep 1;
5703         }
5704
5705         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5706    });
5707 }
5708
5709 sub vm_suspend {
5710     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5711
5712     my $conf;
5713     my $path;
5714     my $storecfg;
5715     my $vmstate;
5716
5717     PVE::QemuConfig->lock_config($vmid, sub {
5718
5719         $conf = PVE::QemuConfig->load_config($vmid);
5720
5721         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5722         PVE::QemuConfig->check_lock($conf)
5723             if !($skiplock || $is_backing_up);
5724
5725         die "cannot suspend to disk during backup\n"
5726             if $is_backing_up && $includestate;
5727
5728         if ($includestate) {
5729             $conf->{lock} = 'suspending';
5730             my $date = strftime("%Y-%m-%d", localtime(time()));
5731             $storecfg = PVE::Storage::config();
5732             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5733             $path = PVE::Storage::path($storecfg, $vmstate);
5734             PVE::QemuConfig->write_config($vmid, $conf);
5735         } else {
5736             vm_mon_cmd($vmid, "stop");
5737         }
5738     });
5739
5740     if ($includestate) {
5741         # save vm state
5742         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5743
5744         eval {
5745             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5746             for(;;) {
5747                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5748                 if (!$state->{status}) {
5749                     die "savevm not active\n";
5750                 } elsif ($state->{status} eq 'active') {
5751                     sleep(1);
5752                     next;
5753                 } elsif ($state->{status} eq 'completed') {
5754                     print "State saved, quitting\n";
5755                     last;
5756                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5757                     die "query-savevm failed with error '$state->{error}'\n"
5758                 } else {
5759                     die "query-savevm returned status '$state->{status}'\n";
5760                 }
5761             }
5762         };
5763         my $err = $@;
5764
5765         PVE::QemuConfig->lock_config($vmid, sub {
5766             $conf = PVE::QemuConfig->load_config($vmid);
5767             if ($err) {
5768                 # cleanup, but leave suspending lock, to indicate something went wrong
5769                 eval {
5770                     vm_mon_cmd($vmid, "savevm-end");
5771                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5772                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5773                     delete $conf->@{qw(vmstate runningmachine)};
5774                     PVE::QemuConfig->write_config($vmid, $conf);
5775                 };
5776                 warn $@ if $@;
5777                 die $err;
5778             }
5779
5780             die "lock changed unexpectedly\n"
5781                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5782
5783             vm_qmp_command($vmid, { execute => "quit" });
5784             $conf->{lock} = 'suspended';
5785             PVE::QemuConfig->write_config($vmid, $conf);
5786         });
5787     }
5788 }
5789
5790 sub vm_resume {
5791     my ($vmid, $skiplock, $nocheck) = @_;
5792
5793     PVE::QemuConfig->lock_config($vmid, sub {
5794         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5795         my $res = $vm_mon_cmd->($vmid, 'query-status');
5796         my $resume_cmd = 'cont';
5797
5798         if ($res->{status} && $res->{status} eq 'suspended') {
5799             $resume_cmd = 'system_wakeup';
5800         }
5801
5802         if (!$nocheck) {
5803
5804             my $conf = PVE::QemuConfig->load_config($vmid);
5805
5806             PVE::QemuConfig->check_lock($conf)
5807                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5808         }
5809
5810         $vm_mon_cmd->($vmid, $resume_cmd);
5811     });
5812 }
5813
5814 sub vm_sendkey {
5815     my ($vmid, $skiplock, $key) = @_;
5816
5817     PVE::QemuConfig->lock_config($vmid, sub {
5818
5819         my $conf = PVE::QemuConfig->load_config($vmid);
5820
5821         # there is no qmp command, so we use the human monitor command
5822         vm_human_monitor_command($vmid, "sendkey $key");
5823     });
5824 }
5825
5826 sub vm_destroy {
5827     my ($storecfg, $vmid, $skiplock) = @_;
5828
5829     PVE::QemuConfig->lock_config($vmid, sub {
5830
5831         my $conf = PVE::QemuConfig->load_config($vmid);
5832
5833         if (!check_running($vmid)) {
5834             destroy_vm($storecfg, $vmid, undef, $skiplock);
5835         } else {
5836             die "VM $vmid is running - destroy failed\n";
5837         }
5838     });
5839 }
5840
5841 # vzdump restore implementaion
5842
5843 sub tar_archive_read_firstfile {
5844     my $archive = shift;
5845
5846     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5847
5848     # try to detect archive type first
5849     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5850         die "unable to open file '$archive'\n";
5851     my $firstfile = <$fh>;
5852     kill 15, $pid;
5853     close $fh;
5854
5855     die "ERROR: archive contaions no data\n" if !$firstfile;
5856     chomp $firstfile;
5857
5858     return $firstfile;
5859 }
5860
5861 sub tar_restore_cleanup {
5862     my ($storecfg, $statfile) = @_;
5863
5864     print STDERR "starting cleanup\n";
5865
5866     if (my $fd = IO::File->new($statfile, "r")) {
5867         while (defined(my $line = <$fd>)) {
5868             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5869                 my $volid = $2;
5870                 eval {
5871                     if ($volid =~ m|^/|) {
5872                         unlink $volid || die 'unlink failed\n';
5873                     } else {
5874                         PVE::Storage::vdisk_free($storecfg, $volid);
5875                     }
5876                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5877                 };
5878                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5879             } else {
5880                 print STDERR "unable to parse line in statfile - $line";
5881             }
5882         }
5883         $fd->close();
5884     }
5885 }
5886
5887 sub restore_archive {
5888     my ($archive, $vmid, $user, $opts) = @_;
5889
5890     my $format = $opts->{format};
5891     my $comp;
5892
5893     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5894         $format = 'tar' if !$format;
5895         $comp = 'gzip';
5896     } elsif ($archive =~ m/\.tar$/) {
5897         $format = 'tar' if !$format;
5898     } elsif ($archive =~ m/.tar.lzo$/) {
5899         $format = 'tar' if !$format;
5900         $comp = 'lzop';
5901     } elsif ($archive =~ m/\.vma$/) {
5902         $format = 'vma' if !$format;
5903     } elsif ($archive =~ m/\.vma\.gz$/) {
5904         $format = 'vma' if !$format;
5905         $comp = 'gzip';
5906     } elsif ($archive =~ m/\.vma\.lzo$/) {
5907         $format = 'vma' if !$format;
5908         $comp = 'lzop';
5909     } else {
5910         $format = 'vma' if !$format; # default
5911     }
5912
5913     # try to detect archive format
5914     if ($format eq 'tar') {
5915         return restore_tar_archive($archive, $vmid, $user, $opts);
5916     } else {
5917         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5918     }
5919 }
5920
5921 sub restore_update_config_line {
5922     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5923
5924     return if $line =~ m/^\#qmdump\#/;
5925     return if $line =~ m/^\#vzdump\#/;
5926     return if $line =~ m/^lock:/;
5927     return if $line =~ m/^unused\d+:/;
5928     return if $line =~ m/^parent:/;
5929
5930     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5931     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5932         # try to convert old 1.X settings
5933         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5934         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5935             my ($model, $macaddr) = split(/\=/, $devconfig);
5936             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5937             my $net = {
5938                 model => $model,
5939                 bridge => "vmbr$ind",
5940                 macaddr => $macaddr,
5941             };
5942             my $netstr = print_net($net);
5943
5944             print $outfd "net$cookie->{netcount}: $netstr\n";
5945             $cookie->{netcount}++;
5946         }
5947     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5948         my ($id, $netstr) = ($1, $2);
5949         my $net = parse_net($netstr);
5950         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5951         $netstr = print_net($net);
5952         print $outfd "$id: $netstr\n";
5953     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5954         my $virtdev = $1;
5955         my $value = $3;
5956         my $di = parse_drive($virtdev, $value);
5957         if (defined($di->{backup}) && !$di->{backup}) {
5958             print $outfd "#$line";
5959         } elsif ($map->{$virtdev}) {
5960             delete $di->{format}; # format can change on restore
5961             $di->{file} = $map->{$virtdev};
5962             $value = print_drive($vmid, $di);
5963             print $outfd "$virtdev: $value\n";
5964         } else {
5965             print $outfd $line;
5966         }
5967     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5968         my $vmgenid = $1;
5969         if ($vmgenid ne '0') {
5970             # always generate a new vmgenid if there was a valid one setup
5971             $vmgenid = generate_uuid();
5972         }
5973         print $outfd "vmgenid: $vmgenid\n";
5974     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5975         my ($uuid, $uuid_str);
5976         UUID::generate($uuid);
5977         UUID::unparse($uuid, $uuid_str);
5978         my $smbios1 = parse_smbios1($2);
5979         $smbios1->{uuid} = $uuid_str;
5980         print $outfd $1.print_smbios1($smbios1)."\n";
5981     } else {
5982         print $outfd $line;
5983     }
5984 }
5985
5986 sub scan_volids {
5987     my ($cfg, $vmid) = @_;
5988
5989     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5990
5991     my $volid_hash = {};
5992     foreach my $storeid (keys %$info) {
5993         foreach my $item (@{$info->{$storeid}}) {
5994             next if !($item->{volid} && $item->{size});
5995             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5996             $volid_hash->{$item->{volid}} = $item;
5997         }
5998     }
5999
6000     return $volid_hash;
6001 }
6002
6003 sub is_volume_in_use {
6004     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6005
6006     my $path = PVE::Storage::path($storecfg, $volid);
6007
6008     my $scan_config = sub {
6009         my ($cref, $snapname) = @_;
6010
6011         foreach my $key (keys %$cref) {
6012             my $value = $cref->{$key};
6013             if (is_valid_drivename($key)) {
6014                 next if $skip_drive && $key eq $skip_drive;
6015                 my $drive = parse_drive($key, $value);
6016                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6017                 return 1 if $volid eq $drive->{file};
6018                 if ($drive->{file} =~ m!^/!) {
6019                     return 1 if $drive->{file} eq $path;
6020                 } else {
6021                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6022                     next if !$storeid;
6023                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6024                     next if !$scfg;
6025                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6026                 }
6027             }
6028         }
6029
6030         return 0;
6031     };
6032
6033     return 1 if &$scan_config($conf);
6034
6035     undef $skip_drive;
6036
6037     foreach my $snapname (keys %{$conf->{snapshots}}) {
6038         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6039     }
6040
6041     return 0;
6042 }
6043
6044 sub update_disksize {
6045     my ($vmid, $conf, $volid_hash) = @_;
6046
6047     my $changes;
6048     my $prefix = "VM $vmid:";
6049
6050     # used and unused disks
6051     my $referenced = {};
6052
6053     # Note: it is allowed to define multiple storages with same path (alias), so
6054     # we need to check both 'volid' and real 'path' (two different volid can point
6055     # to the same path).
6056
6057     my $referencedpath = {};
6058
6059     # update size info
6060     foreach my $opt (keys %$conf) {
6061         if (is_valid_drivename($opt)) {
6062             my $drive = parse_drive($opt, $conf->{$opt});
6063             my $volid = $drive->{file};
6064             next if !$volid;
6065
6066             $referenced->{$volid} = 1;
6067             if ($volid_hash->{$volid} &&
6068                 (my $path = $volid_hash->{$volid}->{path})) {
6069                 $referencedpath->{$path} = 1;
6070             }
6071
6072             next if drive_is_cdrom($drive);
6073             next if !$volid_hash->{$volid};
6074
6075             $drive->{size} = $volid_hash->{$volid}->{size};
6076             my $new = print_drive($vmid, $drive);
6077             if ($new ne $conf->{$opt}) {
6078                 $changes = 1;
6079                 $conf->{$opt} = $new;
6080                 print "$prefix update disk '$opt' information.\n";
6081             }
6082         }
6083     }
6084
6085     # remove 'unusedX' entry if volume is used
6086     foreach my $opt (keys %$conf) {
6087         next if $opt !~ m/^unused\d+$/;
6088         my $volid = $conf->{$opt};
6089         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6090         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6091             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6092             $changes = 1;
6093             delete $conf->{$opt};
6094         }
6095
6096         $referenced->{$volid} = 1;
6097         $referencedpath->{$path} = 1 if $path;
6098     }
6099
6100     foreach my $volid (sort keys %$volid_hash) {
6101         next if $volid =~ m/vm-$vmid-state-/;
6102         next if $referenced->{$volid};
6103         my $path = $volid_hash->{$volid}->{path};
6104         next if !$path; # just to be sure
6105         next if $referencedpath->{$path};
6106         $changes = 1;
6107         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6108         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6109         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6110     }
6111
6112     return $changes;
6113 }
6114
6115 sub rescan {
6116     my ($vmid, $nolock, $dryrun) = @_;
6117
6118     my $cfg = PVE::Storage::config();
6119
6120     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6121     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6122     foreach my $stor (keys %{$cfg->{ids}}) {
6123         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6124     }
6125
6126     print "rescan volumes...\n";
6127     my $volid_hash = scan_volids($cfg, $vmid);
6128
6129     my $updatefn =  sub {
6130         my ($vmid) = @_;
6131
6132         my $conf = PVE::QemuConfig->load_config($vmid);
6133
6134         PVE::QemuConfig->check_lock($conf);
6135
6136         my $vm_volids = {};
6137         foreach my $volid (keys %$volid_hash) {
6138             my $info = $volid_hash->{$volid};
6139             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6140         }
6141
6142         my $changes = update_disksize($vmid, $conf, $vm_volids);
6143
6144         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6145     };
6146
6147     if (defined($vmid)) {
6148         if ($nolock) {
6149             &$updatefn($vmid);
6150         } else {
6151             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6152         }
6153     } else {
6154         my $vmlist = config_list();
6155         foreach my $vmid (keys %$vmlist) {
6156             if ($nolock) {
6157                 &$updatefn($vmid);
6158             } else {
6159                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6160             }
6161         }
6162     }
6163 }
6164
6165 sub restore_vma_archive {
6166     my ($archive, $vmid, $user, $opts, $comp) = @_;
6167
6168     my $readfrom = $archive;
6169
6170     my $cfg = PVE::Storage::config();
6171     my $commands = [];
6172     my $bwlimit = $opts->{bwlimit};
6173
6174     my $dbg_cmdstring = '';
6175     my $add_pipe = sub {
6176         my ($cmd) = @_;
6177         push @$commands, $cmd;
6178         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6179         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6180         $readfrom = '-';
6181     };
6182
6183     my $input = undef;
6184     if ($archive eq '-') {
6185         $input = '<&STDIN';
6186     } else {
6187         # If we use a backup from a PVE defined storage we also consider that
6188         # storage's rate limit:
6189         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6190         if (defined($volid)) {
6191             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6192             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6193             if ($readlimit) {
6194                 print STDERR "applying read rate limit: $readlimit\n";
6195                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6196                 $add_pipe->($cstream);
6197             }
6198         }
6199     }
6200
6201     if ($comp) {
6202         my $cmd;
6203         if ($comp eq 'gzip') {
6204             $cmd = ['zcat', $readfrom];
6205         } elsif ($comp eq 'lzop') {
6206             $cmd = ['lzop', '-d', '-c', $readfrom];
6207         } else {
6208             die "unknown compression method '$comp'\n";
6209         }
6210         $add_pipe->($cmd);
6211     }
6212
6213     my $tmpdir = "/var/tmp/vzdumptmp$$";
6214     rmtree $tmpdir;
6215
6216     # disable interrupts (always do cleanups)
6217     local $SIG{INT} =
6218         local $SIG{TERM} =
6219         local $SIG{QUIT} =
6220         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6221
6222     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6223     POSIX::mkfifo($mapfifo, 0600);
6224     my $fifofh;
6225
6226     my $openfifo = sub {
6227         open($fifofh, '>', $mapfifo) || die $!;
6228     };
6229
6230     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6231
6232     my $oldtimeout;
6233     my $timeout = 5;
6234
6235     my $devinfo = {};
6236
6237     my $rpcenv = PVE::RPCEnvironment::get();
6238
6239     my $conffile = PVE::QemuConfig->config_file($vmid);
6240     my $tmpfn = "$conffile.$$.tmp";
6241
6242     # Note: $oldconf is undef if VM does not exists
6243     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6244     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6245
6246     my %storage_limits;
6247
6248     my $print_devmap = sub {
6249         my $virtdev_hash = {};
6250
6251         my $cfgfn = "$tmpdir/qemu-server.conf";
6252
6253         # we can read the config - that is already extracted
6254         my $fh = IO::File->new($cfgfn, "r") ||
6255             "unable to read qemu-server.conf - $!\n";
6256
6257         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6258         if (-f $fwcfgfn) {
6259             my $pve_firewall_dir = '/etc/pve/firewall';
6260             mkdir $pve_firewall_dir; # make sure the dir exists
6261             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6262         }
6263
6264         while (defined(my $line = <$fh>)) {
6265             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6266                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6267                 die "archive does not contain data for drive '$virtdev'\n"
6268                     if !$devinfo->{$devname};
6269                 if (defined($opts->{storage})) {
6270                     $storeid = $opts->{storage} || 'local';
6271                 } elsif (!$storeid) {
6272                     $storeid = 'local';
6273                 }
6274                 $format = 'raw' if !$format;
6275                 $devinfo->{$devname}->{devname} = $devname;
6276                 $devinfo->{$devname}->{virtdev} = $virtdev;
6277                 $devinfo->{$devname}->{format} = $format;
6278                 $devinfo->{$devname}->{storeid} = $storeid;
6279
6280                 # check permission on storage
6281                 my $pool = $opts->{pool}; # todo: do we need that?
6282                 if ($user ne 'root@pam') {
6283                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6284                 }
6285
6286                 $storage_limits{$storeid} = $bwlimit;
6287
6288                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6289             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6290                 my $virtdev = $1;
6291                 my $drive = parse_drive($virtdev, $2);
6292                 if (drive_is_cloudinit($drive)) {
6293                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6294                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6295                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6296
6297                     my $d = {
6298                         format => $format,
6299                         storeid => $opts->{storage} // $storeid,
6300                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6301                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6302                         name => "vm-$vmid-cloudinit",
6303                         is_cloudinit => 1,
6304                     };
6305                     $virtdev_hash->{$virtdev} = $d;
6306                 }
6307             }
6308         }
6309
6310         foreach my $key (keys %storage_limits) {
6311             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6312             next if !$limit;
6313             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6314             $storage_limits{$key} = $limit * 1024;
6315         }
6316
6317         foreach my $devname (keys %$devinfo) {
6318             die "found no device mapping information for device '$devname'\n"
6319                 if !$devinfo->{$devname}->{virtdev};
6320         }
6321
6322         # create empty/temp config
6323         if ($oldconf) {
6324             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6325             foreach_drive($oldconf, sub {
6326                 my ($ds, $drive) = @_;
6327
6328                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6329
6330                 my $volid = $drive->{file};
6331                 return if !$volid || $volid =~ m|^/|;
6332
6333                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6334                 return if !$path || !$owner || ($owner != $vmid);
6335
6336                 # Note: only delete disk we want to restore
6337                 # other volumes will become unused
6338                 if ($virtdev_hash->{$ds}) {
6339                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6340                     if (my $err = $@) {
6341                         warn $err;
6342                     }
6343                 }
6344             });
6345
6346             # delete vmstate files, after the restore we have no snapshots anymore
6347             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6348                 my $snap = $oldconf->{snapshots}->{$snapname};
6349                 if ($snap->{vmstate}) {
6350                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6351                     if (my $err = $@) {
6352                         warn $err;
6353                     }
6354                 }
6355             }
6356         }
6357
6358         my $map = {};
6359         foreach my $virtdev (sort keys %$virtdev_hash) {
6360             my $d = $virtdev_hash->{$virtdev};
6361             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6362             my $storeid = $d->{storeid};
6363             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6364
6365             my $map_opts = '';
6366             if (my $limit = $storage_limits{$storeid}) {
6367                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6368             }
6369
6370             # test if requested format is supported
6371             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6372             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6373             $d->{format} = $defFormat if !$supported;
6374
6375             my $name;
6376             if ($d->{is_cloudinit}) {
6377                 $name = $d->{name};
6378                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6379             }
6380
6381             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6382             print STDERR "new volume ID is '$volid'\n";
6383             $d->{volid} = $volid;
6384
6385             PVE::Storage::activate_volumes($cfg, [$volid]);
6386
6387             my $write_zeros = 1;
6388             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6389                 $write_zeros = 0;
6390             }
6391
6392             if (!$d->{is_cloudinit}) {
6393                 my $path = PVE::Storage::path($cfg, $volid);
6394
6395                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6396
6397                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6398             }
6399             $map->{$virtdev} = $volid;
6400         }
6401
6402         $fh->seek(0, 0) || die "seek failed - $!\n";
6403
6404         my $outfd = new IO::File ($tmpfn, "w") ||
6405             die "unable to write config for VM $vmid\n";
6406
6407         my $cookie = { netcount => 0 };
6408         while (defined(my $line = <$fh>)) {
6409             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6410         }
6411
6412         $fh->close();
6413         $outfd->close();
6414     };
6415
6416     eval {
6417         # enable interrupts
6418         local $SIG{INT} =
6419             local $SIG{TERM} =
6420             local $SIG{QUIT} =
6421             local $SIG{HUP} =
6422             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6423         local $SIG{ALRM} = sub { die "got timeout\n"; };
6424
6425         $oldtimeout = alarm($timeout);
6426
6427         my $parser = sub {
6428             my $line = shift;
6429
6430             print "$line\n";
6431
6432             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6433                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6434                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6435             } elsif ($line =~ m/^CTIME: /) {
6436                 # we correctly received the vma config, so we can disable
6437                 # the timeout now for disk allocation (set to 10 minutes, so
6438                 # that we always timeout if something goes wrong)
6439                 alarm(600);
6440                 &$print_devmap();
6441                 print $fifofh "done\n";
6442                 my $tmp = $oldtimeout || 0;
6443                 $oldtimeout = undef;
6444                 alarm($tmp);
6445                 close($fifofh);
6446             }
6447         };
6448
6449         print "restore vma archive: $dbg_cmdstring\n";
6450         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6451     };
6452     my $err = $@;
6453
6454     alarm($oldtimeout) if $oldtimeout;
6455
6456     my $vollist = [];
6457     foreach my $devname (keys %$devinfo) {
6458         my $volid = $devinfo->{$devname}->{volid};
6459         push @$vollist, $volid if $volid;
6460     }
6461
6462     PVE::Storage::deactivate_volumes($cfg, $vollist);
6463
6464     unlink $mapfifo;
6465
6466     if ($err) {
6467         rmtree $tmpdir;
6468         unlink $tmpfn;
6469
6470         foreach my $devname (keys %$devinfo) {
6471             my $volid = $devinfo->{$devname}->{volid};
6472             next if !$volid;
6473             eval {
6474                 if ($volid =~ m|^/|) {
6475                     unlink $volid || die 'unlink failed\n';
6476                 } else {
6477                     PVE::Storage::vdisk_free($cfg, $volid);
6478                 }
6479                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6480             };
6481             print STDERR "unable to cleanup '$volid' - $@" if $@;
6482         }
6483         die $err;
6484     }
6485
6486     rmtree $tmpdir;
6487
6488     rename($tmpfn, $conffile) ||
6489         die "unable to commit configuration file '$conffile'\n";
6490
6491     PVE::Cluster::cfs_update(); # make sure we read new file
6492
6493     eval { rescan($vmid, 1); };
6494     warn $@ if $@;
6495 }
6496
6497 sub restore_tar_archive {
6498     my ($archive, $vmid, $user, $opts) = @_;
6499
6500     if ($archive ne '-') {
6501         my $firstfile = tar_archive_read_firstfile($archive);
6502         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6503             if $firstfile ne 'qemu-server.conf';
6504     }
6505
6506     my $storecfg = PVE::Storage::config();
6507
6508     # destroy existing data - keep empty config
6509     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6510     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6511
6512     my $tocmd = "/usr/lib/qemu-server/qmextract";
6513
6514     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6515     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6516     $tocmd .= ' --prealloc' if $opts->{prealloc};
6517     $tocmd .= ' --info' if $opts->{info};
6518
6519     # tar option "xf" does not autodetect compression when read from STDIN,
6520     # so we pipe to zcat
6521     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6522         PVE::Tools::shellquote("--to-command=$tocmd");
6523
6524     my $tmpdir = "/var/tmp/vzdumptmp$$";
6525     mkpath $tmpdir;
6526
6527     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6528     local $ENV{VZDUMP_VMID} = $vmid;
6529     local $ENV{VZDUMP_USER} = $user;
6530
6531     my $conffile = PVE::QemuConfig->config_file($vmid);
6532     my $tmpfn = "$conffile.$$.tmp";
6533
6534     # disable interrupts (always do cleanups)
6535     local $SIG{INT} =
6536         local $SIG{TERM} =
6537         local $SIG{QUIT} =
6538         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6539
6540     eval {
6541         # enable interrupts
6542         local $SIG{INT} =
6543             local $SIG{TERM} =
6544             local $SIG{QUIT} =
6545             local $SIG{HUP} =
6546             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6547
6548         if ($archive eq '-') {
6549             print "extracting archive from STDIN\n";
6550             run_command($cmd, input => "<&STDIN");
6551         } else {
6552             print "extracting archive '$archive'\n";
6553             run_command($cmd);
6554         }
6555
6556         return if $opts->{info};
6557
6558         # read new mapping
6559         my $map = {};
6560         my $statfile = "$tmpdir/qmrestore.stat";
6561         if (my $fd = IO::File->new($statfile, "r")) {
6562             while (defined (my $line = <$fd>)) {
6563                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6564                     $map->{$1} = $2 if $1;
6565                 } else {
6566                     print STDERR "unable to parse line in statfile - $line\n";
6567                 }
6568             }
6569             $fd->close();
6570         }
6571
6572         my $confsrc = "$tmpdir/qemu-server.conf";
6573
6574         my $srcfd = new IO::File($confsrc, "r") ||
6575             die "unable to open file '$confsrc'\n";
6576
6577         my $outfd = new IO::File ($tmpfn, "w") ||
6578             die "unable to write config for VM $vmid\n";
6579
6580         my $cookie = { netcount => 0 };
6581         while (defined (my $line = <$srcfd>)) {
6582             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6583         }
6584
6585         $srcfd->close();
6586         $outfd->close();
6587     };
6588     my $err = $@;
6589
6590     if ($err) {
6591
6592         unlink $tmpfn;
6593
6594         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6595
6596         die $err;
6597     }
6598
6599     rmtree $tmpdir;
6600
6601     rename $tmpfn, $conffile ||
6602         die "unable to commit configuration file '$conffile'\n";
6603
6604     PVE::Cluster::cfs_update(); # make sure we read new file
6605
6606     eval { rescan($vmid, 1); };
6607     warn $@ if $@;
6608 };
6609
6610 sub foreach_storage_used_by_vm {
6611     my ($conf, $func) = @_;
6612
6613     my $sidhash = {};
6614
6615     foreach_drive($conf, sub {
6616         my ($ds, $drive) = @_;
6617         return if drive_is_cdrom($drive);
6618
6619         my $volid = $drive->{file};
6620
6621         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6622         $sidhash->{$sid} = $sid if $sid;
6623     });
6624
6625     foreach my $sid (sort keys %$sidhash) {
6626         &$func($sid);
6627     }
6628 }
6629
6630 sub do_snapshots_with_qemu {
6631     my ($storecfg, $volid) = @_;
6632
6633     my $storage_name = PVE::Storage::parse_volume_id($volid);
6634     my $scfg = $storecfg->{ids}->{$storage_name};
6635
6636     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6637         return 1;
6638     }
6639
6640     if ($volid =~ m/\.(qcow2|qed)$/){
6641         return 1;
6642     }
6643
6644     return undef;
6645 }
6646
6647 sub qga_check_running {
6648     my ($vmid, $nowarn) = @_;
6649
6650     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6651     if ($@) {
6652         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6653         return 0;
6654     }
6655     return 1;
6656 }
6657
6658 sub template_create {
6659     my ($vmid, $conf, $disk) = @_;
6660
6661     my $storecfg = PVE::Storage::config();
6662
6663     foreach_drive($conf, sub {
6664         my ($ds, $drive) = @_;
6665
6666         return if drive_is_cdrom($drive);
6667         return if $disk && $ds ne $disk;
6668
6669         my $volid = $drive->{file};
6670         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6671
6672         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6673         $drive->{file} = $voliddst;
6674         $conf->{$ds} = print_drive($vmid, $drive);
6675         PVE::QemuConfig->write_config($vmid, $conf);
6676     });
6677 }
6678
6679 sub convert_iscsi_path {
6680     my ($path) = @_;
6681
6682     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6683         my $portal = $1;
6684         my $target = $2;
6685         my $lun = $3;
6686
6687         my $initiator_name = get_initiator_name();
6688
6689         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6690                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6691     }
6692
6693     die "cannot convert iscsi path '$path', unkown format\n";
6694 }
6695
6696 sub qemu_img_convert {
6697     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6698
6699     my $storecfg = PVE::Storage::config();
6700     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6701     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6702
6703     if ($src_storeid && $dst_storeid) {
6704
6705         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6706
6707         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6708         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6709
6710         my $src_format = qemu_img_format($src_scfg, $src_volname);
6711         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6712
6713         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6714         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6715
6716         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6717         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6718
6719         my $cmd = [];
6720         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6721         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6722         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6723         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6724
6725         if ($src_is_iscsi) {
6726             push @$cmd, '--image-opts';
6727             $src_path = convert_iscsi_path($src_path);
6728         } else {
6729             push @$cmd, '-f', $src_format;
6730         }
6731
6732         if ($dst_is_iscsi) {
6733             push @$cmd, '--target-image-opts';
6734             $dst_path = convert_iscsi_path($dst_path);
6735         } else {
6736             push @$cmd, '-O', $dst_format;
6737         }
6738
6739         push @$cmd, $src_path;
6740
6741         if (!$dst_is_iscsi && $is_zero_initialized) {
6742             push @$cmd, "zeroinit:$dst_path";
6743         } else {
6744             push @$cmd, $dst_path;
6745         }
6746
6747         my $parser = sub {
6748             my $line = shift;
6749             if($line =~ m/\((\S+)\/100\%\)/){
6750                 my $percent = $1;
6751                 my $transferred = int($size * $percent / 100);
6752                 my $remaining = $size - $transferred;
6753
6754                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6755             }
6756
6757         };
6758
6759         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6760         my $err = $@;
6761         die "copy failed: $err" if $err;
6762     }
6763 }
6764
6765 sub qemu_img_format {
6766     my ($scfg, $volname) = @_;
6767
6768     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6769         return $1;
6770     } else {
6771         return "raw";
6772     }
6773 }
6774
6775 sub qemu_drive_mirror {
6776     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6777
6778     $jobs = {} if !$jobs;
6779
6780     my $qemu_target;
6781     my $format;
6782     $jobs->{"drive-$drive"} = {};
6783
6784     if ($dst_volid =~ /^nbd:/) {
6785         $qemu_target = $dst_volid;
6786         $format = "nbd";
6787     } else {
6788         my $storecfg = PVE::Storage::config();
6789         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6790
6791         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6792
6793         $format = qemu_img_format($dst_scfg, $dst_volname);
6794
6795         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6796
6797         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6798     }
6799
6800     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6801     $opts->{format} = $format if $format;
6802
6803     if (defined($bwlimit)) {
6804         $opts->{speed} = $bwlimit * 1024;
6805         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6806     } else {
6807         print "drive mirror is starting for drive-$drive\n";
6808     }
6809
6810     # if a job already runs for this device we get an error, catch it for cleanup
6811     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6812     if (my $err = $@) {
6813         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6814         warn "$@\n" if $@;
6815         die "mirroring error: $err\n";
6816     }
6817
6818     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6819 }
6820
6821 sub qemu_drive_mirror_monitor {
6822     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6823
6824     eval {
6825         my $err_complete = 0;
6826
6827         while (1) {
6828             die "storage migration timed out\n" if $err_complete > 300;
6829
6830             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6831
6832             my $running_mirror_jobs = {};
6833             foreach my $stat (@$stats) {
6834                 next if $stat->{type} ne 'mirror';
6835                 $running_mirror_jobs->{$stat->{device}} = $stat;
6836             }
6837
6838             my $readycounter = 0;
6839
6840             foreach my $job (keys %$jobs) {
6841
6842                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6843                     print "$job : finished\n";
6844                     delete $jobs->{$job};
6845                     next;
6846                 }
6847
6848                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6849
6850                 my $busy = $running_mirror_jobs->{$job}->{busy};
6851                 my $ready = $running_mirror_jobs->{$job}->{ready};
6852                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6853                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6854                     my $remaining = $total - $transferred;
6855                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6856
6857                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6858                 }
6859
6860                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6861             }
6862
6863             last if scalar(keys %$jobs) == 0;
6864
6865             if ($readycounter == scalar(keys %$jobs)) {
6866                 print "all mirroring jobs are ready \n";
6867                 last if $skipcomplete; #do the complete later
6868
6869                 if ($vmiddst && $vmiddst != $vmid) {
6870                     my $agent_running = $qga && qga_check_running($vmid);
6871                     if ($agent_running) {
6872                         print "freeze filesystem\n";
6873                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6874                     } else {
6875                         print "suspend vm\n";
6876                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6877                     }
6878
6879                     # if we clone a disk for a new target vm, we don't switch the disk
6880                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6881
6882                     if ($agent_running) {
6883                         print "unfreeze filesystem\n";
6884                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6885                     } else {
6886                         print "resume vm\n";
6887                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6888                     }
6889
6890                     last;
6891                 } else {
6892
6893                     foreach my $job (keys %$jobs) {
6894                         # try to switch the disk if source and destination are on the same guest
6895                         print "$job: Completing block job...\n";
6896
6897                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6898                         if ($@ =~ m/cannot be completed/) {
6899                             print "$job: Block job cannot be completed, try again.\n";
6900                             $err_complete++;
6901                         }else {
6902                             print "$job: Completed successfully.\n";
6903                             $jobs->{$job}->{complete} = 1;
6904                         }
6905                     }
6906                 }
6907             }
6908             sleep 1;
6909         }
6910     };
6911     my $err = $@;
6912
6913     if ($err) {
6914         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6915         die "mirroring error: $err";
6916     }
6917
6918 }
6919
6920 sub qemu_blockjobs_cancel {
6921     my ($vmid, $jobs) = @_;
6922
6923     foreach my $job (keys %$jobs) {
6924         print "$job: Cancelling block job\n";
6925         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6926         $jobs->{$job}->{cancel} = 1;
6927     }
6928
6929     while (1) {
6930         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6931
6932         my $running_jobs = {};
6933         foreach my $stat (@$stats) {
6934             $running_jobs->{$stat->{device}} = $stat;
6935         }
6936
6937         foreach my $job (keys %$jobs) {
6938
6939             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6940                 print "$job: Done.\n";
6941                 delete $jobs->{$job};
6942             }
6943         }
6944
6945         last if scalar(keys %$jobs) == 0;
6946
6947         sleep 1;
6948     }
6949 }
6950
6951 sub clone_disk {
6952     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6953         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6954
6955     my $newvolid;
6956
6957     if (!$full) {
6958         print "create linked clone of drive $drivename ($drive->{file})\n";
6959         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6960         push @$newvollist, $newvolid;
6961     } else {
6962
6963         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6964         $storeid = $storage if $storage;
6965
6966         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6967         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6968
6969         print "create full clone of drive $drivename ($drive->{file})\n";
6970         my $name = undef;
6971         if (drive_is_cloudinit($drive)) {
6972             $name = "vm-$newvmid-cloudinit";
6973             $snapname = undef;
6974             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
6975             if ($dst_format ne 'raw') {
6976                 $name .= ".$dst_format";
6977             }
6978         }
6979         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6980         push @$newvollist, $newvolid;
6981
6982         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6983
6984         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6985         if (!$running || $snapname) {
6986             # TODO: handle bwlimits
6987             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6988         } else {
6989
6990             my $kvmver = get_running_qemu_version ($vmid);
6991             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6992                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6993                     if $drive->{iothread};
6994             }
6995
6996             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6997         }
6998     }
6999
7000     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7001
7002     my $disk = $drive;
7003     $disk->{format} = undef;
7004     $disk->{file} = $newvolid;
7005     $disk->{size} = $size;
7006
7007     return $disk;
7008 }
7009
7010 # this only works if VM is running
7011 sub get_current_qemu_machine {
7012     my ($vmid) = @_;
7013
7014     my $cmd = { execute => 'query-machines', arguments => {} };
7015     my $res = vm_qmp_command($vmid, $cmd);
7016
7017     my ($current, $default);
7018     foreach my $e (@$res) {
7019         $default = $e->{name} if $e->{'is-default'};
7020         $current = $e->{name} if $e->{'is-current'};
7021     }
7022
7023     # fallback to the default machine if current is not supported by qemu
7024     return $current || $default || 'pc';
7025 }
7026
7027 sub get_running_qemu_version {
7028     my ($vmid) = @_;
7029     my $cmd = { execute => 'query-version', arguments => {} };
7030     my $res = vm_qmp_command($vmid, $cmd);
7031     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7032 }
7033
7034 sub qemu_machine_feature_enabled {
7035     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7036
7037     my $current_major;
7038     my $current_minor;
7039
7040     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7041
7042         $current_major = $3;
7043         $current_minor = $4;
7044
7045     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7046
7047         $current_major = $1;
7048         $current_minor = $2;
7049     }
7050
7051     return 1 if $current_major > $version_major ||
7052                 ($current_major == $version_major &&
7053                  $current_minor >= $version_minor);
7054 }
7055
7056 sub qemu_machine_pxe {
7057     my ($vmid, $conf, $machine) = @_;
7058
7059     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7060
7061     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7062         $machine .= '.pxe';
7063     }
7064
7065     return $machine;
7066 }
7067
7068 sub qemu_use_old_bios_files {
7069     my ($machine_type) = @_;
7070
7071     return if !$machine_type;
7072
7073     my $use_old_bios_files = undef;
7074
7075     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7076         $machine_type = $1;
7077         $use_old_bios_files = 1;
7078     } else {
7079         my $kvmver = kvm_user_version();
7080         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7081         # load new efi bios files on migration. So this hack is required to allow
7082         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7083         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7084         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7085     }
7086
7087     return ($use_old_bios_files, $machine_type);
7088 }
7089
7090 sub create_efidisk($$$$$) {
7091     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7092
7093     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7094     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7095
7096     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7097     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7098     PVE::Storage::activate_volumes($storecfg, [$volid]);
7099
7100     my $path = PVE::Storage::path($storecfg, $volid);
7101     eval {
7102         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7103     };
7104     die "Copying EFI vars image failed: $@" if $@;
7105
7106     return ($volid, $vars_size);
7107 }
7108
7109 sub vm_iothreads_list {
7110     my ($vmid) = @_;
7111
7112     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7113
7114     my $iothreads = {};
7115     foreach my $iothread (@$res) {
7116         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7117     }
7118
7119     return $iothreads;
7120 }
7121
7122 sub scsihw_infos {
7123     my ($conf, $drive) = @_;
7124
7125     my $maxdev = 0;
7126
7127     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7128         $maxdev = 7;
7129     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7130         $maxdev = 1;
7131     } else {
7132         $maxdev = 256;
7133     }
7134
7135     my $controller = int($drive->{index} / $maxdev);
7136     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7137
7138     return ($maxdev, $controller, $controller_prefix);
7139 }
7140
7141 sub add_hyperv_enlightenments {
7142     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7143
7144     return if $winversion < 6;
7145     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7146
7147     if ($gpu_passthrough || defined($hv_vendor_id)) {
7148         $hv_vendor_id //= 'proxmox';
7149         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7150     }
7151
7152     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7153         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7154         push @$cpuFlags , 'hv_vapic';
7155         push @$cpuFlags , 'hv_time';
7156     } else {
7157         push @$cpuFlags , 'hv_spinlocks=0xffff';
7158     }
7159
7160     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7161         push @$cpuFlags , 'hv_reset';
7162         push @$cpuFlags , 'hv_vpindex';
7163         push @$cpuFlags , 'hv_runtime';
7164     }
7165
7166     if ($winversion >= 7) {
7167         push @$cpuFlags , 'hv_relaxed';
7168
7169         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7170             push @$cpuFlags , 'hv_synic';
7171             push @$cpuFlags , 'hv_stimer';
7172         }
7173     }
7174 }
7175
7176 sub windows_version {
7177     my ($ostype) = @_;
7178
7179     return 0 if !$ostype;
7180
7181     my $winversion = 0;
7182
7183     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7184         $winversion = 5;
7185     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7186         $winversion = 6;
7187     } elsif ($ostype =~ m/^win(\d+)$/) {
7188         $winversion = $1;
7189     }
7190
7191     return $winversion;
7192 }
7193
7194 sub resolve_dst_disk_format {
7195         my ($storecfg, $storeid, $src_volname, $format) = @_;
7196         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7197
7198         if (!$format) {
7199             # if no target format is specified, use the source disk format as hint
7200             if ($src_volname) {
7201                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7202                 $format = qemu_img_format($scfg, $src_volname);
7203             } else {
7204                 return $defFormat;
7205             }
7206         }
7207
7208         # test if requested format is supported - else use default
7209         my $supported = grep { $_ eq $format } @$validFormats;
7210         $format = $defFormat if !$supported;
7211         return $format;
7212 }
7213
7214 sub resolve_first_disk {
7215     my $conf = shift;
7216     my @disks = PVE::QemuServer::valid_drive_names();
7217     my $firstdisk;
7218     foreach my $ds (reverse @disks) {
7219         next if !$conf->{$ds};
7220         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7221         next if PVE::QemuServer::drive_is_cdrom($disk);
7222         $firstdisk = $ds;
7223     }
7224     return $firstdisk;
7225 }
7226
7227 sub generate_uuid {
7228     my ($uuid, $uuid_str);
7229     UUID::generate($uuid);
7230     UUID::unparse($uuid, $uuid_str);
7231     return $uuid_str;
7232 }
7233
7234 sub generate_smbios1_uuid {
7235     return "uuid=".generate_uuid();
7236 }
7237
7238 sub nbd_stop {
7239     my ($vmid) = @_;
7240
7241     vm_mon_cmd($vmid, 'nbd-server-stop');
7242 }
7243
7244 # bash completion helper
7245
7246 sub complete_backup_archives {
7247     my ($cmdname, $pname, $cvalue) = @_;
7248
7249     my $cfg = PVE::Storage::config();
7250
7251     my $storeid;
7252
7253     if ($cvalue =~ m/^([^:]+):/) {
7254         $storeid = $1;
7255     }
7256
7257     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7258
7259     my $res = [];
7260     foreach my $id (keys %$data) {
7261         foreach my $item (@{$data->{$id}}) {
7262             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7263             push @$res, $item->{volid} if defined($item->{volid});
7264         }
7265     }
7266
7267     return $res;
7268 }
7269
7270 my $complete_vmid_full = sub {
7271     my ($running) = @_;
7272
7273     my $idlist = vmstatus();
7274
7275     my $res = [];
7276
7277     foreach my $id (keys %$idlist) {
7278         my $d = $idlist->{$id};
7279         if (defined($running)) {
7280             next if $d->{template};
7281             next if $running && $d->{status} ne 'running';
7282             next if !$running && $d->{status} eq 'running';
7283         }
7284         push @$res, $id;
7285
7286     }
7287     return $res;
7288 };
7289
7290 sub complete_vmid {
7291     return &$complete_vmid_full();
7292 }
7293
7294 sub complete_vmid_stopped {
7295     return &$complete_vmid_full(0);
7296 }
7297
7298 sub complete_vmid_running {
7299     return &$complete_vmid_full(1);
7300 }
7301
7302 sub complete_storage {
7303
7304     my $cfg = PVE::Storage::config();
7305     my $ids = $cfg->{ids};
7306
7307     my $res = [];
7308     foreach my $sid (keys %$ids) {
7309         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7310         next if !$ids->{$sid}->{content}->{images};
7311         push @$res, $sid;
7312     }
7313
7314     return $res;
7315 }
7316
7317 1;