PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     }
 641 };
 642
 643 my $cicustom_fmt = {
 644     meta => {
 645         type => 'string',
 646         optional => 1,
 647         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 648         format => 'pve-volume-id',
 649         format_description => 'volume',
 650     },
 651     network => {
 652         type => 'string',
 653         optional => 1,
 654         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 655         format => 'pve-volume-id',
 656         format_description => 'volume',
 657     },
 658     user => {
 659         type => 'string',
 660         optional => 1,
 661         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 662         format => 'pve-volume-id',
 663         format_description => 'volume',
 664     },
 665 };
 666 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 667
 668 my $confdesc_cloudinit = {
 669     citype => {
 670         optional => 1,
 671         type => 'string',
 672         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 673         enum => ['configdrive2', 'nocloud'],
 674     },
 675     ciuser => {
 676         optional => 1,
 677         type => 'string',
 678         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 679     },
 680     cipassword => {
 681         optional => 1,
 682         type => 'string',
 683         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 684     },
 685     cicustom => {
 686         optional => 1,
 687         type => 'string',
 688         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 689         format => 'pve-qm-cicustom',
 690     },
 691     searchdomain => {
 692         optional => 1,
 693         type => 'string',
 694         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 695     },
 696     nameserver => {
 697         optional => 1,
 698         type => 'string', format => 'address-list',
 699         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 700     },
 701     sshkeys => {
 702         optional => 1,
 703         type => 'string',
 704         format => 'urlencoded',
 705         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 706     },
 707 };
 708
 709 # what about other qemu settings ?
 710 #cpu => 'string',
 711 #machine => 'string',
 712 #fda => 'file',
 713 #fdb => 'file',
 714 #mtdblock => 'file',
 715 #sd => 'file',
 716 #pflash => 'file',
 717 #snapshot => 'bool',
 718 #bootp => 'file',
 719 ##tftp => 'dir',
 720 ##smb => 'dir',
 721 #kernel => 'file',
 722 #append => 'string',
 723 #initrd => 'file',
 724 ##soundhw => 'string',
 725
 726 while (my ($k, $v) = each %$confdesc) {
 727     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 728 }
 729
 730 my $MAX_IDE_DISKS = 4;
 731 my $MAX_SCSI_DISKS = 14;
 732 my $MAX_VIRTIO_DISKS = 16;
 733 my $MAX_SATA_DISKS = 6;
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_UNUSED_DISKS = 256;
 737 my $MAX_HOSTPCI_DEVICES = 4;
 738 my $MAX_SERIAL_PORTS = 4;
 739 my $MAX_PARALLEL_PORTS = 3;
 740 my $MAX_NUMA = 8;
 741
 742 my $numa_fmt = {
 743     cpus => {
 744         type => "string",
 745         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 746         description => "CPUs accessing this NUMA node.",
 747         format_description => "id[-id];...",
 748     },
 749     memory => {
 750         type => "number",
 751         description => "Amount of memory this NUMA node provides.",
 752         optional => 1,
 753     },
 754     hostnodes => {
 755         type => "string",
 756         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 757         description => "Host NUMA nodes to use.",
 758         format_description => "id[-id];...",
 759         optional => 1,
 760     },
 761     policy => {
 762         type => 'string',
 763         enum => [qw(preferred bind interleave)],
 764         description => "NUMA allocation policy.",
 765         optional => 1,
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 769 my $numadesc = {
 770     optional => 1,
 771     type => 'string', format => $numa_fmt,
 772     description => "NUMA topology.",
 773 };
 774 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 775
 776 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 777     $confdesc->{"numa$i"} = $numadesc;
 778 }
 779
 780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 781                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 782                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 784
 785 my $net_fmt_bridge_descr = <<__EOD__;
 786 Bridge to attach the network device to. The Proxmox VE standard bridge
 787 is called 'vmbr0'.
 788
 789 If you do not specify a bridge, we create a kvm user (NATed) network
 790 device, which provides DHCP and DNS services. The following addresses
 791 are used:
 792
 793  10.0.2.2   Gateway
 794  10.0.2.3   DNS Server
 795  10.0.2.4   SMB Server
 796
 797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 798 __EOD__
 799
 800 my $net_fmt = {
 801     macaddr  => get_standard_option('mac-addr', {
 802         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 803     }),
 804     model => {
 805         type => 'string',
 806         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 807         enum => $nic_model_list,
 808         default_key => 1,
 809     },
 810     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 811     bridge => {
 812         type => 'string',
 813         description => $net_fmt_bridge_descr,
 814         format_description => 'bridge',
 815         optional => 1,
 816     },
 817     queues => {
 818         type => 'integer',
 819         minimum => 0, maximum => 16,
 820         description => 'Number of packet queues to be used on the device.',
 821         optional => 1,
 822     },
 823     rate => {
 824         type => 'number',
 825         minimum => 0,
 826         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 827         optional => 1,
 828     },
 829     tag => {
 830         type => 'integer',
 831         minimum => 1, maximum => 4094,
 832         description => 'VLAN tag to apply to packets on this interface.',
 833         optional => 1,
 834     },
 835     trunks => {
 836         type => 'string',
 837         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 838         description => 'VLAN trunks to pass through this interface.',
 839         format_description => 'vlanid[;vlanid...]',
 840         optional => 1,
 841     },
 842     firewall => {
 843         type => 'boolean',
 844         description => 'Whether this interface should be protected by the firewall.',
 845         optional => 1,
 846     },
 847     link_down => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be disconnected (like pulling the plug).',
 850         optional => 1,
 851     },
 852 };
 853
 854 my $netdesc = {
 855     optional => 1,
 856     type => 'string', format => $net_fmt,
 857     description => "Specify network devices.",
 858 };
 859
 860 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 861
 862 my $ipconfig_fmt = {
 863     ip => {
 864         type => 'string',
 865         format => 'pve-ipv4-config',
 866         format_description => 'IPv4Format/CIDR',
 867         description => 'IPv4 address in CIDR format.',
 868         optional => 1,
 869         default => 'dhcp',
 870     },
 871     gw => {
 872         type => 'string',
 873         format => 'ipv4',
 874         format_description => 'GatewayIPv4',
 875         description => 'Default gateway for IPv4 traffic.',
 876         optional => 1,
 877         requires => 'ip',
 878     },
 879     ip6 => {
 880         type => 'string',
 881         format => 'pve-ipv6-config',
 882         format_description => 'IPv6Format/CIDR',
 883         description => 'IPv6 address in CIDR format.',
 884         optional => 1,
 885         default => 'dhcp',
 886     },
 887     gw6 => {
 888         type => 'string',
 889         format => 'ipv6',
 890         format_description => 'GatewayIPv6',
 891         description => 'Default gateway for IPv6 traffic.',
 892         optional => 1,
 893         requires => 'ip6',
 894     },
 895 };
 896 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 897 my $ipconfigdesc = {
 898     optional => 1,
 899     type => 'string', format => 'pve-qm-ipconfig',
 900     description => <<'EODESCR',
 901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 902
 903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 904
 905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 907
 908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 909 EODESCR
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 912
 913 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 914     $confdesc->{"net$i"} = $netdesc;
 915     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 916 }
 917
 918 foreach my $key (keys %$confdesc_cloudinit) {
 919     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 920 }
 921
 922 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 923 sub verify_volume_id_or_qm_path {
 924     my ($volid, $noerr) = @_;
 925
 926     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 927         return $volid;
 928     }
 929
 930     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 931     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 932     if ($@) {
 933         return undef if $noerr;
 934         die $@;
 935     }
 936     return $volid;
 937 }
 938
 939 my $drivename_hash;
 940
 941 my %drivedesc_base = (
 942     volume => { alias => 'file' },
 943     file => {
 944         type => 'string',
 945         format => 'pve-volume-id-or-qm-path',
 946         default_key => 1,
 947         format_description => 'volume',
 948         description => "The drive's backing volume.",
 949     },
 950     media => {
 951         type => 'string',
 952         enum => [qw(cdrom disk)],
 953         description => "The drive's media type.",
 954         default => 'disk',
 955         optional => 1
 956     },
 957     cyls => {
 958         type => 'integer',
 959         description => "Force the drive's physical geometry to have a specific cylinder count.",
 960         optional => 1
 961     },
 962     heads => {
 963         type => 'integer',
 964         description => "Force the drive's physical geometry to have a specific head count.",
 965         optional => 1
 966     },
 967     secs => {
 968         type => 'integer',
 969         description => "Force the drive's physical geometry to have a specific sector count.",
 970         optional => 1
 971     },
 972     trans => {
 973         type => 'string',
 974         enum => [qw(none lba auto)],
 975         description => "Force disk geometry bios translation mode.",
 976         optional => 1,
 977     },
 978     snapshot => {
 979         type => 'boolean',
 980         description => "Controls qemu's snapshot mode feature."
 981             . " If activated, changes made to the disk are temporary and will"
 982             . " be discarded when the VM is shutdown.",
 983         optional => 1,
 984     },
 985     cache => {
 986         type => 'string',
 987         enum => [qw(none writethrough writeback unsafe directsync)],
 988         description => "The drive's cache mode",
 989         optional => 1,
 990     },
 991     format => get_standard_option('pve-qm-image-format'),
 992     size => {
 993         type => 'string',
 994         format => 'disk-size',
 995         format_description => 'DiskSize',
 996         description => "Disk size. This is purely informational and has no effect.",
 997         optional => 1,
 998     },
 999     backup => {
1000         type => 'boolean',
1001         description => "Whether the drive should be included when making backups.",
1002         optional => 1,
1003     },
1004     replicate => {
1005         type => 'boolean',
1006         description => 'Whether the drive should considered for replication jobs.',
1007         optional => 1,
1008         default => 1,
1009     },
1010     rerror => {
1011         type => 'string',
1012         enum => [qw(ignore report stop)],
1013         description => 'Read error action.',
1014         optional => 1,
1015     },
1016     werror => {
1017         type => 'string',
1018         enum => [qw(enospc ignore report stop)],
1019         description => 'Write error action.',
1020         optional => 1,
1021     },
1022     aio => {
1023         type => 'string',
1024         enum => [qw(native threads)],
1025         description => 'AIO type to use.',
1026         optional => 1,
1027     },
1028     discard => {
1029         type => 'string',
1030         enum => [qw(ignore on)],
1031         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1032         optional => 1,
1033     },
1034     detect_zeroes => {
1035         type => 'boolean',
1036         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1037         optional => 1,
1038     },
1039     serial => {
1040         type => 'string',
1041         format => 'urlencoded',
1042         format_description => 'serial',
1043         maxLength => 20*3, # *3 since it's %xx url enoded
1044         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1045         optional => 1,
1046     },
1047     shared => {
1048         type => 'boolean',
1049         description => 'Mark this locally-managed volume as available on all nodes',
1050         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1051         optional => 1,
1052         default => 0,
1053     }
1054 );
1055
1056 my %iothread_fmt = ( iothread => {
1057         type => 'boolean',
1058         description => "Whether to use iothreads for this drive",
1059         optional => 1,
1060 });
1061
1062 my %model_fmt = (
1063     model => {
1064         type => 'string',
1065         format => 'urlencoded',
1066         format_description => 'model',
1067         maxLength => 40*3, # *3 since it's %xx url enoded
1068         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1069         optional => 1,
1070     },
1071 );
1072
1073 my %queues_fmt = (
1074     queues => {
1075         type => 'integer',
1076         description => "Number of queues.",
1077         minimum => 2,
1078         optional => 1
1079     }
1080 );
1081
1082 my %scsiblock_fmt = (
1083     scsiblock => {
1084         type => 'boolean',
1085         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1086         optional => 1,
1087         default => 0,
1088     },
1089 );
1090
1091 my %ssd_fmt = (
1092     ssd => {
1093         type => 'boolean',
1094         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1095         optional => 1,
1096     },
1097 );
1098
1099 my %wwn_fmt = (
1100     wwn => {
1101         type => 'string',
1102         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1103         format_description => 'wwn',
1104         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1105         optional => 1,
1106     },
1107 );
1108
1109 my $add_throttle_desc = sub {
1110     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1111     my $d = {
1112         type => $type,
1113         format_description => $unit,
1114         description => "Maximum $what in $longunit.",
1115         optional => 1,
1116     };
1117     $d->{minimum} = $minimum if defined($minimum);
1118     $drivedesc_base{$key} = $d;
1119 };
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1122 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1123 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1124 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1130
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1138
1139 # burst lengths
1140 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1146
1147 # legacy support
1148 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1152
1153 my $ide_fmt = {
1154     %drivedesc_base,
1155     %model_fmt,
1156     %ssd_fmt,
1157     %wwn_fmt,
1158 };
1159 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1160
1161 my $idedesc = {
1162     optional => 1,
1163     type => 'string', format => $ide_fmt,
1164     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1165 };
1166 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1167
1168 my $scsi_fmt = {
1169     %drivedesc_base,
1170     %iothread_fmt,
1171     %queues_fmt,
1172     %scsiblock_fmt,
1173     %ssd_fmt,
1174     %wwn_fmt,
1175 };
1176 my $scsidesc = {
1177     optional => 1,
1178     type => 'string', format => $scsi_fmt,
1179     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1182
1183 my $sata_fmt = {
1184     %drivedesc_base,
1185     %ssd_fmt,
1186     %wwn_fmt,
1187 };
1188 my $satadesc = {
1189     optional => 1,
1190     type => 'string', format => $sata_fmt,
1191     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1192 };
1193 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1194
1195 my $virtio_fmt = {
1196     %drivedesc_base,
1197     %iothread_fmt,
1198 };
1199 my $virtiodesc = {
1200     optional => 1,
1201     type => 'string', format => $virtio_fmt,
1202     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1205
1206 my $alldrive_fmt = {
1207     %drivedesc_base,
1208     %iothread_fmt,
1209     %model_fmt,
1210     %queues_fmt,
1211     %scsiblock_fmt,
1212     %ssd_fmt,
1213     %wwn_fmt,
1214 };
1215
1216 my $efidisk_fmt = {
1217     volume => { alias => 'file' },
1218     file => {
1219         type => 'string',
1220         format => 'pve-volume-id-or-qm-path',
1221         default_key => 1,
1222         format_description => 'volume',
1223         description => "The drive's backing volume.",
1224     },
1225     format => get_standard_option('pve-qm-image-format'),
1226     size => {
1227         type => 'string',
1228         format => 'disk-size',
1229         format_description => 'DiskSize',
1230         description => "Disk size. This is purely informational and has no effect.",
1231         optional => 1,
1232     },
1233 };
1234
1235 my $efidisk_desc = {
1236     optional => 1,
1237     type => 'string', format => $efidisk_fmt,
1238     description => "Configure a Disk for storing EFI vars",
1239 };
1240
1241 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1242
1243 my $usb_fmt = {
1244     host => {
1245         default_key => 1,
1246         type => 'string', format => 'pve-qm-usb-device',
1247         format_description => 'HOSTUSBDEVICE|spice',
1248         description => <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1250
1251  'bus-port(.port)*' (decimal numbers) or
1252  'vendor_id:product_id' (hexadeciaml numbers) or
1253  'spice'
1254
1255 You can use the 'lsusb -t' command to list existing usb devices.
1256
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1258
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1260 EODESCR
1261     },
1262     usb3 => {
1263         optional => 1,
1264         type => 'boolean',
1265         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1266         default => 0,
1267     },
1268 };
1269
1270 my $usbdesc = {
1271     optional => 1,
1272     type => 'string', format => $usb_fmt,
1273     description => "Configure an USB device (n is 0 to 4).",
1274 };
1275 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1276
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1278 my $hostpci_fmt = {
1279     host => {
1280         default_key => 1,
1281         type => 'string',
1282         pattern => qr/$PCIRE(;$PCIRE)*/,
1283         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1284         description => <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1287
1288 'bus:dev.func' (hexadecimal numbers)
1289
1290 You can us the 'lspci' command to list existing PCI devices.
1291 EODESCR
1292     },
1293     rombar => {
1294         type => 'boolean',
1295         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1296         optional => 1,
1297         default => 1,
1298     },
1299     romfile => {
1300         type => 'string',
1301         pattern => '[^,;]+',
1302         format_description => 'string',
1303         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1304         optional => 1,
1305     },
1306     pcie => {
1307         type => 'boolean',
1308         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1309         optional => 1,
1310         default => 0,
1311     },
1312     'x-vga' => {
1313         type => 'boolean',
1314         description =>  "Enable vfio-vga device support.",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'mdev' => {
1319         type => 'string',
1320         format_description => 'string',
1321         pattern => '[^/\.:]+',
1322         optional => 1,
1323         description => <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1327 EODESCR
1328     }
1329 };
1330 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1331
1332 my $hostpcidesc = {
1333         optional => 1,
1334         type => 'string', format => 'pve-qm-hostpci',
1335         description => "Map host PCI devices into guest.",
1336         verbose_description =>  <<EODESCR,
1337 Map host PCI devices into guest.
1338
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1346
1347 my $serialdesc = {
1348         optional => 1,
1349         type => 'string',
1350         pattern => '(/dev/.+|socket)',
1351         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1352         verbose_description =>  <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1356
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1358
1359 CAUTION: Experimental! User reported problems with this option.
1360 EODESCR
1361 };
1362
1363 my $paralleldesc= {
1364         optional => 1,
1365         type => 'string',
1366         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1367         description =>  "Map host parallel devices (n is 0 to 2).",
1368         verbose_description =>  <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1370
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1372
1373 CAUTION: Experimental! User reported problems with this option.
1374 EODESCR
1375 };
1376
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1378     $confdesc->{"parallel$i"} = $paralleldesc;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1382     $confdesc->{"serial$i"} = $serialdesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1386     $confdesc->{"hostpci$i"} = $hostpcidesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1390     $drivename_hash->{"ide$i"} = 1;
1391     $confdesc->{"ide$i"} = $idedesc;
1392 }
1393
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1395     $drivename_hash->{"sata$i"} = 1;
1396     $confdesc->{"sata$i"} = $satadesc;
1397 }
1398
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1400     $drivename_hash->{"scsi$i"} = 1;
1401     $confdesc->{"scsi$i"} = $scsidesc ;
1402 }
1403
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1405     $drivename_hash->{"virtio$i"} = 1;
1406     $confdesc->{"virtio$i"} = $virtiodesc;
1407 }
1408
1409 $drivename_hash->{efidisk0} = 1;
1410 $confdesc->{efidisk0} = $efidisk_desc;
1411
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1413     $confdesc->{"usb$i"} = $usbdesc;
1414 }
1415
1416 my $unuseddesc = {
1417     optional => 1,
1418     type => 'string', format => 'pve-volume-id',
1419     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1420 };
1421
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1423     $confdesc->{"unused$i"} = $unuseddesc;
1424 }
1425
1426 my $kvm_api_version = 0;
1427
1428 sub kvm_version {
1429     return $kvm_api_version if $kvm_api_version;
1430
1431     open my $fh, '<', '/dev/kvm'
1432         or return undef;
1433
1434     # 0xae00 => KVM_GET_API_VERSION
1435     $kvm_api_version = ioctl($fh, 0xae00, 0);
1436
1437     return $kvm_api_version;
1438 }
1439
1440 my $kvm_user_version;
1441
1442 sub kvm_user_version {
1443
1444     return $kvm_user_version if $kvm_user_version;
1445
1446     $kvm_user_version = 'unknown';
1447
1448     my $code = sub {
1449         my $line = shift;
1450         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451             $kvm_user_version = $2;
1452         }
1453     };
1454
1455     eval { run_command("kvm -version", outfunc => $code); };
1456     warn $@ if $@;
1457
1458     return $kvm_user_version;
1459
1460 }
1461
1462 sub kernel_has_vhost_net {
1463     return -c '/dev/vhost-net';
1464 }
1465
1466 sub valid_drive_names {
1467     # order is important - used to autoselect boot disk
1468     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1472             'efidisk0');
1473 }
1474
1475 sub is_valid_drivename {
1476     my $dev = shift;
1477
1478     return defined($drivename_hash->{$dev});
1479 }
1480
1481 sub option_exists {
1482     my $key = shift;
1483     return defined($confdesc->{$key});
1484 }
1485
1486 sub nic_models {
1487     return $nic_model_list;
1488 }
1489
1490 sub os_list_description {
1491
1492     return {
1493         other => 'Other',
1494         wxp => 'Windows XP',
1495         w2k => 'Windows 2000',
1496         w2k3 =>, 'Windows 2003',
1497         w2k8 => 'Windows 2008',
1498         wvista => 'Windows Vista',
1499         win7 => 'Windows 7',
1500         win8 => 'Windows 8/2012',
1501         win10 => 'Windows 10/2016',
1502         l24 => 'Linux 2.4',
1503         l26 => 'Linux 2.6',
1504     };
1505 }
1506
1507 my $cdrom_path;
1508
1509 sub get_cdrom_path {
1510
1511     return  $cdrom_path if $cdrom_path;
1512
1513     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1514     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1515     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1516 }
1517
1518 sub get_iso_path {
1519     my ($storecfg, $vmid, $cdrom) = @_;
1520
1521     if ($cdrom eq 'cdrom') {
1522         return get_cdrom_path();
1523     } elsif ($cdrom eq 'none') {
1524         return '';
1525     } elsif ($cdrom =~ m|^/|) {
1526         return $cdrom;
1527     } else {
1528         return PVE::Storage::path($storecfg, $cdrom);
1529     }
1530 }
1531
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id {
1534     my ($vmid, $file, $media) = @_;
1535
1536      if (!($file eq 'none' || $file eq 'cdrom' ||
1537           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1538
1539         return undef if $file =~ m|/|;
1540
1541         if ($media && $media eq 'cdrom') {
1542             $file = "local:iso/$file";
1543         } else {
1544             $file = "local:$vmid/$file";
1545         }
1546     }
1547
1548     return $file;
1549 }
1550
1551 sub verify_media_type {
1552     my ($opt, $vtype, $media) = @_;
1553
1554     return if !$media;
1555
1556     my $etype;
1557     if ($media eq 'disk') {
1558         $etype = 'images';
1559     } elsif ($media eq 'cdrom') {
1560         $etype = 'iso';
1561     } else {
1562         die "internal error";
1563     }
1564
1565     return if ($vtype eq $etype);
1566
1567     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1568 }
1569
1570 sub cleanup_drive_path {
1571     my ($opt, $storecfg, $drive) = @_;
1572
1573     # try to convert filesystem paths to volume IDs
1574
1575     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1576         ($drive->{file} !~ m|^/dev/.+|) &&
1577         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1578         ($drive->{file} !~ m/^\d+$/)) {
1579         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1580         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1582         verify_media_type($opt, $vtype, $drive->{media});
1583         $drive->{file} = $volid;
1584     }
1585
1586     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1587 }
1588
1589 sub parse_hotplug_features {
1590     my ($data) = @_;
1591
1592     my $res = {};
1593
1594     return $res if $data eq '0';
1595
1596     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1597
1598     foreach my $feature (PVE::Tools::split_list($data)) {
1599         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1600             $res->{$1} = 1;
1601         } else {
1602             die "invalid hotplug feature '$feature'\n";
1603         }
1604     }
1605     return $res;
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1609 sub pve_verify_hotplug_features {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_hotplug_features($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse hotplug option\n";
1617 }
1618
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 #        [,iothread=on][,serial=serial][,model=model]
1624
1625 sub parse_drive {
1626     my ($key, $data) = @_;
1627
1628     my ($interface, $index);
1629
1630     if ($key =~ m/^([^\d]+)(\d+)$/) {
1631         $interface = $1;
1632         $index = $2;
1633     } else {
1634         return undef;
1635     }
1636
1637     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1638                                      : $confdesc->{$key}->{format};
1639     if (!$desc) {
1640         warn "invalid drive key: $key\n";
1641         return undef;
1642     }
1643     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1644     return undef if !$res;
1645     $res->{interface} = $interface;
1646     $res->{index} = $index;
1647
1648     my $error = 0;
1649     foreach my $opt (qw(bps bps_rd bps_wr)) {
1650         if (my $bps = defined(delete $res->{$opt})) {
1651             if (defined($res->{"m$opt"})) {
1652                 warn "both $opt and m$opt specified\n";
1653                 ++$error;
1654                 next;
1655             }
1656             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1657         }
1658     }
1659
1660     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661     for my $requirement (
1662         [mbps_max => 'mbps'],
1663         [mbps_rd_max => 'mbps_rd'],
1664         [mbps_wr_max => 'mbps_wr'],
1665         [miops_max => 'miops'],
1666         [miops_rd_max => 'miops_rd'],
1667         [miops_wr_max => 'miops_wr'],
1668         [bps_max_length => 'mbps_max'],
1669         [bps_rd_max_length => 'mbps_rd_max'],
1670         [bps_wr_max_length => 'mbps_wr_max'],
1671         [iops_max_length => 'iops_max'],
1672         [iops_rd_max_length => 'iops_rd_max'],
1673         [iops_wr_max_length => 'iops_wr_max']) {
1674         my ($option, $requires) = @$requirement;
1675         if ($res->{$option} && !$res->{$requires}) {
1676             warn "$option requires $requires\n";
1677             ++$error;
1678         }
1679     }
1680
1681     return undef if $error;
1682
1683     return undef if $res->{mbps_rd} && $res->{mbps};
1684     return undef if $res->{mbps_wr} && $res->{mbps};
1685     return undef if $res->{iops_rd} && $res->{iops};
1686     return undef if $res->{iops_wr} && $res->{iops};
1687
1688     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1689         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1690         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1691         return undef if $res->{interface} eq 'virtio';
1692     }
1693
1694     if (my $size = $res->{size}) {
1695         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1696     }
1697
1698     return $res;
1699 }
1700
1701 sub print_drive {
1702     my ($vmid, $drive) = @_;
1703     my $data = { %$drive };
1704     delete $data->{$_} for qw(index interface);
1705     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1706 }
1707
1708 sub scsi_inquiry {
1709     my($fh, $noerr) = @_;
1710
1711     my $SG_IO = 0x2285;
1712     my $SG_GET_VERSION_NUM = 0x2282;
1713
1714     my $versionbuf = "\x00" x 8;
1715     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1716     if (!$ret) {
1717         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1718         return undef;
1719     }
1720     my $version = unpack("I", $versionbuf);
1721     if ($version < 30000) {
1722         die "scsi generic interface too old\n"  if !$noerr;
1723         return undef;
1724     }
1725
1726     my $buf = "\x00" x 36;
1727     my $sensebuf = "\x00" x 8;
1728     my $cmd = pack("C x3 C x1", 0x12, 36);
1729
1730     # see /usr/include/scsi/sg.h
1731     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1732
1733     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734                       length($sensebuf), 0, length($buf), $buf,
1735                       $cmd, $sensebuf, 6000);
1736
1737     $ret = ioctl($fh, $SG_IO, $packet);
1738     if (!$ret) {
1739         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1740         return undef;
1741     }
1742
1743     my @res = unpack($sg_io_hdr_t, $packet);
1744     if ($res[17] || $res[18]) {
1745         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my $res = {};
1750     (my $byte0, my $byte1, $res->{vendor},
1751      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1752
1753     $res->{removable} = $byte1 & 128 ? 1 : 0;
1754     $res->{type} = $byte0 & 31;
1755
1756     return $res;
1757 }
1758
1759 sub path_is_scsi {
1760     my ($path) = @_;
1761
1762     my $fh = IO::File->new("+<$path") || return undef;
1763     my $res = scsi_inquiry($fh, 1);
1764     close($fh);
1765
1766     return $res;
1767 }
1768
1769 sub machine_type_is_q35 {
1770     my ($conf) = @_;
1771
1772     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1773 }
1774
1775 sub print_tabletdevice_full {
1776     my ($conf, $arch) = @_;
1777
1778     my $q35 = machine_type_is_q35($conf);
1779
1780     # we use uhci for old VMs because tablet driver was buggy in older qemu
1781     my $usbbus;
1782     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1783         $usbbus = 'ehci';
1784     } else {
1785         $usbbus = 'uhci';
1786     }
1787
1788     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1789 }
1790
1791 sub print_keyboarddevice_full {
1792     my ($conf, $arch, $machine) = @_;
1793
1794     return undef if $arch ne 'aarch64';
1795
1796     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1797 }
1798
1799 sub print_drivedevice_full {
1800     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1801
1802     my $device = '';
1803     my $maxdev = 0;
1804
1805     if ($drive->{interface} eq 'virtio') {
1806         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1809     } elsif ($drive->{interface} eq 'scsi') {
1810
1811         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1812         my $unit = $drive->{index} % $maxdev;
1813         my $devicetype = 'hd';
1814         my $path = '';
1815         if (drive_is_cdrom($drive)) {
1816             $devicetype = 'cd';
1817         } else {
1818             if ($drive->{file} =~ m|^/|) {
1819                 $path = $drive->{file};
1820                 if (my $info = path_is_scsi($path)) {
1821                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1822                         $devicetype = 'block';
1823                     } elsif ($info->{type} == 1) { # tape
1824                         $devicetype = 'generic';
1825                     }
1826                 }
1827             } else {
1828                  $path = PVE::Storage::path($storecfg, $drive->{file});
1829             }
1830
1831             if($path =~ m/^iscsi\:\/\//){
1832                 $devicetype = 'generic';
1833             }
1834         }
1835
1836         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1837             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1838         } else {
1839             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1840         }
1841
1842         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843             $device .= ",rotation_rate=1";
1844         }
1845         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1846
1847     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1848         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1849         my $controller = int($drive->{index} / $maxdev);
1850         my $unit = $drive->{index} % $maxdev;
1851         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1852
1853         $device = "ide-$devicetype";
1854         if ($drive->{interface} eq 'ide') {
1855             $device .= ",bus=ide.$controller,unit=$unit";
1856         } else {
1857             $device .= ",bus=ahci$controller.$unit";
1858         }
1859         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1860
1861         if ($devicetype eq 'hd') {
1862             if (my $model = $drive->{model}) {
1863                 $model = URI::Escape::uri_unescape($model);
1864                 $device .= ",model=$model";
1865             }
1866             if ($drive->{ssd}) {
1867                 $device .= ",rotation_rate=1";
1868             }
1869         }
1870         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1871     } elsif ($drive->{interface} eq 'usb') {
1872         die "implement me";
1873         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1874     } else {
1875         die "unsupported interface type";
1876     }
1877
1878     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1879
1880     if (my $serial = $drive->{serial}) {
1881         $serial = URI::Escape::uri_unescape($serial);
1882         $device .= ",serial=$serial";
1883     }
1884
1885
1886     return $device;
1887 }
1888
1889 sub get_initiator_name {
1890     my $initiator;
1891
1892     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893     while (defined(my $line = <$fh>)) {
1894         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1895         $initiator = $1;
1896         last;
1897     }
1898     $fh->close();
1899
1900     return $initiator;
1901 }
1902
1903 sub print_drive_full {
1904     my ($storecfg, $vmid, $drive) = @_;
1905
1906     my $path;
1907     my $volid = $drive->{file};
1908     my $format;
1909
1910     if (drive_is_cdrom($drive)) {
1911         $path = get_iso_path($storecfg, $vmid, $volid);
1912     } else {
1913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1914         if ($storeid) {
1915             $path = PVE::Storage::path($storecfg, $volid);
1916             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1917             $format = qemu_img_format($scfg, $volname);
1918         } else {
1919             $path = $volid;
1920             $format = "raw";
1921         }
1922    }
1923
1924     my $opts = '';
1925     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926     foreach my $o (@qemu_drive_options) {
1927         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1928     }
1929
1930     # snapshot only accepts on|off
1931     if (defined($drive->{snapshot})) {
1932         my $v = $drive->{snapshot} ? 'on' : 'off';
1933         $opts .= ",snapshot=$v";
1934     }
1935
1936     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1937         my ($dir, $qmpname) = @$type;
1938         if (my $v = $drive->{"mbps$dir"}) {
1939             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1940         }
1941         if (my $v = $drive->{"mbps${dir}_max"}) {
1942             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1943         }
1944         if (my $v = $drive->{"bps${dir}_max_length"}) {
1945             $opts .= ",throttling.bps$qmpname-max-length=$v";
1946         }
1947         if (my $v = $drive->{"iops${dir}"}) {
1948             $opts .= ",throttling.iops$qmpname=$v";
1949         }
1950         if (my $v = $drive->{"iops${dir}_max"}) {
1951             $opts .= ",throttling.iops$qmpname-max=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}_max_length"}) {
1954             $opts .= ",throttling.iops$qmpname-max-length=$v";
1955         }
1956     }
1957
1958     $opts .= ",format=$format" if $format && !$drive->{format};
1959
1960     my $cache_direct = 0;
1961
1962     if (my $cache = $drive->{cache}) {
1963         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964     } elsif (!drive_is_cdrom($drive)) {
1965         $opts .= ",cache=none";
1966         $cache_direct = 1;
1967     }
1968
1969     # aio native works only with O_DIRECT
1970     if (!$drive->{aio}) {
1971         if($cache_direct) {
1972             $opts .= ",aio=native";
1973         } else {
1974             $opts .= ",aio=threads";
1975         }
1976     }
1977
1978     if (!drive_is_cdrom($drive)) {
1979         my $detectzeroes;
1980         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1981             $detectzeroes = 'off';
1982         } elsif ($drive->{discard}) {
1983             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1984         } else {
1985             # This used to be our default with discard not being specified:
1986             $detectzeroes = 'on';
1987         }
1988         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1989     }
1990
1991     my $pathinfo = $path ? "file=$path," : '';
1992
1993     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1994 }
1995
1996 sub print_netdevice_full {
1997     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1998
1999     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2000
2001     my $device = $net->{model};
2002     if ($net->{model} eq 'virtio') {
2003          $device = 'virtio-net-pci';
2004      };
2005
2006     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2007     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2009         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010         my $vectors = $net->{queues} * 2 + 2;
2011         $tmpstr .= ",vectors=$vectors,mq=on";
2012     }
2013     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2014
2015     if ($use_old_bios_files) {
2016         my $romfile;
2017         if ($device eq 'virtio-net-pci') {
2018             $romfile = 'pxe-virtio.rom';
2019         } elsif ($device eq 'e1000') {
2020             $romfile = 'pxe-e1000.rom';
2021         } elsif ($device eq 'ne2k') {
2022             $romfile = 'pxe-ne2k_pci.rom';
2023         } elsif ($device eq 'pcnet') {
2024             $romfile = 'pxe-pcnet.rom';
2025         } elsif ($device eq 'rtl8139') {
2026             $romfile = 'pxe-rtl8139.rom';
2027         }
2028         $tmpstr .= ",romfile=$romfile" if $romfile;
2029     }
2030
2031     return $tmpstr;
2032 }
2033
2034 sub print_netdev_full {
2035     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2036
2037     my $i = '';
2038     if ($netid =~ m/^net(\d+)$/) {
2039         $i = int($1);
2040     }
2041
2042     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2043
2044     my $ifname = "tap${vmid}i$i";
2045
2046     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047     die "interface name '$ifname' is too long (max 15 character)\n"
2048         if length($ifname) >= 16;
2049
2050     my $vhostparam = '';
2051     if (is_native($arch)) {
2052         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2053     }
2054
2055     my $vmname = $conf->{name} || "vm$vmid";
2056
2057     my $netdev = "";
2058     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2059
2060     if ($net->{bridge}) {
2061         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2062     } else {
2063         $netdev = "type=user,id=$netid,hostname=$vmname";
2064     }
2065
2066     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2067
2068     return $netdev;
2069 }
2070
2071
2072 sub print_cpu_device {
2073     my ($conf, $id) = @_;
2074
2075     my $kvm = $conf->{kvm} // 1;
2076     my $cpu = $kvm ? "kvm64" : "qemu64";
2077     if (my $cputype = $conf->{cpu}) {
2078         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2079             or die "Cannot parse cpu description: $cputype\n";
2080         $cpu = $cpuconf->{cputype};
2081     }
2082
2083     my $cores = $conf->{cores} || 1;
2084
2085     my $current_core = ($id - 1) % $cores;
2086     my $current_socket = int(($id - 1 - $current_core)/$cores);
2087
2088     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2089 }
2090
2091 my $vga_map = {
2092     'cirrus' => 'cirrus-vga',
2093     'std' => 'VGA',
2094     'vmware' => 'vmware-svga',
2095     'virtio' => 'virtio-vga',
2096 };
2097
2098 sub print_vga_device {
2099     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2100
2101     my $type = $vga_map->{$vga->{type}};
2102     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103         $type = 'virtio-gpu';
2104     }
2105     my $vgamem_mb = $vga->{memory};
2106     if ($qxlnum) {
2107         $type = $id ? 'qxl' : 'qxl-vga';
2108     }
2109     die "no devicetype for $vga->{type}\n" if !$type;
2110
2111     my $memory = "";
2112     if ($vgamem_mb) {
2113         if ($vga->{type} eq 'virtio') {
2114             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2115             $memory = ",max_hostmem=$bytes";
2116         } elsif ($qxlnum) {
2117             # from https://www.spice-space.org/multiple-monitors.html
2118             $memory = ",vgamem_mb=$vga->{memory}";
2119             my $ram = $vgamem_mb * 4;
2120             my $vram = $vgamem_mb * 2;
2121             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2122         } else {
2123             $memory = ",vgamem_mb=$vga->{memory}";
2124         }
2125     } elsif ($qxlnum && $id) {
2126         $memory = ",ram_size=67108864,vram_size=33554432";
2127     }
2128
2129     my $q35 = machine_type_is_q35($conf);
2130     my $vgaid = "vga" . ($id // '');
2131     my $pciaddr;
2132
2133     if ($q35 && $vgaid eq 'vga') {
2134         # the first display uses pcie.0 bus on q35 machines
2135         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2136     } else {
2137         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2138     }
2139
2140     return "$type,id=${vgaid}${memory}${pciaddr}";
2141 }
2142
2143 sub drive_is_cloudinit {
2144     my ($drive) = @_;
2145     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2146 }
2147
2148 sub drive_is_cdrom {
2149     my ($drive, $exclude_cloudinit) = @_;
2150
2151     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2152
2153     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2154
2155 }
2156
2157 sub parse_number_sets {
2158     my ($set) = @_;
2159     my $res = [];
2160     foreach my $part (split(/;/, $set)) {
2161         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163             push @$res, [ $1, $2 ];
2164         } else {
2165             die "invalid range: $part\n";
2166         }
2167     }
2168     return $res;
2169 }
2170
2171 sub parse_numa {
2172     my ($data) = @_;
2173
2174     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2175     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2176     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2177     return $res;
2178 }
2179
2180 sub parse_hostpci {
2181     my ($value) = @_;
2182
2183     return undef if !$value;
2184
2185     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2186
2187     my @idlist = split(/;/, $res->{host});
2188     delete $res->{host};
2189     foreach my $id (@idlist) {
2190         if ($id =~ m/\./) { # full id 00:00.1
2191             push @{$res->{pciid}}, {
2192                 id => $id,
2193             };
2194         } else { # partial id 00:00
2195             $res->{pciid} = PVE::SysFSTools::lspci($id);
2196         }
2197     }
2198     return $res;
2199 }
2200
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2202 sub parse_net {
2203     my ($data) = @_;
2204
2205     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2206     if ($@) {
2207         warn $@;
2208         return undef;
2209     }
2210     if (!defined($res->{macaddr})) {
2211         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2212         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2213     }
2214     return $res;
2215 }
2216
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig {
2219     my ($data) = @_;
2220
2221     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2222     if ($@) {
2223         warn $@;
2224         return undef;
2225     }
2226
2227     if ($res->{gw} && !$res->{ip}) {
2228         warn 'gateway specified without specifying an IP address';
2229         return undef;
2230     }
2231     if ($res->{gw6} && !$res->{ip6}) {
2232         warn 'IPv6 gateway specified without specifying an IPv6 address';
2233         return undef;
2234     }
2235     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2236         warn 'gateway specified together with DHCP';
2237         return undef;
2238     }
2239     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2240         # gw6 + auto/dhcp
2241         warn "IPv6 gateway specified together with $res->{ip6} address";
2242         return undef;
2243     }
2244
2245     if (!$res->{ip} && !$res->{ip6}) {
2246         return { ip => 'dhcp', ip6 => 'dhcp' };
2247     }
2248
2249     return $res;
2250 }
2251
2252 sub print_net {
2253     my $net = shift;
2254
2255     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2256 }
2257
2258 sub add_random_macs {
2259     my ($settings) = @_;
2260
2261     foreach my $opt (keys %$settings) {
2262         next if $opt !~ m/^net(\d+)$/;
2263         my $net = parse_net($settings->{$opt});
2264         next if !$net;
2265         $settings->{$opt} = print_net($net);
2266     }
2267 }
2268
2269 sub vm_is_volid_owner {
2270     my ($storecfg, $vmid, $volid) = @_;
2271
2272     if ($volid !~  m|^/|) {
2273         my ($path, $owner);
2274         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2275         if ($owner && ($owner == $vmid)) {
2276             return 1;
2277         }
2278     }
2279
2280     return undef;
2281 }
2282
2283 sub split_flagged_list {
2284     my $text = shift || '';
2285     $text =~ s/[,;]/ /g;
2286     $text =~ s/^\s+//;
2287     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2288 }
2289
2290 sub join_flagged_list {
2291     my ($how, $lst) = @_;
2292     join $how, map { $lst->{$_} . $_ } keys %$lst;
2293 }
2294
2295 sub vmconfig_delete_pending_option {
2296     my ($conf, $key, $force) = @_;
2297
2298     delete $conf->{pending}->{$key};
2299     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2300     $pending_delete_hash->{$key} = $force ? '!' : '';
2301     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2302 }
2303
2304 sub vmconfig_undelete_pending_option {
2305     my ($conf, $key) = @_;
2306
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     delete $pending_delete_hash->{$key};
2309
2310     if (%$pending_delete_hash) {
2311         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2312     } else {
2313         delete $conf->{pending}->{delete};
2314     }
2315 }
2316
2317 sub vmconfig_register_unused_drive {
2318     my ($storecfg, $vmid, $conf, $drive) = @_;
2319
2320     if (drive_is_cloudinit($drive)) {
2321         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2322         warn $@ if $@;
2323     } elsif (!drive_is_cdrom($drive)) {
2324         my $volid = $drive->{file};
2325         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2326             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2327         }
2328     }
2329 }
2330
2331 sub vmconfig_cleanup_pending {
2332     my ($conf) = @_;
2333
2334     # remove pending changes when nothing changed
2335     my $changes;
2336     foreach my $opt (keys %{$conf->{pending}}) {
2337         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2338             $changes = 1;
2339             delete $conf->{pending}->{$opt};
2340         }
2341     }
2342
2343     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2344     my $pending_delete_hash = {};
2345     while (my ($opt, $force) = each %$current_delete_hash) {
2346         if (defined($conf->{$opt})) {
2347             $pending_delete_hash->{$opt} = $force;
2348         } else {
2349             $changes = 1;
2350         }
2351     }
2352
2353     if (%$pending_delete_hash) {
2354         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2355     } else {
2356         delete $conf->{pending}->{delete};
2357     }
2358
2359     return $changes;
2360 }
2361
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2363 my $smbios1_fmt = {
2364     uuid => {
2365         type => 'string',
2366         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367         format_description => 'UUID',
2368         description => "Set SMBIOS1 UUID.",
2369         optional => 1,
2370     },
2371     version => {
2372         type => 'string',
2373         pattern => '[A-Za-z0-9+\/]+={0,2}',
2374         format_description => 'Base64 encoded string',
2375         description => "Set SMBIOS1 version.",
2376         optional => 1,
2377     },
2378     serial => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 serial number.",
2383         optional => 1,
2384     },
2385     manufacturer => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 manufacturer.",
2390         optional => 1,
2391     },
2392     product => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 product ID.",
2397         optional => 1,
2398     },
2399     sku => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 SKU string.",
2404         optional => 1,
2405     },
2406     family => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 family string.",
2411         optional => 1,
2412     },
2413     base64 => {
2414         type => 'boolean',
2415         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2416         optional => 1,
2417     },
2418 };
2419
2420 sub parse_smbios1 {
2421     my ($data) = @_;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub print_smbios1 {
2429     my ($smbios1) = @_;
2430     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2431 }
2432
2433 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2434
2435 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2436 sub verify_bootdisk {
2437     my ($value, $noerr) = @_;
2438
2439     return $value if is_valid_drivename($value);
2440
2441     return undef if $noerr;
2442
2443     die "invalid boot disk '$value'\n";
2444 }
2445
2446 sub parse_watchdog {
2447     my ($value) = @_;
2448
2449     return undef if !$value;
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2452     warn $@ if $@;
2453     return $res;
2454 }
2455
2456 sub parse_guest_agent {
2457     my ($value) = @_;
2458
2459     return {} if !defined($value->{agent});
2460
2461     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2462     warn $@ if $@;
2463
2464     # if the agent is disabled ignore the other potentially set properties
2465     return {} if !$res->{enabled};
2466     return $res;
2467 }
2468
2469 sub parse_vga {
2470     my ($value) = @_;
2471
2472     return {} if !$value;
2473     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2474     warn $@ if $@;
2475     return $res;
2476 }
2477
2478 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2479 sub verify_usb_device {
2480     my ($value, $noerr) = @_;
2481
2482     return $value if parse_usb_device($value);
2483
2484     return undef if $noerr;
2485
2486     die "unable to parse usb device\n";
2487 }
2488
2489 # add JSON properties for create and set function
2490 sub json_config_properties {
2491     my $prop = shift;
2492
2493     foreach my $opt (keys %$confdesc) {
2494         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495         $prop->{$opt} = $confdesc->{$opt};
2496     }
2497
2498     return $prop;
2499 }
2500
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties {
2503
2504     return dclone($confdesc_cloudinit);
2505 }
2506
2507 sub check_type {
2508     my ($key, $value) = @_;
2509
2510     die "unknown setting '$key'\n" if !$confdesc->{$key};
2511
2512     my $type = $confdesc->{$key}->{type};
2513
2514     if (!defined($value)) {
2515         die "got undefined value\n";
2516     }
2517
2518     if ($value =~ m/[\n\r]/) {
2519         die "property contains a line feed\n";
2520     }
2521
2522     if ($type eq 'boolean') {
2523         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525         die "type check ('boolean') failed - got '$value'\n";
2526     } elsif ($type eq 'integer') {
2527         return int($1) if $value =~ m/^(\d+)$/;
2528         die "type check ('integer') failed - got '$value'\n";
2529     } elsif ($type eq 'number') {
2530         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531         die "type check ('number') failed - got '$value'\n";
2532     } elsif ($type eq 'string') {
2533         if (my $fmt = $confdesc->{$key}->{format}) {
2534             PVE::JSONSchema::check_format($fmt, $value);
2535             return $value;
2536         }
2537         $value =~ s/^\"(.*)\"$/$1/;
2538         return $value;
2539     } else {
2540         die "internal error"
2541     }
2542 }
2543
2544 sub touch_config {
2545     my ($vmid) = @_;
2546
2547     my $conf = PVE::QemuConfig->config_file($vmid);
2548     utime undef, undef, $conf;
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conffile = PVE::QemuConfig->config_file($vmid);
2555
2556     my $conf = PVE::QemuConfig->load_config($vmid);
2557
2558     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2559
2560     if ($conf->{template}) {
2561         # check if any base image is still used by a linked clone
2562         foreach_drive($conf, sub {
2563                 my ($ds, $drive) = @_;
2564
2565                 return if drive_is_cdrom($drive);
2566
2567                 my $volid = $drive->{file};
2568
2569                 return if !$volid || $volid =~ m|^/|;
2570
2571                 die "base volume '$volid' is still in use by linked cloned\n"
2572                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2573
2574         });
2575     }
2576
2577     # only remove disks owned by this VM
2578     foreach_drive($conf, sub {
2579         my ($ds, $drive) = @_;
2580
2581         return if drive_is_cdrom($drive, 1);
2582
2583         my $volid = $drive->{file};
2584
2585         return if !$volid || $volid =~ m|^/|;
2586
2587         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2588         return if !$path || !$owner || ($owner != $vmid);
2589
2590         eval {
2591             PVE::Storage::vdisk_free($storecfg, $volid);
2592         };
2593         warn "Could not remove disk '$volid', check manually: $@" if $@;
2594
2595     });
2596
2597     if ($keep_empty_config) {
2598         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2599     } else {
2600         unlink $conffile;
2601     }
2602
2603     # also remove unused disk
2604     eval {
2605         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2606
2607         eval {
2608             PVE::Storage::foreach_volid($dl, sub {
2609                 my ($volid, $sid, $volname, $d) = @_;
2610                 PVE::Storage::vdisk_free($storecfg, $volid);
2611             });
2612         };
2613         warn $@ if $@;
2614
2615     };
2616     warn $@ if $@;
2617 }
2618
2619 sub parse_vm_config {
2620     my ($filename, $raw) = @_;
2621
2622     return undef if !defined($raw);
2623
2624     my $res = {
2625         digest => Digest::SHA::sha1_hex($raw),
2626         snapshots => {},
2627         pending => {},
2628     };
2629
2630     $filename =~ m|/qemu-server/(\d+)\.conf$|
2631         || die "got strange filename '$filename'";
2632
2633     my $vmid = $1;
2634
2635     my $conf = $res;
2636     my $descr;
2637     my $section = '';
2638
2639     my @lines = split(/\n/, $raw);
2640     foreach my $line (@lines) {
2641         next if $line =~ m/^\s*$/;
2642
2643         if ($line =~ m/^\[PENDING\]\s*$/i) {
2644             $section = 'pending';
2645             if (defined($descr)) {
2646                 $descr =~ s/\s+$//;
2647                 $conf->{description} = $descr;
2648             }
2649             $descr = undef;
2650             $conf = $res->{$section} = {};
2651             next;
2652
2653         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2654             $section = $1;
2655             if (defined($descr)) {
2656                 $descr =~ s/\s+$//;
2657                 $conf->{description} = $descr;
2658             }
2659             $descr = undef;
2660             $conf = $res->{snapshots}->{$section} = {};
2661             next;
2662         }
2663
2664         if ($line =~ m/^\#(.*)\s*$/) {
2665             $descr = '' if !defined($descr);
2666             $descr .= PVE::Tools::decode_text($1) . "\n";
2667             next;
2668         }
2669
2670         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($2);
2673         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674             $conf->{snapstate} = $1;
2675         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2676             my $key = $1;
2677             my $value = $2;
2678             $conf->{$key} = $value;
2679         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2680             my $value = $1;
2681             if ($section eq 'pending') {
2682                 $conf->{delete} = $value; # we parse this later
2683             } else {
2684                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2685             }
2686         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2687             my $key = $1;
2688             my $value = $2;
2689             eval { $value = check_type($key, $value); };
2690             if ($@) {
2691                 warn "vm $vmid - unable to parse value of '$key' - $@";
2692             } else {
2693                 $key = 'ide2' if $key eq 'cdrom';
2694                 my $fmt = $confdesc->{$key}->{format};
2695                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696                     my $v = parse_drive($key, $value);
2697                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2698                         $v->{file} = $volid;
2699                         $value = print_drive($vmid, $v);
2700                     } else {
2701                         warn "vm $vmid - unable to parse value of '$key'\n";
2702                         next;
2703                     }
2704                 }
2705
2706                 $conf->{$key} = $value;
2707             }
2708         }
2709     }
2710
2711     if (defined($descr)) {
2712         $descr =~ s/\s+$//;
2713         $conf->{description} = $descr;
2714     }
2715     delete $res->{snapstate}; # just to be sure
2716
2717     return $res;
2718 }
2719
2720 sub write_vm_config {
2721     my ($filename, $conf) = @_;
2722
2723     delete $conf->{snapstate}; # just to be sure
2724
2725     if ($conf->{cdrom}) {
2726         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2727         $conf->{ide2} = $conf->{cdrom};
2728         delete $conf->{cdrom};
2729     }
2730
2731     # we do not use 'smp' any longer
2732     if ($conf->{sockets}) {
2733         delete $conf->{smp};
2734     } elsif ($conf->{smp}) {
2735         $conf->{sockets} = $conf->{smp};
2736         delete $conf->{cores};
2737         delete $conf->{smp};
2738     }
2739
2740     my $used_volids = {};
2741
2742     my $cleanup_config = sub {
2743         my ($cref, $pending, $snapname) = @_;
2744
2745         foreach my $key (keys %$cref) {
2746             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747                 $key eq 'snapstate' || $key eq 'pending';
2748             my $value = $cref->{$key};
2749             if ($key eq 'delete') {
2750                 die "propertry 'delete' is only allowed in [PENDING]\n"
2751                     if !$pending;
2752                 # fixme: check syntax?
2753                 next;
2754             }
2755             eval { $value = check_type($key, $value); };
2756             die "unable to parse value of '$key' - $@" if $@;
2757
2758             $cref->{$key} = $value;
2759
2760             if (!$snapname && is_valid_drivename($key)) {
2761                 my $drive = parse_drive($key, $value);
2762                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2763             }
2764         }
2765     };
2766
2767     &$cleanup_config($conf);
2768
2769     &$cleanup_config($conf->{pending}, 1);
2770
2771     foreach my $snapname (keys %{$conf->{snapshots}}) {
2772         die "internal error" if $snapname eq 'pending';
2773         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2774     }
2775
2776     # remove 'unusedX' settings if we re-add a volume
2777     foreach my $key (keys %$conf) {
2778         my $value = $conf->{$key};
2779         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780             delete $conf->{$key};
2781         }
2782     }
2783
2784     my $generate_raw_config = sub {
2785         my ($conf, $pending) = @_;
2786
2787         my $raw = '';
2788
2789         # add description as comment to top of file
2790         if (defined(my $descr = $conf->{description})) {
2791             if ($descr) {
2792                 foreach my $cl (split(/\n/, $descr)) {
2793                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2794                 }
2795             } else {
2796                 $raw .= "#\n" if $pending;
2797             }
2798         }
2799
2800         foreach my $key (sort keys %$conf) {
2801             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802             $raw .= "$key: $conf->{$key}\n";
2803         }
2804         return $raw;
2805     };
2806
2807     my $raw = &$generate_raw_config($conf);
2808
2809     if (scalar(keys %{$conf->{pending}})){
2810         $raw .= "\n[PENDING]\n";
2811         $raw .= &$generate_raw_config($conf->{pending}, 1);
2812     }
2813
2814     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2815         $raw .= "\n[$snapname]\n";
2816         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2817     }
2818
2819     return $raw;
2820 }
2821
2822 sub load_defaults {
2823
2824     my $res = {};
2825
2826     # we use static defaults from our JSON schema configuration
2827     foreach my $key (keys %$confdesc) {
2828         if (defined(my $default = $confdesc->{$key}->{default})) {
2829             $res->{$key} = $default;
2830         }
2831     }
2832
2833     return $res;
2834 }
2835
2836 sub config_list {
2837     my $vmlist = PVE::Cluster::get_vmlist();
2838     my $res = {};
2839     return $res if !$vmlist || !$vmlist->{ids};
2840     my $ids = $vmlist->{ids};
2841
2842     foreach my $vmid (keys %$ids) {
2843         my $d = $ids->{$vmid};
2844         next if !$d->{node} || $d->{node} ne $nodename;
2845         next if !$d->{type} || $d->{type} ne 'qemu';
2846         $res->{$vmid}->{exists} = 1;
2847     }
2848     return $res;
2849 }
2850
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources {
2853     my ($conf, $noerr) = @_;
2854
2855     my @loc_res = ();
2856
2857     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2858     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2859
2860     push @loc_res, "ivshmem" if $conf->{ivshmem};
2861
2862     foreach my $k (keys %$conf) {
2863         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864         # sockets are safe: they will recreated be on the target side post-migrate
2865         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2867     }
2868
2869     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2870
2871     return \@loc_res;
2872 }
2873
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability {
2876     my ($storecfg, $conf, $node) = @_;
2877
2878     foreach_drive($conf, sub {
2879         my ($ds, $drive) = @_;
2880
2881         my $volid = $drive->{file};
2882         return if !$volid;
2883
2884         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2885         return if !$sid;
2886
2887         # check if storage is available on both nodes
2888         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2889         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2890    });
2891 }
2892
2893 # list nodes where all VM images are available (used by has_feature API)
2894 sub shared_nodes {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => 1 } @$nodelist };
2899     my $nodename = PVE::INotify::nodename();
2900
2901     foreach_drive($conf, sub {
2902         my ($ds, $drive) = @_;
2903
2904         my $volid = $drive->{file};
2905         return if !$volid;
2906
2907         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2908         if ($storeid) {
2909             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2910             if ($scfg->{disable}) {
2911                 $nodehash = {};
2912             } elsif (my $avail = $scfg->{nodes}) {
2913                 foreach my $node (keys %$nodehash) {
2914                     delete $nodehash->{$node} if !$avail->{$node};
2915                 }
2916             } elsif (!$scfg->{shared}) {
2917                 foreach my $node (keys %$nodehash) {
2918                     delete $nodehash->{$node} if $node ne $nodename
2919                 }
2920             }
2921         }
2922     });
2923
2924     return $nodehash
2925 }
2926
2927 sub check_cmdline {
2928     my ($pidfile, $pid) = @_;
2929
2930     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2931     if (defined($fh)) {
2932         my $line = <$fh>;
2933         $fh->close;
2934         return undef if !$line;
2935         my @param = split(/\0/, $line);
2936
2937         my $cmd = $param[0];
2938         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2939
2940         for (my $i = 0; $i < scalar (@param); $i++) {
2941             my $p = $param[$i];
2942             next if !$p;
2943             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2944                 my $p = $param[$i+1];
2945                 return 1 if $p && ($p eq $pidfile);
2946                 return undef;
2947             }
2948         }
2949     }
2950     return undef;
2951 }
2952
2953 sub check_running {
2954     my ($vmid, $nocheck, $node) = @_;
2955
2956     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2957
2958     die "unable to find configuration file for VM $vmid - no such machine\n"
2959         if !$nocheck && ! -f $filename;
2960
2961     my $pidfile = pidfile_name($vmid);
2962
2963     if (my $fd = IO::File->new("<$pidfile")) {
2964         my $st = stat($fd);
2965         my $line = <$fd>;
2966         close($fd);
2967
2968         my $mtime = $st->mtime;
2969         if ($mtime > time()) {
2970             warn "file '$filename' modified in future\n";
2971         }
2972
2973         if ($line =~ m/^(\d+)$/) {
2974             my $pid = $1;
2975             if (check_cmdline($pidfile, $pid)) {
2976                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2977                     return $pid;
2978                 }
2979             }
2980         }
2981     }
2982
2983     return undef;
2984 }
2985
2986 sub vzlist {
2987
2988     my $vzlist = config_list();
2989
2990     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2991
2992     while (defined(my $de = $fd->read)) {
2993         next if $de !~ m/^(\d+)\.pid$/;
2994         my $vmid = $1;
2995         next if !defined($vzlist->{$vmid});
2996         if (my $pid = check_running($vmid)) {
2997             $vzlist->{$vmid}->{pid} = $pid;
2998         }
2999     }
3000
3001     return $vzlist;
3002 }
3003
3004 sub disksize {
3005     my ($storecfg, $conf) = @_;
3006
3007     my $bootdisk = $conf->{bootdisk};
3008     return undef if !$bootdisk;
3009     return undef if !is_valid_drivename($bootdisk);
3010
3011     return undef if !$conf->{$bootdisk};
3012
3013     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3014     return undef if !defined($drive);
3015
3016     return undef if drive_is_cdrom($drive);
3017
3018     my $volid = $drive->{file};
3019     return undef if !$volid;
3020
3021     return $drive->{size};
3022 }
3023
3024 our $vmstatus_return_properties = {
3025     vmid => get_standard_option('pve-vmid'),
3026     status => {
3027         description => "Qemu process status.",
3028         type => 'string',
3029         enum => ['stopped', 'running'],
3030     },
3031     maxmem => {
3032         description => "Maximum memory in bytes.",
3033         type => 'integer',
3034         optional => 1,
3035         renderer => 'bytes',
3036     },
3037     maxdisk => {
3038         description => "Root disk size in bytes.",
3039         type => 'integer',
3040         optional => 1,
3041         renderer => 'bytes',
3042     },
3043     name => {
3044         description => "VM name.",
3045         type => 'string',
3046         optional => 1,
3047     },
3048     qmpstatus => {
3049         description => "Qemu QMP agent status.",
3050         type => 'string',
3051         optional => 1,
3052     },
3053     pid => {
3054         description => "PID of running qemu process.",
3055         type => 'integer',
3056         optional => 1,
3057     },
3058     uptime => {
3059         description => "Uptime.",
3060         type => 'integer',
3061         optional => 1,
3062         renderer => 'duration',
3063     },
3064     cpus => {
3065         description => "Maximum usable CPUs.",
3066         type => 'number',
3067         optional => 1,
3068     },
3069     lock => {
3070         description => "The current config lock, if any.",
3071         type => 'string',
3072         optional => 1,
3073     }
3074 };
3075
3076 my $last_proc_pid_stat;
3077
3078 # get VM status information
3079 # This must be fast and should not block ($full == false)
3080 # We only query KVM using QMP if $full == true (this can be slow)
3081 sub vmstatus {
3082     my ($opt_vmid, $full) = @_;
3083
3084     my $res = {};
3085
3086     my $storecfg = PVE::Storage::config();
3087
3088     my $list = vzlist();
3089     my $defaults = load_defaults();
3090
3091     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3092
3093     my $cpucount = $cpuinfo->{cpus} || 1;
3094
3095     foreach my $vmid (keys %$list) {
3096         next if $opt_vmid && ($vmid ne $opt_vmid);
3097
3098         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3099         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3100
3101         my $d = { vmid => $vmid };
3102         $d->{pid} = $list->{$vmid}->{pid};
3103
3104         # fixme: better status?
3105         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3106
3107         my $size = disksize($storecfg, $conf);
3108         if (defined($size)) {
3109             $d->{disk} = 0; # no info available
3110             $d->{maxdisk} = $size;
3111         } else {
3112             $d->{disk} = 0;
3113             $d->{maxdisk} = 0;
3114         }
3115
3116         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3117             * ($conf->{cores} || $defaults->{cores});
3118         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3119         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3120
3121         $d->{name} = $conf->{name} || "VM $vmid";
3122         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3123             : $defaults->{memory}*(1024*1024);
3124
3125         if ($conf->{balloon}) {
3126             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3127             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3128                 : $defaults->{shares};
3129         }
3130
3131         $d->{uptime} = 0;
3132         $d->{cpu} = 0;
3133         $d->{mem} = 0;
3134
3135         $d->{netout} = 0;
3136         $d->{netin} = 0;
3137
3138         $d->{diskread} = 0;
3139         $d->{diskwrite} = 0;
3140
3141         $d->{template} = PVE::QemuConfig->is_template($conf);
3142
3143         $d->{serial} = 1 if conf_has_serial($conf);
3144         $d->{lock} = $conf->{lock} if $conf->{lock};
3145
3146         $res->{$vmid} = $d;
3147     }
3148
3149     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3150     foreach my $dev (keys %$netdev) {
3151         next if $dev !~ m/^tap([1-9]\d*)i/;
3152         my $vmid = $1;
3153         my $d = $res->{$vmid};
3154         next if !$d;
3155
3156         $d->{netout} += $netdev->{$dev}->{receive};
3157         $d->{netin} += $netdev->{$dev}->{transmit};
3158
3159         if ($full) {
3160             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3161             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3162         }
3163
3164     }
3165
3166     my $ctime = gettimeofday;
3167
3168     foreach my $vmid (keys %$list) {
3169
3170         my $d = $res->{$vmid};
3171         my $pid = $d->{pid};
3172         next if !$pid;
3173
3174         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3175         next if !$pstat; # not running
3176
3177         my $used = $pstat->{utime} + $pstat->{stime};
3178
3179         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3180
3181         if ($pstat->{vsize}) {
3182             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3183         }
3184
3185         my $old = $last_proc_pid_stat->{$pid};
3186         if (!$old) {
3187             $last_proc_pid_stat->{$pid} = {
3188                 time => $ctime,
3189                 used => $used,
3190                 cpu => 0,
3191             };
3192             next;
3193         }
3194
3195         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3196
3197         if ($dtime > 1000) {
3198             my $dutime = $used -  $old->{used};
3199
3200             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3201             $last_proc_pid_stat->{$pid} = {
3202                 time => $ctime,
3203                 used => $used,
3204                 cpu => $d->{cpu},
3205             };
3206         } else {
3207             $d->{cpu} = $old->{cpu};
3208         }
3209     }
3210
3211     return $res if !$full;
3212
3213     my $qmpclient = PVE::QMPClient->new();
3214
3215     my $ballooncb = sub {
3216         my ($vmid, $resp) = @_;
3217
3218         my $info = $resp->{'return'};
3219         return if !$info->{max_mem};
3220
3221         my $d = $res->{$vmid};
3222
3223         # use memory assigned to VM
3224         $d->{maxmem} = $info->{max_mem};
3225         $d->{balloon} = $info->{actual};
3226
3227         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3228             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3229             $d->{freemem} = $info->{free_mem};
3230         }
3231
3232         $d->{ballooninfo} = $info;
3233     };
3234
3235     my $blockstatscb = sub {
3236         my ($vmid, $resp) = @_;
3237         my $data = $resp->{'return'} || [];
3238         my $totalrdbytes = 0;
3239         my $totalwrbytes = 0;
3240
3241         for my $blockstat (@$data) {
3242             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3243             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3244
3245             $blockstat->{device} =~ s/drive-//;
3246             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3247         }
3248         $res->{$vmid}->{diskread} = $totalrdbytes;
3249         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3250     };
3251
3252     my $statuscb = sub {
3253         my ($vmid, $resp) = @_;
3254
3255         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3256         # this fails if ballon driver is not loaded, so this must be
3257         # the last commnand (following command are aborted if this fails).
3258         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3259
3260         my $status = 'unknown';
3261         if (!defined($status = $resp->{'return'}->{status})) {
3262             warn "unable to get VM status\n";
3263             return;
3264         }
3265
3266         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3267     };
3268
3269     foreach my $vmid (keys %$list) {
3270         next if $opt_vmid && ($vmid ne $opt_vmid);
3271         next if !$res->{$vmid}->{pid}; # not running
3272         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3273     }
3274
3275     $qmpclient->queue_execute(undef, 2);
3276
3277     foreach my $vmid (keys %$list) {
3278         next if $opt_vmid && ($vmid ne $opt_vmid);
3279         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3280     }
3281
3282     return $res;
3283 }
3284
3285 sub foreach_drive {
3286     my ($conf, $func, @param) = @_;
3287
3288     foreach my $ds (valid_drive_names()) {
3289         next if !defined($conf->{$ds});
3290
3291         my $drive = parse_drive($ds, $conf->{$ds});
3292         next if !$drive;
3293
3294         &$func($ds, $drive, @param);
3295     }
3296 }
3297
3298 sub foreach_volid {
3299     my ($conf, $func, @param) = @_;
3300
3301     my $volhash = {};
3302
3303     my $test_volid = sub {
3304         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3305
3306         return if !$volid;
3307
3308         $volhash->{$volid}->{cdrom} //= 1;
3309         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3310
3311         $volhash->{$volid}->{replicate} //= 0;
3312         $volhash->{$volid}->{replicate} = 1 if $replicate;
3313
3314         $volhash->{$volid}->{shared} //= 0;
3315         $volhash->{$volid}->{shared} = 1 if $shared;
3316
3317         $volhash->{$volid}->{referenced_in_config} //= 0;
3318         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3319
3320         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3321             if defined($snapname);
3322     };
3323
3324     foreach_drive($conf, sub {
3325         my ($ds, $drive) = @_;
3326         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3327     });
3328
3329     foreach my $snapname (keys %{$conf->{snapshots}}) {
3330         my $snap = $conf->{snapshots}->{$snapname};
3331         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3332         foreach_drive($snap, sub {
3333             my ($ds, $drive) = @_;
3334             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3335         });
3336     }
3337
3338     foreach my $volid (keys %$volhash) {
3339         &$func($volid, $volhash->{$volid}, @param);
3340     }
3341 }
3342
3343 sub conf_has_serial {
3344     my ($conf) = @_;
3345
3346     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3347         if ($conf->{"serial$i"}) {
3348             return 1;
3349         }
3350     }
3351
3352     return 0;
3353 }
3354
3355 sub vga_conf_has_spice {
3356     my ($vga) = @_;
3357
3358     my $vgaconf = parse_vga($vga);
3359     my $vgatype = $vgaconf->{type};
3360     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3361
3362     return $1 || 1;
3363 }
3364
3365 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3366 sub get_host_arch() {
3367     $host_arch = (POSIX::uname())[4] if !$host_arch;
3368     return $host_arch;
3369 }
3370
3371 sub is_native($) {
3372     my ($arch) = @_;
3373     return get_host_arch() eq $arch;
3374 }
3375
3376 my $default_machines = {
3377     x86_64 => 'pc',
3378     aarch64 => 'virt',
3379 };
3380
3381 sub get_basic_machine_info {
3382     my ($conf, $forcemachine) = @_;
3383
3384     my $arch = $conf->{arch} // get_host_arch();
3385     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3386     return ($arch, $machine);
3387 }
3388
3389 sub get_ovmf_files($) {
3390     my ($arch) = @_;
3391
3392     my $ovmf = $OVMF->{$arch}
3393         or die "no OVMF images known for architecture '$arch'\n";
3394
3395     return @$ovmf;
3396 }
3397
3398 my $Arch2Qemu = {
3399     aarch64 => '/usr/bin/qemu-system-aarch64',
3400     x86_64 => '/usr/bin/qemu-system-x86_64',
3401 };
3402 sub get_command_for_arch($) {
3403     my ($arch) = @_;
3404     return '/usr/bin/kvm' if is_native($arch);
3405
3406     my $cmd = $Arch2Qemu->{$arch}
3407         or die "don't know how to emulate architecture '$arch'\n";
3408     return $cmd;
3409 }
3410
3411 sub get_cpu_options {
3412     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3413
3414     my $cpuFlags = [];
3415     my $ostype = $conf->{ostype};
3416
3417     my $cpu = $kvm ? "kvm64" : "qemu64";
3418     if ($arch eq 'aarch64') {
3419         $cpu = 'cortex-a57';
3420     }
3421     my $hv_vendor_id;
3422     if (my $cputype = $conf->{cpu}) {
3423         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3424             or die "Cannot parse cpu description: $cputype\n";
3425         $cpu = $cpuconf->{cputype};
3426         $kvm_off = 1 if $cpuconf->{hidden};
3427         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3428
3429         if (defined(my $flags = $cpuconf->{flags})) {
3430             push @$cpuFlags, split(";", $flags);
3431         }
3432     }
3433
3434     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3435
3436     push @$cpuFlags , '-x2apic'
3437         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3438
3439     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3440
3441     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3442
3443     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3444
3445         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3446         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3447     }
3448
3449     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3450
3451     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3452
3453     push @$cpuFlags, 'kvm=off' if $kvm_off;
3454
3455     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3456         push @$cpuFlags, "vendor=${cpu_vendor}"
3457             if $cpu_vendor ne 'default';
3458     } elsif ($arch ne 'aarch64') {
3459         die "internal error"; # should not happen
3460     }
3461
3462     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3463
3464     return ('-cpu', $cpu);
3465 }
3466
3467 sub config_to_command {
3468     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3469
3470     my $cmd = [];
3471     my $globalFlags = [];
3472     my $machineFlags = [];
3473     my $rtcFlags = [];
3474     my $devices = [];
3475     my $pciaddr = '';
3476     my $bridges = {};
3477     my $kvmver = kvm_user_version();
3478     my $vernum = 0; # unknown
3479     my $ostype = $conf->{ostype};
3480     my $winversion = windows_version($ostype);
3481     my $kvm = $conf->{kvm};
3482
3483     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3484     $kvm //= 1 if is_native($arch);
3485
3486     if ($kvm) {
3487         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3488             if !defined kvm_version();
3489     }
3490
3491     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3492         $vernum = $1*1000000+$2*1000;
3493     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3494         $vernum = $1*1000000+$2*1000+$3;
3495     }
3496
3497     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3498
3499     my $have_ovz = -f '/proc/vz/vestat';
3500
3501     my $q35 = machine_type_is_q35($conf);
3502     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3503     my $use_old_bios_files = undef;
3504     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3505
3506     my $cpuunits = defined($conf->{cpuunits}) ?
3507             $conf->{cpuunits} : $defaults->{cpuunits};
3508
3509     push @$cmd, get_command_for_arch($arch);
3510
3511     push @$cmd, '-id', $vmid;
3512
3513     my $vmname = $conf->{name} || "vm$vmid";
3514
3515     push @$cmd, '-name', $vmname;
3516
3517     my $use_virtio = 0;
3518
3519     my $qmpsocket = qmp_socket($vmid);
3520     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3521     push @$cmd, '-mon', "chardev=qmp,mode=control";
3522
3523     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3524         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3525         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3526     }
3527
3528     push @$cmd, '-pidfile' , pidfile_name($vmid);
3529
3530     push @$cmd, '-daemonize';
3531
3532     if ($conf->{smbios1}) {
3533         my $smbios_conf = parse_smbios1($conf->{smbios1});
3534         if ($smbios_conf->{base64}) {
3535             # Do not pass base64 flag to qemu
3536             delete $smbios_conf->{base64};
3537             my $smbios_string = "";
3538             foreach my $key (keys %$smbios_conf) {
3539                 my $value;
3540                 if ($key eq "uuid") {
3541                     $value = $smbios_conf->{uuid}
3542                 } else {
3543                     $value = decode_base64($smbios_conf->{$key});
3544                 }
3545                 # qemu accepts any binary data, only commas need escaping by double comma
3546                 $value =~ s/,/,,/g;
3547                 $smbios_string .= "," . $key . "=" . $value if $value;
3548             }
3549             push @$cmd, '-smbios', "type=1" . $smbios_string;
3550         } else {
3551             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3552         }
3553     }
3554
3555     if ($conf->{vmgenid}) {
3556         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3557     }
3558
3559     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3560     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3561         die "uefi base image not found\n" if ! -f $ovmf_code;
3562
3563         my $path;
3564         my $format;
3565         if (my $efidisk = $conf->{efidisk0}) {
3566             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3567             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3568             $format = $d->{format};
3569             if ($storeid) {
3570                 $path = PVE::Storage::path($storecfg, $d->{file});
3571                 if (!defined($format)) {
3572                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3573                     $format = qemu_img_format($scfg, $volname);
3574                 }
3575             } else {
3576                 $path = $d->{file};
3577                 die "efidisk format must be specified\n"
3578                     if !defined($format);
3579             }
3580         } else {
3581             warn "no efidisk configured! Using temporary efivars disk.\n";
3582             $path = "/tmp/$vmid-ovmf.fd";
3583             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3584             $format = 'raw';
3585         }
3586
3587         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3588         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3589     }
3590
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3594     push @$devices, @usbcontrollers if @usbcontrollers;
3595     my $vga = parse_vga($conf->{vga});
3596
3597     my $qxlnum = vga_conf_has_spice($conf->{vga});
3598     $vga->{type} = 'qxl' if $qxlnum;
3599
3600     if (!$vga->{type}) {
3601         if ($arch eq 'aarch64') {
3602             $vga->{type} = 'virtio';
3603         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3604             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3605         } else {
3606             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3607         }
3608     }
3609
3610     # enable absolute mouse coordinates (needed by vnc)
3611     my $tablet;
3612     if (defined($conf->{tablet})) {
3613         $tablet = $conf->{tablet};
3614     } else {
3615         $tablet = $defaults->{tablet};
3616         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3618     }
3619
3620     if ($tablet) {
3621         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3622         my $kbd = print_keyboarddevice_full($conf, $arch);
3623         push @$devices, '-device', $kbd if defined($kbd);
3624     }
3625
3626     my $kvm_off = 0;
3627     my $gpu_passthrough;
3628
3629     # host pci devices
3630     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3631         my $d = parse_hostpci($conf->{"hostpci$i"});
3632         next if !$d;
3633
3634         my $pcie = $d->{pcie};
3635         if ($pcie) {
3636             die "q35 machine model is not enabled" if !$q35;
3637             # win7 wants to have the pcie devices directly on the pcie bus
3638             # instead of in the root port
3639             if ($winversion == 7) {
3640                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3641             } else {
3642                 $pciaddr = print_pcie_addr("hostpci$i");
3643             }
3644         } else {
3645             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3646         }
3647
3648         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3649         my $romfile = $d->{romfile};
3650
3651         my $xvga = '';
3652         if ($d->{'x-vga'}) {
3653             $xvga = ',x-vga=on';
3654             $kvm_off = 1;
3655             $vga->{type} = 'none' if !defined($conf->{vga});
3656             $gpu_passthrough = 1;
3657
3658             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3659                 $xvga = "";
3660             }
3661         }
3662         my $pcidevices = $d->{pciid};
3663         my $multifunction = 1 if @$pcidevices > 1;
3664         my $sysfspath;
3665         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3666             my $id = $pcidevices->[0]->{id};
3667             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3668             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3669         } elsif ($d->{mdev}) {
3670             warn "ignoring mediated device with multifunction device\n";
3671         }
3672
3673         my $j=0;
3674         foreach my $pcidevice (@$pcidevices) {
3675
3676             my $id = "hostpci$i";
3677             $id .= ".$j" if $multifunction;
3678             my $addr = $pciaddr;
3679             $addr .= ".$j" if $multifunction;
3680             my $devicestr = "vfio-pci";
3681             if ($sysfspath) {
3682                 $devicestr .= ",sysfsdev=$sysfspath";
3683             } else {
3684                 $devicestr .= ",host=$pcidevice->{id}";
3685             }
3686             $devicestr .= ",id=$id$addr";
3687
3688             if($j == 0){
3689                 $devicestr .= "$rombar$xvga";
3690                 $devicestr .= ",multifunction=on" if $multifunction;
3691                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3692             }
3693
3694             push @$devices, '-device', $devicestr;
3695             $j++;
3696         }
3697     }
3698
3699     # usb devices
3700     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3701     push @$devices, @usbdevices if @usbdevices;
3702     # serial devices
3703     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3704         if (my $path = $conf->{"serial$i"}) {
3705             if ($path eq 'socket') {
3706                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3708                 # On aarch64, serial0 is the UART device. Qemu only allows
3709                 # connecting UART devices via the '-serial' command line, as
3710                 # the device has a fixed slot on the hardware...
3711                 if ($arch eq 'aarch64' && $i == 0) {
3712                     push @$devices, '-serial', "chardev:serial$i";
3713                 } else {
3714                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3715                 }
3716             } else {
3717                 die "no such serial device\n" if ! -c $path;
3718                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3719                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3720             }
3721         }
3722     }
3723
3724     # parallel devices
3725     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3726         if (my $path = $conf->{"parallel$i"}) {
3727             die "no such parallel device\n" if ! -c $path;
3728             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3729             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3730             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3731         }
3732     }
3733
3734
3735     my $sockets = 1;
3736     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3737     $sockets = $conf->{sockets} if  $conf->{sockets};
3738
3739     my $cores = $conf->{cores} || 1;
3740
3741     my $maxcpus = $sockets * $cores;
3742
3743     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3744
3745     my $allowed_vcpus = $cpuinfo->{cpus};
3746
3747     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3748         if ($allowed_vcpus < $maxcpus);
3749
3750     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3751
3752         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753         for (my $i = 2; $i <= $vcpus; $i++)  {
3754             my $cpustr = print_cpu_device($conf,$i);
3755             push @$cmd, '-device', $cpustr;
3756         }
3757
3758     } else {
3759
3760         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3761     }
3762     push @$cmd, '-nodefaults';
3763
3764     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3765
3766     my $bootindex_hash = {};
3767     my $i = 1;
3768     foreach my $o (split(//, $bootorder)) {
3769         $bootindex_hash->{$o} = $i*100;
3770         $i++;
3771     }
3772
3773     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3774
3775     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3776
3777     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3778
3779     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3780         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3781         my $socket = vnc_socket($vmid);
3782         push @$cmd,  '-vnc', "unix:$socket,password";
3783     } else {
3784         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3785         push @$cmd, '-nographic';
3786     }
3787
3788     # time drift fix
3789     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3790
3791     my $useLocaltime = $conf->{localtime};
3792
3793     if ($winversion >= 5) { # windows
3794         $useLocaltime = 1 if !defined($conf->{localtime});
3795
3796         # use time drift fix when acpi is enabled
3797         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3798             $tdf = 1 if !defined($conf->{tdf});
3799         }
3800     }
3801
3802     if ($winversion >= 6) {
3803         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3804         push @$cmd, '-no-hpet';
3805     }
3806
3807     push @$rtcFlags, 'driftfix=slew' if $tdf;
3808
3809     if (!$kvm) {
3810         push @$machineFlags, 'accel=tcg';
3811     }
3812
3813     if ($machine_type) {
3814         push @$machineFlags, "type=${machine_type}";
3815     }
3816
3817     if ($conf->{startdate}) {
3818         push @$rtcFlags, "base=$conf->{startdate}";
3819     } elsif ($useLocaltime) {
3820         push @$rtcFlags, 'base=localtime';
3821     }
3822
3823     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3824
3825     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3826
3827     push @$cmd, '-S' if $conf->{freeze};
3828
3829     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3830
3831     # enable sound
3832     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3833     #push @$cmd, '-soundhw', 'es1370';
3834     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3835
3836     if (parse_guest_agent($conf)->{enabled}) {
3837         my $qgasocket = qmp_socket($vmid, 1);
3838         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3839         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3840         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3841         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3842     }
3843
3844     my $spice_port;
3845
3846     if ($qxlnum) {
3847         if ($qxlnum > 1) {
3848             if ($winversion){
3849                 for(my $i = 1; $i < $qxlnum; $i++){
3850                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3851                 }
3852             } else {
3853                 # assume other OS works like Linux
3854                 my ($ram, $vram) = ("134217728", "67108864");
3855                 if ($vga->{memory}) {
3856                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3857                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3858                 }
3859                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3860                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3861             }
3862         }
3863
3864         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3865
3866         my $nodename = PVE::INotify::nodename();
3867         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3868         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3869         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3870         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3871         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3872
3873         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3874
3875         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3876         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3877         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3878     }
3879
3880     # enable balloon by default, unless explicitly disabled
3881     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3882         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3883         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3884     }
3885
3886     if ($conf->{watchdog}) {
3887         my $wdopts = parse_watchdog($conf->{watchdog});
3888         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3889         my $watchdog = $wdopts->{model} || 'i6300esb';
3890         push @$devices, '-device', "$watchdog$pciaddr";
3891         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3892     }
3893
3894     my $vollist = [];
3895     my $scsicontroller = {};
3896     my $ahcicontroller = {};
3897     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3898
3899     # Add iscsi initiator name if available
3900     if (my $initiator = get_initiator_name()) {
3901         push @$devices, '-iscsi', "initiator-name=$initiator";
3902     }
3903
3904     foreach_drive($conf, sub {
3905         my ($ds, $drive) = @_;
3906
3907         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3908             push @$vollist, $drive->{file};
3909         }
3910
3911         # ignore efidisk here, already added in bios/fw handling code above
3912         return if $drive->{interface} eq 'efidisk';
3913
3914         $use_virtio = 1 if $ds =~ m/^virtio/;
3915
3916         if (drive_is_cdrom ($drive)) {
3917             if ($bootindex_hash->{d}) {
3918                 $drive->{bootindex} = $bootindex_hash->{d};
3919                 $bootindex_hash->{d} += 1;
3920             }
3921         } else {
3922             if ($bootindex_hash->{c}) {
3923                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3924                 $bootindex_hash->{c} += 1;
3925             }
3926         }
3927
3928         if($drive->{interface} eq 'virtio'){
3929            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3930         }
3931
3932         if ($drive->{interface} eq 'scsi') {
3933
3934             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3935
3936             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3937             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3938
3939             my $iothread = '';
3940             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3941                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3942                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3943             } elsif ($drive->{iothread}) {
3944                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3945             }
3946
3947             my $queues = '';
3948             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3949                 $queues = ",num_queues=$drive->{queues}";
3950             }
3951
3952             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3953             $scsicontroller->{$controller}=1;
3954         }
3955
3956         if ($drive->{interface} eq 'sata') {
3957            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3958            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3959            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3960            $ahcicontroller->{$controller}=1;
3961         }
3962
3963         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3964         push @$devices, '-drive',$drive_cmd;
3965         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3966     });
3967
3968     for (my $i = 0; $i < $MAX_NETS; $i++) {
3969          next if !$conf->{"net$i"};
3970          my $d = parse_net($conf->{"net$i"});
3971          next if !$d;
3972
3973          $use_virtio = 1 if $d->{model} eq 'virtio';
3974
3975          if ($bootindex_hash->{n}) {
3976             $d->{bootindex} = $bootindex_hash->{n};
3977             $bootindex_hash->{n} += 1;
3978          }
3979
3980          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3981          push @$devices, '-netdev', $netdevfull;
3982
3983          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3984          push @$devices, '-device', $netdevicefull;
3985     }
3986
3987     if ($conf->{ivshmem}) {
3988         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3989
3990         my $bus;
3991         if ($q35) {
3992             $bus = print_pcie_addr("ivshmem");
3993         } else {
3994             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3995         }
3996
3997         my $ivshmem_name = $ivshmem->{name} // $vmid;
3998         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3999
4000         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4001         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4002     }
4003
4004     if (!$q35) {
4005         # add pci bridges
4006         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4007            $bridges->{1} = 1;
4008            $bridges->{2} = 1;
4009         }
4010
4011         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4012
4013         while (my ($k, $v) = each %$bridges) {
4014             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4015             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4016         }
4017     }
4018
4019     push @$cmd, @$devices;
4020     push @$cmd, '-rtc', join(',', @$rtcFlags)
4021         if scalar(@$rtcFlags);
4022     push @$cmd, '-machine', join(',', @$machineFlags)
4023         if scalar(@$machineFlags);
4024     push @$cmd, '-global', join(',', @$globalFlags)
4025         if scalar(@$globalFlags);
4026
4027     if (my $vmstate = $conf->{vmstate}) {
4028         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4029         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4030         push @$cmd, '-loadstate', $statepath;
4031     }
4032
4033     # add custom args
4034     if ($conf->{args}) {
4035         my $aa = PVE::Tools::split_args($conf->{args});
4036         push @$cmd, @$aa;
4037     }
4038
4039     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4040 }
4041
4042 sub vnc_socket {
4043     my ($vmid) = @_;
4044     return "${var_run_tmpdir}/$vmid.vnc";
4045 }
4046
4047 sub spice_port {
4048     my ($vmid) = @_;
4049
4050     my $res = vm_mon_cmd($vmid, 'query-spice');
4051
4052     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4053 }
4054
4055 sub qmp_socket {
4056     my ($vmid, $qga, $name) = @_;
4057     my $sockettype = $qga ? 'qga' : 'qmp';
4058     my $ext = $name ? '-'.$name : '';
4059     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4060 }
4061
4062 sub pidfile_name {
4063     my ($vmid) = @_;
4064     return "${var_run_tmpdir}/$vmid.pid";
4065 }
4066
4067 sub vm_devices_list {
4068     my ($vmid) = @_;
4069
4070     my $res = vm_mon_cmd($vmid, 'query-pci');
4071     my $devices_to_check = [];
4072     my $devices = {};
4073     foreach my $pcibus (@$res) {
4074         push @$devices_to_check, @{$pcibus->{devices}},
4075     }
4076
4077     while (@$devices_to_check) {
4078         my $to_check = [];
4079         for my $d (@$devices_to_check) {
4080             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4081             next if !$d->{'pci_bridge'};
4082
4083             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4084             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4085         }
4086         $devices_to_check = $to_check;
4087     }
4088
4089     my $resblock = vm_mon_cmd($vmid, 'query-block');
4090     foreach my $block (@$resblock) {
4091         if($block->{device} =~ m/^drive-(\S+)/){
4092                 $devices->{$1} = 1;
4093         }
4094     }
4095
4096     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4097     foreach my $mice (@$resmice) {
4098         if ($mice->{name} eq 'QEMU HID Tablet') {
4099             $devices->{tablet} = 1;
4100             last;
4101         }
4102     }
4103
4104     # for usb devices there is no query-usb
4105     # but we can iterate over the entries in
4106     # qom-list path=/machine/peripheral
4107     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4108     foreach my $per (@$resperipheral) {
4109         if ($per->{name} =~ m/^usb\d+$/) {
4110             $devices->{$per->{name}} = 1;
4111         }
4112     }
4113
4114     return $devices;
4115 }
4116
4117 sub vm_deviceplug {
4118     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4119
4120     my $q35 = machine_type_is_q35($conf);
4121
4122     my $devices_list = vm_devices_list($vmid);
4123     return 1 if defined($devices_list->{$deviceid});
4124
4125     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4126
4127     if ($deviceid eq 'tablet') {
4128
4129         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4130
4131     } elsif ($deviceid eq 'keyboard') {
4132
4133         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4134
4135     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4136
4137         die "usb hotplug currently not reliable\n";
4138         # since we can't reliably hot unplug all added usb devices
4139         # and usb passthrough disables live migration
4140         # we disable usb hotplugging for now
4141         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4142
4143     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4144
4145         qemu_iothread_add($vmid, $deviceid, $device);
4146
4147         qemu_driveadd($storecfg, $vmid, $device);
4148         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4149
4150         qemu_deviceadd($vmid, $devicefull);
4151         eval { qemu_deviceaddverify($vmid, $deviceid); };
4152         if (my $err = $@) {
4153             eval { qemu_drivedel($vmid, $deviceid); };
4154             warn $@ if $@;
4155             die $err;
4156         }
4157
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159
4160
4161         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4162         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4163         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4164
4165         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4166
4167         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4168             qemu_iothread_add($vmid, $deviceid, $device);
4169             $devicefull .= ",iothread=iothread-$deviceid";
4170         }
4171
4172         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4173             $devicefull .= ",num_queues=$device->{queues}";
4174         }
4175
4176         qemu_deviceadd($vmid, $devicefull);
4177         qemu_deviceaddverify($vmid, $deviceid);
4178
4179     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4180
4181         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4182         qemu_driveadd($storecfg, $vmid, $device);
4183
4184         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4185         eval { qemu_deviceadd($vmid, $devicefull); };
4186         if (my $err = $@) {
4187             eval { qemu_drivedel($vmid, $deviceid); };
4188             warn $@ if $@;
4189             die $err;
4190         }
4191
4192     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4193
4194         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4195
4196         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4197         my $use_old_bios_files = undef;
4198         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4199
4200         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4201         qemu_deviceadd($vmid, $netdevicefull);
4202         eval {
4203             qemu_deviceaddverify($vmid, $deviceid);
4204             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4205         };
4206         if (my $err = $@) {
4207             eval { qemu_netdevdel($vmid, $deviceid); };
4208             warn $@ if $@;
4209             die $err;
4210         }
4211
4212     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4213
4214         my $bridgeid = $2;
4215         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4216         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4217
4218         qemu_deviceadd($vmid, $devicefull);
4219         qemu_deviceaddverify($vmid, $deviceid);
4220
4221     } else {
4222         die "can't hotplug device '$deviceid'\n";
4223     }
4224
4225     return 1;
4226 }
4227
4228 # fixme: this should raise exceptions on error!
4229 sub vm_deviceunplug {
4230     my ($vmid, $conf, $deviceid) = @_;
4231
4232     my $devices_list = vm_devices_list($vmid);
4233     return 1 if !defined($devices_list->{$deviceid});
4234
4235     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4236
4237     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240
4241     } elsif ($deviceid =~ m/^usb\d+$/) {
4242
4243         die "usb hotplug currently not reliable\n";
4244         # when unplugging usb devices this way,
4245         # there may be remaining usb controllers/hubs
4246         # so we disable it for now
4247         qemu_devicedel($vmid, $deviceid);
4248         qemu_devicedelverify($vmid, $deviceid);
4249
4250     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4251
4252         qemu_devicedel($vmid, $deviceid);
4253         qemu_devicedelverify($vmid, $deviceid);
4254         qemu_drivedel($vmid, $deviceid);
4255         qemu_iothread_del($conf, $vmid, $deviceid);
4256
4257     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4258
4259         qemu_devicedel($vmid, $deviceid);
4260         qemu_devicedelverify($vmid, $deviceid);
4261         qemu_iothread_del($conf, $vmid, $deviceid);
4262
4263     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4264
4265         qemu_devicedel($vmid, $deviceid);
4266         qemu_drivedel($vmid, $deviceid);
4267         qemu_deletescsihw($conf, $vmid, $deviceid);
4268
4269     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4270
4271         qemu_devicedel($vmid, $deviceid);
4272         qemu_devicedelverify($vmid, $deviceid);
4273         qemu_netdevdel($vmid, $deviceid);
4274
4275     } else {
4276         die "can't unplug device '$deviceid'\n";
4277     }
4278
4279     return 1;
4280 }
4281
4282 sub qemu_deviceadd {
4283     my ($vmid, $devicefull) = @_;
4284
4285     $devicefull = "driver=".$devicefull;
4286     my %options =  split(/[=,]/, $devicefull);
4287
4288     vm_mon_cmd($vmid, "device_add" , %options);
4289 }
4290
4291 sub qemu_devicedel {
4292     my ($vmid, $deviceid) = @_;
4293
4294     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4295 }
4296
4297 sub qemu_iothread_add {
4298     my($vmid, $deviceid, $device) = @_;
4299
4300     if ($device->{iothread}) {
4301         my $iothreads = vm_iothreads_list($vmid);
4302         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4303     }
4304 }
4305
4306 sub qemu_iothread_del {
4307     my($conf, $vmid, $deviceid) = @_;
4308
4309     my $confid = $deviceid;
4310     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4311         $confid = 'scsi' . $1;
4312     }
4313     my $device = parse_drive($confid, $conf->{$confid});
4314     if ($device->{iothread}) {
4315         my $iothreads = vm_iothreads_list($vmid);
4316         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4317     }
4318 }
4319
4320 sub qemu_objectadd {
4321     my($vmid, $objectid, $qomtype) = @_;
4322
4323     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4324
4325     return 1;
4326 }
4327
4328 sub qemu_objectdel {
4329     my($vmid, $objectid) = @_;
4330
4331     vm_mon_cmd($vmid, "object-del", id => $objectid);
4332
4333     return 1;
4334 }
4335
4336 sub qemu_driveadd {
4337     my ($storecfg, $vmid, $device) = @_;
4338
4339     my $drive = print_drive_full($storecfg, $vmid, $device);
4340     $drive =~ s/\\/\\\\/g;
4341     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4342
4343     # If the command succeeds qemu prints: "OK"
4344     return 1 if $ret =~ m/OK/s;
4345
4346     die "adding drive failed: $ret\n";
4347 }
4348
4349 sub qemu_drivedel {
4350     my($vmid, $deviceid) = @_;
4351
4352     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4353     $ret =~ s/^\s+//;
4354
4355     return 1 if $ret eq "";
4356
4357     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4358     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4359
4360     die "deleting drive $deviceid failed : $ret\n";
4361 }
4362
4363 sub qemu_deviceaddverify {
4364     my ($vmid, $deviceid) = @_;
4365
4366     for (my $i = 0; $i <= 5; $i++) {
4367          my $devices_list = vm_devices_list($vmid);
4368          return 1 if defined($devices_list->{$deviceid});
4369          sleep 1;
4370     }
4371
4372     die "error on hotplug device '$deviceid'\n";
4373 }
4374
4375
4376 sub qemu_devicedelverify {
4377     my ($vmid, $deviceid) = @_;
4378
4379     # need to verify that the device is correctly removed as device_del
4380     # is async and empty return is not reliable
4381
4382     for (my $i = 0; $i <= 5; $i++) {
4383          my $devices_list = vm_devices_list($vmid);
4384          return 1 if !defined($devices_list->{$deviceid});
4385          sleep 1;
4386     }
4387
4388     die "error on hot-unplugging device '$deviceid'\n";
4389 }
4390
4391 sub qemu_findorcreatescsihw {
4392     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4393
4394     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4395
4396     my $scsihwid="$controller_prefix$controller";
4397     my $devices_list = vm_devices_list($vmid);
4398
4399     if(!defined($devices_list->{$scsihwid})) {
4400         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4401     }
4402
4403     return 1;
4404 }
4405
4406 sub qemu_deletescsihw {
4407     my ($conf, $vmid, $opt) = @_;
4408
4409     my $device = parse_drive($opt, $conf->{$opt});
4410
4411     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4412         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4413         return 1;
4414     }
4415
4416     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4417
4418     my $devices_list = vm_devices_list($vmid);
4419     foreach my $opt (keys %{$devices_list}) {
4420         if (PVE::QemuServer::is_valid_drivename($opt)) {
4421             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4422             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4423                 return 1;
4424             }
4425         }
4426     }
4427
4428     my $scsihwid="scsihw$controller";
4429
4430     vm_deviceunplug($vmid, $conf, $scsihwid);
4431
4432     return 1;
4433 }
4434
4435 sub qemu_add_pci_bridge {
4436     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4437
4438     my $bridges = {};
4439
4440     my $bridgeid;
4441
4442     print_pci_addr($device, $bridges, $arch, $machine_type);
4443
4444     while (my ($k, $v) = each %$bridges) {
4445         $bridgeid = $k;
4446     }
4447     return 1 if !defined($bridgeid) || $bridgeid < 1;
4448
4449     my $bridge = "pci.$bridgeid";
4450     my $devices_list = vm_devices_list($vmid);
4451
4452     if (!defined($devices_list->{$bridge})) {
4453         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4454     }
4455
4456     return 1;
4457 }
4458
4459 sub qemu_set_link_status {
4460     my ($vmid, $device, $up) = @_;
4461
4462     vm_mon_cmd($vmid, "set_link", name => $device,
4463                up => $up ? JSON::true : JSON::false);
4464 }
4465
4466 sub qemu_netdevadd {
4467     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4468
4469     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4470     my %options =  split(/[=,]/, $netdev);
4471
4472     vm_mon_cmd($vmid, "netdev_add",  %options);
4473     return 1;
4474 }
4475
4476 sub qemu_netdevdel {
4477     my ($vmid, $deviceid) = @_;
4478
4479     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4480 }
4481
4482 sub qemu_usb_hotplug {
4483     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4484
4485     return if !$device;
4486
4487     # remove the old one first
4488     vm_deviceunplug($vmid, $conf, $deviceid);
4489
4490     # check if xhci controller is necessary and available
4491     if ($device->{usb3}) {
4492
4493         my $devicelist = vm_devices_list($vmid);
4494
4495         if (!$devicelist->{xhci}) {
4496             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4497             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4498         }
4499     }
4500     my $d = parse_usb_device($device->{host});
4501     $d->{usb3} = $device->{usb3};
4502
4503     # add the new one
4504     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4505 }
4506
4507 sub qemu_cpu_hotplug {
4508     my ($vmid, $conf, $vcpus) = @_;
4509
4510     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4511
4512     my $sockets = 1;
4513     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4514     $sockets = $conf->{sockets} if  $conf->{sockets};
4515     my $cores = $conf->{cores} || 1;
4516     my $maxcpus = $sockets * $cores;
4517
4518     $vcpus = $maxcpus if !$vcpus;
4519
4520     die "you can't add more vcpus than maxcpus\n"
4521         if $vcpus > $maxcpus;
4522
4523     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4524
4525     if ($vcpus < $currentvcpus) {
4526
4527         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4528
4529             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4530                 qemu_devicedel($vmid, "cpu$i");
4531                 my $retry = 0;
4532                 my $currentrunningvcpus = undef;
4533                 while (1) {
4534                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4535                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4536                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4537                     $retry++;
4538                     sleep 1;
4539                 }
4540                 #update conf after each succesfull cpu unplug
4541                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4542                 PVE::QemuConfig->write_config($vmid, $conf);
4543             }
4544         } else {
4545             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4546         }
4547
4548         return;
4549     }
4550
4551     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4552     die "vcpus in running vm does not match its configuration\n"
4553         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4554
4555     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4556
4557         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4558             my $cpustr = print_cpu_device($conf, $i);
4559             qemu_deviceadd($vmid, $cpustr);
4560
4561             my $retry = 0;
4562             my $currentrunningvcpus = undef;
4563             while (1) {
4564                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4565                 last if scalar(@{$currentrunningvcpus}) == $i;
4566                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4567                 sleep 1;
4568                 $retry++;
4569             }
4570             #update conf after each succesfull cpu hotplug
4571             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4572             PVE::QemuConfig->write_config($vmid, $conf);
4573         }
4574     } else {
4575
4576         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4577             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4578         }
4579     }
4580 }
4581
4582 sub qemu_block_set_io_throttle {
4583     my ($vmid, $deviceid,
4584         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4585         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4586         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4587         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4588
4589     return if !check_running($vmid) ;
4590
4591     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4592         bps => int($bps),
4593         bps_rd => int($bps_rd),
4594         bps_wr => int($bps_wr),
4595         iops => int($iops),
4596         iops_rd => int($iops_rd),
4597         iops_wr => int($iops_wr),
4598         bps_max => int($bps_max),
4599         bps_rd_max => int($bps_rd_max),
4600         bps_wr_max => int($bps_wr_max),
4601         iops_max => int($iops_max),
4602         iops_rd_max => int($iops_rd_max),
4603         iops_wr_max => int($iops_wr_max),
4604         bps_max_length => int($bps_max_length),
4605         bps_rd_max_length => int($bps_rd_max_length),
4606         bps_wr_max_length => int($bps_wr_max_length),
4607         iops_max_length => int($iops_max_length),
4608         iops_rd_max_length => int($iops_rd_max_length),
4609         iops_wr_max_length => int($iops_wr_max_length),
4610     );
4611
4612 }
4613
4614 # old code, only used to shutdown old VM after update
4615 sub __read_avail {
4616     my ($fh, $timeout) = @_;
4617
4618     my $sel = new IO::Select;
4619     $sel->add($fh);
4620
4621     my $res = '';
4622     my $buf;
4623
4624     my @ready;
4625     while (scalar (@ready = $sel->can_read($timeout))) {
4626         my $count;
4627         if ($count = $fh->sysread($buf, 8192)) {
4628             if ($buf =~ /^(.*)\(qemu\) $/s) {
4629                 $res .= $1;
4630                 last;
4631             } else {
4632                 $res .= $buf;
4633             }
4634         } else {
4635             if (!defined($count)) {
4636                 die "$!\n";
4637             }
4638             last;
4639         }
4640     }
4641
4642     die "monitor read timeout\n" if !scalar(@ready);
4643
4644     return $res;
4645 }
4646
4647 sub qemu_block_resize {
4648     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4649
4650     my $running = check_running($vmid);
4651
4652     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4653
4654     return if !$running;
4655
4656     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4657
4658 }
4659
4660 sub qemu_volume_snapshot {
4661     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4666         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4667     } else {
4668         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4669     }
4670 }
4671
4672 sub qemu_volume_snapshot_delete {
4673     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4674
4675     my $running = check_running($vmid);
4676
4677     if($running) {
4678
4679         $running = undef;
4680         my $conf = PVE::QemuConfig->load_config($vmid);
4681         foreach_drive($conf, sub {
4682             my ($ds, $drive) = @_;
4683             $running = 1 if $drive->{file} eq $volid;
4684         });
4685     }
4686
4687     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4688         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4689     } else {
4690         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4691     }
4692 }
4693
4694 sub set_migration_caps {
4695     my ($vmid) = @_;
4696
4697     my $cap_ref = [];
4698
4699     my $enabled_cap = {
4700         "auto-converge" => 1,
4701         "xbzrle" => 1,
4702         "x-rdma-pin-all" => 0,
4703         "zero-blocks" => 0,
4704         "compress" => 0
4705     };
4706
4707     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4708
4709     for my $supported_capability (@$supported_capabilities) {
4710         push @$cap_ref, {
4711             capability => $supported_capability->{capability},
4712             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4713         };
4714     }
4715
4716     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4717 }
4718
4719 my $fast_plug_option = {
4720     'lock' => 1,
4721     'name' => 1,
4722     'onboot' => 1,
4723     'shares' => 1,
4724     'startup' => 1,
4725     'description' => 1,
4726     'protection' => 1,
4727     'vmstatestorage' => 1,
4728     'hookscript' => 1,
4729 };
4730
4731 # hotplug changes in [PENDING]
4732 # $selection hash can be used to only apply specified options, for
4733 # example: { cores => 1 } (only apply changed 'cores')
4734 # $errors ref is used to return error messages
4735 sub vmconfig_hotplug_pending {
4736     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4737
4738     my $defaults = load_defaults();
4739     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4740
4741     # commit values which do not have any impact on running VM first
4742     # Note: those option cannot raise errors, we we do not care about
4743     # $selection and always apply them.
4744
4745     my $add_error = sub {
4746         my ($opt, $msg) = @_;
4747         $errors->{$opt} = "hotplug problem - $msg";
4748     };
4749
4750     my $changes = 0;
4751     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4752         if ($fast_plug_option->{$opt}) {
4753             $conf->{$opt} = $conf->{pending}->{$opt};
4754             delete $conf->{pending}->{$opt};
4755             $changes = 1;
4756         }
4757     }
4758
4759     if ($changes) {
4760         PVE::QemuConfig->write_config($vmid, $conf);
4761         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4762     }
4763
4764     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4765
4766     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4767     while (my ($opt, $force) = each %$pending_delete_hash) {
4768         next if $selection && !$selection->{$opt};
4769         eval {
4770             if ($opt eq 'hotplug') {
4771                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4772             } elsif ($opt eq 'tablet') {
4773                 die "skip\n" if !$hotplug_features->{usb};
4774                 if ($defaults->{tablet}) {
4775                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4776                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4777                         if $arch eq 'aarch64';
4778                 } else {
4779                     vm_deviceunplug($vmid, $conf, 'tablet');
4780                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4781                 }
4782             } elsif ($opt =~ m/^usb\d+/) {
4783                 die "skip\n";
4784                 # since we cannot reliably hot unplug usb devices
4785                 # we are disabling it
4786                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4787                 vm_deviceunplug($vmid, $conf, $opt);
4788             } elsif ($opt eq 'vcpus') {
4789                 die "skip\n" if !$hotplug_features->{cpu};
4790                 qemu_cpu_hotplug($vmid, $conf, undef);
4791             } elsif ($opt eq 'balloon') {
4792                 # enable balloon device is not hotpluggable
4793                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4794                 # here we reset the ballooning value to memory
4795                 my $balloon = $conf->{memory} || $defaults->{memory};
4796                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4797             } elsif ($fast_plug_option->{$opt}) {
4798                 # do nothing
4799             } elsif ($opt =~ m/^net(\d+)$/) {
4800                 die "skip\n" if !$hotplug_features->{network};
4801                 vm_deviceunplug($vmid, $conf, $opt);
4802             } elsif (is_valid_drivename($opt)) {
4803                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4804                 vm_deviceunplug($vmid, $conf, $opt);
4805                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4806             } elsif ($opt =~ m/^memory$/) {
4807                 die "skip\n" if !$hotplug_features->{memory};
4808                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4809             } elsif ($opt eq 'cpuunits') {
4810                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4811             } elsif ($opt eq 'cpulimit') {
4812                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4813             } else {
4814                 die "skip\n";
4815             }
4816         };
4817         if (my $err = $@) {
4818             &$add_error($opt, $err) if $err ne "skip\n";
4819         } else {
4820             # save new config if hotplug was successful
4821             delete $conf->{$opt};
4822             vmconfig_undelete_pending_option($conf, $opt);
4823             PVE::QemuConfig->write_config($vmid, $conf);
4824             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825         }
4826     }
4827
4828     my $apply_pending_cloudinit;
4829     $apply_pending_cloudinit = sub {
4830         my ($key, $value) = @_;
4831         $apply_pending_cloudinit = sub {}; # once is enough
4832
4833         my @cloudinit_opts = keys %$confdesc_cloudinit;
4834         foreach my $opt (keys %{$conf->{pending}}) {
4835             next if !grep { $_ eq $opt } @cloudinit_opts;
4836             $conf->{$opt} = delete $conf->{pending}->{$opt};
4837         }
4838
4839         my $new_conf = { %$conf };
4840         $new_conf->{$key} = $value;
4841         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4842     };
4843
4844     foreach my $opt (keys %{$conf->{pending}}) {
4845         next if $selection && !$selection->{$opt};
4846         my $value = $conf->{pending}->{$opt};
4847         eval {
4848             if ($opt eq 'hotplug') {
4849                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4850             } elsif ($opt eq 'tablet') {
4851                 die "skip\n" if !$hotplug_features->{usb};
4852                 if ($value == 1) {
4853                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4854                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4855                         if $arch eq 'aarch64';
4856                 } elsif ($value == 0) {
4857                     vm_deviceunplug($vmid, $conf, 'tablet');
4858                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4859                 }
4860             } elsif ($opt =~ m/^usb\d+$/) {
4861                 die "skip\n";
4862                 # since we cannot reliably hot unplug usb devices
4863                 # we are disabling it
4864                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4865                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4866                 die "skip\n" if !$d;
4867                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4868             } elsif ($opt eq 'vcpus') {
4869                 die "skip\n" if !$hotplug_features->{cpu};
4870                 qemu_cpu_hotplug($vmid, $conf, $value);
4871             } elsif ($opt eq 'balloon') {
4872                 # enable/disable balloning device is not hotpluggable
4873                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4874                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4875                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4876
4877                 # allow manual ballooning if shares is set to zero
4878                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4879                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4880                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4881                 }
4882             } elsif ($opt =~ m/^net(\d+)$/) {
4883                 # some changes can be done without hotplug
4884                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4885                                     $vmid, $opt, $value, $arch, $machine_type);
4886             } elsif (is_valid_drivename($opt)) {
4887                 # some changes can be done without hotplug
4888                 my $drive = parse_drive($opt, $value);
4889                 if (drive_is_cloudinit($drive)) {
4890                     &$apply_pending_cloudinit($opt, $value);
4891                 }
4892                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4893                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4894             } elsif ($opt =~ m/^memory$/) { #dimms
4895                 die "skip\n" if !$hotplug_features->{memory};
4896                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4897             } elsif ($opt eq 'cpuunits') {
4898                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4899             } elsif ($opt eq 'cpulimit') {
4900                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4901                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4902             } else {
4903                 die "skip\n";  # skip non-hot-pluggable options
4904             }
4905         };
4906         if (my $err = $@) {
4907             &$add_error($opt, $err) if $err ne "skip\n";
4908         } else {
4909             # save new config if hotplug was successful
4910             $conf->{$opt} = $value;
4911             delete $conf->{pending}->{$opt};
4912             PVE::QemuConfig->write_config($vmid, $conf);
4913             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4914         }
4915     }
4916 }
4917
4918 sub try_deallocate_drive {
4919     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4920
4921     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4922         my $volid = $drive->{file};
4923         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4924             my $sid = PVE::Storage::parse_volume_id($volid);
4925             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4926
4927             # check if the disk is really unused
4928             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4929                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4930             PVE::Storage::vdisk_free($storecfg, $volid);
4931             return 1;
4932         } else {
4933             # If vm is not owner of this disk remove from config
4934             return 1;
4935         }
4936     }
4937
4938     return undef;
4939 }
4940
4941 sub vmconfig_delete_or_detach_drive {
4942     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4943
4944     my $drive = parse_drive($opt, $conf->{$opt});
4945
4946     my $rpcenv = PVE::RPCEnvironment::get();
4947     my $authuser = $rpcenv->get_user();
4948
4949     if ($force) {
4950         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4951         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4952     } else {
4953         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4954     }
4955 }
4956
4957 sub vmconfig_apply_pending {
4958     my ($vmid, $conf, $storecfg) = @_;
4959
4960     # cold plug
4961
4962     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4963     while (my ($opt, $force) = each %$pending_delete_hash) {
4964         die "internal error" if $opt =~ m/^unused/;
4965         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4966         if (!defined($conf->{$opt})) {
4967             vmconfig_undelete_pending_option($conf, $opt);
4968             PVE::QemuConfig->write_config($vmid, $conf);
4969         } elsif (is_valid_drivename($opt)) {
4970             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4971             vmconfig_undelete_pending_option($conf, $opt);
4972             delete $conf->{$opt};
4973             PVE::QemuConfig->write_config($vmid, $conf);
4974         } else {
4975             vmconfig_undelete_pending_option($conf, $opt);
4976             delete $conf->{$opt};
4977             PVE::QemuConfig->write_config($vmid, $conf);
4978         }
4979     }
4980
4981     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4982
4983     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4984         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4985
4986         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4987             # skip if nothing changed
4988         } elsif (is_valid_drivename($opt)) {
4989             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4990                 if defined($conf->{$opt});
4991             $conf->{$opt} = $conf->{pending}->{$opt};
4992         } else {
4993             $conf->{$opt} = $conf->{pending}->{$opt};
4994         }
4995
4996         delete $conf->{pending}->{$opt};
4997         PVE::QemuConfig->write_config($vmid, $conf);
4998     }
4999 }
5000
5001 my $safe_num_ne = sub {
5002     my ($a, $b) = @_;
5003
5004     return 0 if !defined($a) && !defined($b);
5005     return 1 if !defined($a);
5006     return 1 if !defined($b);
5007
5008     return $a != $b;
5009 };
5010
5011 my $safe_string_ne = sub {
5012     my ($a, $b) = @_;
5013
5014     return 0 if !defined($a) && !defined($b);
5015     return 1 if !defined($a);
5016     return 1 if !defined($b);
5017
5018     return $a ne $b;
5019 };
5020
5021 sub vmconfig_update_net {
5022     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5023
5024     my $newnet = parse_net($value);
5025
5026     if ($conf->{$opt}) {
5027         my $oldnet = parse_net($conf->{$opt});
5028
5029         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5030             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5031             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5032             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5033
5034             # for non online change, we try to hot-unplug
5035             die "skip\n" if !$hotplug;
5036             vm_deviceunplug($vmid, $conf, $opt);
5037         } else {
5038
5039             die "internal error" if $opt !~ m/net(\d+)/;
5040             my $iface = "tap${vmid}i$1";
5041
5042             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5043                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5044                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5045                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5046                 PVE::Network::tap_unplug($iface);
5047                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5048             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5049                 # Rate can be applied on its own but any change above needs to
5050                 # include the rate in tap_plug since OVS resets everything.
5051                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5052             }
5053
5054             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5055                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5056             }
5057
5058             return 1;
5059         }
5060     }
5061
5062     if ($hotplug) {
5063         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5064     } else {
5065         die "skip\n";
5066     }
5067 }
5068
5069 sub vmconfig_update_disk {
5070     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5071
5072     # fixme: do we need force?
5073
5074     my $drive = parse_drive($opt, $value);
5075
5076     if ($conf->{$opt}) {
5077
5078         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5079
5080             my $media = $drive->{media} || 'disk';
5081             my $oldmedia = $old_drive->{media} || 'disk';
5082             die "unable to change media type\n" if $media ne $oldmedia;
5083
5084             if (!drive_is_cdrom($old_drive)) {
5085
5086                 if ($drive->{file} ne $old_drive->{file}) {
5087
5088                     die "skip\n" if !$hotplug;
5089
5090                     # unplug and register as unused
5091                     vm_deviceunplug($vmid, $conf, $opt);
5092                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5093
5094                 } else {
5095                     # update existing disk
5096
5097                     # skip non hotpluggable value
5098                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5099                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5100                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5101                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5102                         die "skip\n";
5103                     }
5104
5105                     # apply throttle
5106                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5107                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5108                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5109                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5110                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5111                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5112                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5113                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5114                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5115                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5116                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5117                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5118                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5119                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5120                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5121                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5122                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5123                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5124
5125                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5126                                                    ($drive->{mbps} || 0)*1024*1024,
5127                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5128                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5129                                                    $drive->{iops} || 0,
5130                                                    $drive->{iops_rd} || 0,
5131                                                    $drive->{iops_wr} || 0,
5132                                                    ($drive->{mbps_max} || 0)*1024*1024,
5133                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5134                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5135                                                    $drive->{iops_max} || 0,
5136                                                    $drive->{iops_rd_max} || 0,
5137                                                    $drive->{iops_wr_max} || 0,
5138                                                    $drive->{bps_max_length} || 1,
5139                                                    $drive->{bps_rd_max_length} || 1,
5140                                                    $drive->{bps_wr_max_length} || 1,
5141                                                    $drive->{iops_max_length} || 1,
5142                                                    $drive->{iops_rd_max_length} || 1,
5143                                                    $drive->{iops_wr_max_length} || 1);
5144
5145                     }
5146
5147                     return 1;
5148                 }
5149
5150             } else { # cdrom
5151
5152                 if ($drive->{file} eq 'none') {
5153                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5154                     if (drive_is_cloudinit($old_drive)) {
5155                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5156                     }
5157                 } else {
5158                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5159                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5160                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5161                 }
5162
5163                 return 1;
5164             }
5165         }
5166     }
5167
5168     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5169     # hotplug new disks
5170     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5171     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5172 }
5173
5174 sub vm_start {
5175     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5176         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5177
5178     PVE::QemuConfig->lock_config($vmid, sub {
5179         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5180
5181         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5182
5183         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5184
5185         PVE::QemuConfig->check_lock($conf)
5186             if !($skiplock || $is_suspended);
5187
5188         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5189
5190         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5191             vmconfig_apply_pending($vmid, $conf, $storecfg);
5192             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5193         }
5194
5195         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5196
5197         my $defaults = load_defaults();
5198
5199         # set environment variable useful inside network script
5200         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5201
5202         my $local_volumes = {};
5203
5204         if ($targetstorage) {
5205             foreach_drive($conf, sub {
5206                 my ($ds, $drive) = @_;
5207
5208                 return if drive_is_cdrom($drive);
5209
5210                 my $volid = $drive->{file};
5211
5212                 return if !$volid;
5213
5214                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5215
5216                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5217                 return if $scfg->{shared};
5218                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5219             });
5220
5221             my $format = undef;
5222
5223             foreach my $opt (sort keys %$local_volumes) {
5224
5225                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5226                 my $drive = parse_drive($opt, $conf->{$opt});
5227
5228                 #if remote storage is specified, use default format
5229                 if ($targetstorage && $targetstorage ne "1") {
5230                     $storeid = $targetstorage;
5231                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5232                     $format = $defFormat;
5233                 } else {
5234                     #else we use same format than original
5235                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5236                     $format = qemu_img_format($scfg, $volid);
5237                 }
5238
5239                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5240                 my $newdrive = $drive;
5241                 $newdrive->{format} = $format;
5242                 $newdrive->{file} = $newvolid;
5243                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5244                 $local_volumes->{$opt} = $drivestr;
5245                 #pass drive to conf for command line
5246                 $conf->{$opt} = $drivestr;
5247             }
5248         }
5249
5250         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5251
5252         if ($is_suspended) {
5253             # enforce machine type on suspended vm to ensure HW compatibility
5254             $forcemachine = $conf->{runningmachine};
5255             print "Resuming suspended VM\n";
5256         }
5257
5258         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5259
5260         my $migrate_port = 0;
5261         my $migrate_uri;
5262         if ($statefile) {
5263             if ($statefile eq 'tcp') {
5264                 my $localip = "localhost";
5265                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5266                 my $nodename = PVE::INotify::nodename();
5267
5268                 if (!defined($migration_type)) {
5269                     if (defined($datacenterconf->{migration}->{type})) {
5270                         $migration_type = $datacenterconf->{migration}->{type};
5271                     } else {
5272                         $migration_type = 'secure';
5273                     }
5274                 }
5275
5276                 if ($migration_type eq 'insecure') {
5277                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5278                     if ($migrate_network_addr) {
5279                         $localip = $migrate_network_addr;
5280                     } else {
5281                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5282                     }
5283
5284                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5285                 }
5286
5287                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5288                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5289                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5290                 push @$cmd, '-incoming', $migrate_uri;
5291                 push @$cmd, '-S';
5292
5293             } elsif ($statefile eq 'unix') {
5294                 # should be default for secure migrations as a ssh TCP forward
5295                 # tunnel is not deterministic reliable ready and fails regurarly
5296                 # to set up in time, so use UNIX socket forwards
5297                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5298                 unlink $socket_addr;
5299
5300                 $migrate_uri = "unix:$socket_addr";
5301
5302                 push @$cmd, '-incoming', $migrate_uri;
5303                 push @$cmd, '-S';
5304
5305             } else {
5306                 push @$cmd, '-loadstate', $statefile;
5307             }
5308         } elsif ($paused) {
5309             push @$cmd, '-S';
5310         }
5311
5312         # host pci devices
5313         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5314           my $d = parse_hostpci($conf->{"hostpci$i"});
5315           next if !$d;
5316           my $pcidevices = $d->{pciid};
5317           foreach my $pcidevice (@$pcidevices) {
5318                 my $pciid = $pcidevice->{id};
5319
5320                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5321                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5322                 die "no pci device info for device '$pciid'\n" if !$info;
5323
5324                 if ($d->{mdev}) {
5325                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5326                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5327                 } else {
5328                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5329                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5330                     die "can't reset pci device '$pciid'\n"
5331                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5332                 }
5333           }
5334         }
5335
5336         PVE::Storage::activate_volumes($storecfg, $vollist);
5337
5338         eval {
5339             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5340                 outfunc => sub {}, errfunc => sub {});
5341         };
5342         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5343         # timeout should be more than enough here...
5344         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5345
5346         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5347                                                   : $defaults->{cpuunits};
5348
5349         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5350         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5351
5352         my %properties = (
5353             Slice => 'qemu.slice',
5354             KillMode => 'none',
5355             CPUShares => $cpuunits
5356         );
5357
5358         if (my $cpulimit = $conf->{cpulimit}) {
5359             $properties{CPUQuota} = int($cpulimit * 100);
5360         }
5361         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5362
5363         my $run_qemu = sub {
5364             PVE::Tools::run_fork sub {
5365                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5366                 run_command($cmd, %run_params);
5367             };
5368         };
5369
5370         if ($conf->{hugepages}) {
5371
5372             my $code = sub {
5373                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5374                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5375
5376                 PVE::QemuServer::Memory::hugepages_mount();
5377                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5378
5379                 eval { $run_qemu->() };
5380                 if (my $err = $@) {
5381                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5382                     die $err;
5383                 }
5384
5385                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5386             };
5387             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5388
5389         } else {
5390             eval { $run_qemu->() };
5391         }
5392
5393         if (my $err = $@) {
5394             # deactivate volumes if start fails
5395             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5396             die "start failed: $err";
5397         }
5398
5399         print "migration listens on $migrate_uri\n" if $migrate_uri;
5400
5401         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5402             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5403             warn $@ if $@;
5404         }
5405
5406         #start nbd server for storage migration
5407         if ($targetstorage) {
5408             my $nodename = PVE::INotify::nodename();
5409             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5410             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5411             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5412             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5413
5414             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5415
5416             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5417
5418             foreach my $opt (sort keys %$local_volumes) {
5419                 my $volid = $local_volumes->{$opt};
5420                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5421                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5422                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5423             }
5424         }
5425
5426         if ($migratedfrom) {
5427             eval {
5428                 set_migration_caps($vmid);
5429             };
5430             warn $@ if $@;
5431
5432             if ($spice_port) {
5433                 print "spice listens on port $spice_port\n";
5434                 if ($spice_ticket) {
5435                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5436                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5437                 }
5438             }
5439
5440         } else {
5441             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5442                 if !$statefile && $conf->{balloon};
5443
5444             foreach my $opt (keys %$conf) {
5445                 next if $opt !~  m/^net\d+$/;
5446                 my $nicconf = parse_net($conf->{$opt});
5447                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5448             }
5449         }
5450
5451         vm_mon_cmd_nocheck($vmid, 'qom-set',
5452                     path => "machine/peripheral/balloon0",
5453                     property => "guest-stats-polling-interval",
5454                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5455
5456         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5457             print "Resumed VM, removing state\n";
5458             delete $conf->@{qw(lock vmstate runningmachine)};
5459             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5460             PVE::Storage::vdisk_free($storecfg, $vmstate);
5461             PVE::QemuConfig->write_config($vmid, $conf);
5462         }
5463
5464         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5465     });
5466 }
5467
5468 sub vm_mon_cmd {
5469     my ($vmid, $execute, %params) = @_;
5470
5471     my $cmd = { execute => $execute, arguments => \%params };
5472     vm_qmp_command($vmid, $cmd);
5473 }
5474
5475 sub vm_mon_cmd_nocheck {
5476     my ($vmid, $execute, %params) = @_;
5477
5478     my $cmd = { execute => $execute, arguments => \%params };
5479     vm_qmp_command($vmid, $cmd, 1);
5480 }
5481
5482 sub vm_qmp_command {
5483     my ($vmid, $cmd, $nocheck) = @_;
5484
5485     my $res;
5486
5487     my $timeout;
5488     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5489         $timeout = $cmd->{arguments}->{timeout};
5490         delete $cmd->{arguments}->{timeout};
5491     }
5492
5493     eval {
5494         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5495         my $sname = qmp_socket($vmid);
5496         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5497             my $qmpclient = PVE::QMPClient->new();
5498
5499             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5500         } else {
5501             die "unable to open monitor socket\n";
5502         }
5503     };
5504     if (my $err = $@) {
5505         syslog("err", "VM $vmid qmp command failed - $err");
5506         die $err;
5507     }
5508
5509     return $res;
5510 }
5511
5512 sub vm_human_monitor_command {
5513     my ($vmid, $cmdline) = @_;
5514
5515     my $res;
5516
5517     my $cmd = {
5518         execute => 'human-monitor-command',
5519         arguments => { 'command-line' => $cmdline},
5520     };
5521
5522     return vm_qmp_command($vmid, $cmd);
5523 }
5524
5525 sub vm_commandline {
5526     my ($storecfg, $vmid, $snapname) = @_;
5527
5528     my $conf = PVE::QemuConfig->load_config($vmid);
5529
5530     if ($snapname) {
5531         my $snapshot = $conf->{snapshots}->{$snapname};
5532         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5533
5534         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5535
5536         $conf = $snapshot;
5537     }
5538
5539     my $defaults = load_defaults();
5540
5541     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5542
5543     return PVE::Tools::cmd2string($cmd);
5544 }
5545
5546 sub vm_reset {
5547     my ($vmid, $skiplock) = @_;
5548
5549     PVE::QemuConfig->lock_config($vmid, sub {
5550
5551         my $conf = PVE::QemuConfig->load_config($vmid);
5552
5553         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5554
5555         vm_mon_cmd($vmid, "system_reset");
5556     });
5557 }
5558
5559 sub get_vm_volumes {
5560     my ($conf) = @_;
5561
5562     my $vollist = [];
5563     foreach_volid($conf, sub {
5564         my ($volid, $attr) = @_;
5565
5566         return if $volid =~ m|^/|;
5567
5568         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5569         return if !$sid;
5570
5571         push @$vollist, $volid;
5572     });
5573
5574     return $vollist;
5575 }
5576
5577 sub vm_stop_cleanup {
5578     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5579
5580     eval {
5581
5582         if (!$keepActive) {
5583             my $vollist = get_vm_volumes($conf);
5584             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5585         }
5586
5587         foreach my $ext (qw(mon qmp pid vnc qga)) {
5588             unlink "/var/run/qemu-server/${vmid}.$ext";
5589         }
5590
5591         if ($conf->{ivshmem}) {
5592             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5593             # just delete it for now, VMs which have this already open do not
5594             # are affected, but new VMs will get a separated one. If this
5595             # becomes an issue we either add some sort of ref-counting or just
5596             # add a "don't delete on stop" flag to the ivshmem format.
5597             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5598         }
5599
5600         foreach my $key (keys %$conf) {
5601             next if $key !~ m/^hostpci(\d+)$/;
5602             my $hostpciindex = $1;
5603             my $d = parse_hostpci($conf->{$key});
5604             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5605
5606             foreach my $pci (@{$d->{pciid}}) {
5607                 my $pciid = $pci->{id};
5608                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5609             }
5610         }
5611
5612         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5613     };
5614     warn $@ if $@; # avoid errors - just warn
5615 }
5616
5617 # Note: use $nockeck to skip tests if VM configuration file exists.
5618 # We need that when migration VMs to other nodes (files already moved)
5619 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5620 sub vm_stop {
5621     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5622
5623     $force = 1 if !defined($force) && !$shutdown;
5624
5625     if ($migratedfrom){
5626         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5627         kill 15, $pid if $pid;
5628         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5629         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5630         return;
5631     }
5632
5633     PVE::QemuConfig->lock_config($vmid, sub {
5634
5635         my $pid = check_running($vmid, $nocheck);
5636         return if !$pid;
5637
5638         my $conf;
5639         if (!$nocheck) {
5640             $conf = PVE::QemuConfig->load_config($vmid);
5641             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5642             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5643                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5644                 $timeout = $opts->{down} if $opts->{down};
5645             }
5646             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5647         }
5648
5649         eval {
5650             if ($shutdown) {
5651                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5652                     vm_qmp_command($vmid, {
5653                         execute => "guest-shutdown",
5654                         arguments => { timeout => $timeout }
5655                     }, $nocheck);
5656                 } else {
5657                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5658                 }
5659             } else {
5660                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5661             }
5662         };
5663         my $err = $@;
5664
5665         if (!$err) {
5666             $timeout = 60 if !defined($timeout);
5667
5668             my $count = 0;
5669             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5670                 $count++;
5671                 sleep 1;
5672             }
5673
5674             if ($count >= $timeout) {
5675                 if ($force) {
5676                     warn "VM still running - terminating now with SIGTERM\n";
5677                     kill 15, $pid;
5678                 } else {
5679                     die "VM quit/powerdown failed - got timeout\n";
5680                 }
5681             } else {
5682                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5683                 return;
5684             }
5685         } else {
5686             if ($force) {
5687                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5688                 kill 15, $pid;
5689             } else {
5690                 die "VM quit/powerdown failed\n";
5691             }
5692         }
5693
5694         # wait again
5695         $timeout = 10;
5696
5697         my $count = 0;
5698         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5699             $count++;
5700             sleep 1;
5701         }
5702
5703         if ($count >= $timeout) {
5704             warn "VM still running - terminating now with SIGKILL\n";
5705             kill 9, $pid;
5706             sleep 1;
5707         }
5708
5709         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5710    });
5711 }
5712
5713 sub vm_suspend {
5714     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5715
5716     my $conf;
5717     my $path;
5718     my $storecfg;
5719     my $vmstate;
5720
5721     PVE::QemuConfig->lock_config($vmid, sub {
5722
5723         $conf = PVE::QemuConfig->load_config($vmid);
5724
5725         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5726         PVE::QemuConfig->check_lock($conf)
5727             if !($skiplock || $is_backing_up);
5728
5729         die "cannot suspend to disk during backup\n"
5730             if $is_backing_up && $includestate;
5731
5732         if ($includestate) {
5733             $conf->{lock} = 'suspending';
5734             my $date = strftime("%Y-%m-%d", localtime(time()));
5735             $storecfg = PVE::Storage::config();
5736             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5737             $path = PVE::Storage::path($storecfg, $vmstate);
5738             PVE::QemuConfig->write_config($vmid, $conf);
5739         } else {
5740             vm_mon_cmd($vmid, "stop");
5741         }
5742     });
5743
5744     if ($includestate) {
5745         # save vm state
5746         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5747
5748         eval {
5749             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5750             for(;;) {
5751                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5752                 if (!$state->{status}) {
5753                     die "savevm not active\n";
5754                 } elsif ($state->{status} eq 'active') {
5755                     sleep(1);
5756                     next;
5757                 } elsif ($state->{status} eq 'completed') {
5758                     print "State saved, quitting\n";
5759                     last;
5760                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5761                     die "query-savevm failed with error '$state->{error}'\n"
5762                 } else {
5763                     die "query-savevm returned status '$state->{status}'\n";
5764                 }
5765             }
5766         };
5767         my $err = $@;
5768
5769         PVE::QemuConfig->lock_config($vmid, sub {
5770             $conf = PVE::QemuConfig->load_config($vmid);
5771             if ($err) {
5772                 # cleanup, but leave suspending lock, to indicate something went wrong
5773                 eval {
5774                     vm_mon_cmd($vmid, "savevm-end");
5775                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5776                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5777                     delete $conf->@{qw(vmstate runningmachine)};
5778                     PVE::QemuConfig->write_config($vmid, $conf);
5779                 };
5780                 warn $@ if $@;
5781                 die $err;
5782             }
5783
5784             die "lock changed unexpectedly\n"
5785                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5786
5787             vm_qmp_command($vmid, { execute => "quit" });
5788             $conf->{lock} = 'suspended';
5789             PVE::QemuConfig->write_config($vmid, $conf);
5790         });
5791     }
5792 }
5793
5794 sub vm_resume {
5795     my ($vmid, $skiplock, $nocheck) = @_;
5796
5797     PVE::QemuConfig->lock_config($vmid, sub {
5798         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5799         my $res = $vm_mon_cmd->($vmid, 'query-status');
5800         my $resume_cmd = 'cont';
5801
5802         if ($res->{status} && $res->{status} eq 'suspended') {
5803             $resume_cmd = 'system_wakeup';
5804         }
5805
5806         if (!$nocheck) {
5807
5808             my $conf = PVE::QemuConfig->load_config($vmid);
5809
5810             PVE::QemuConfig->check_lock($conf)
5811                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5812         }
5813
5814         $vm_mon_cmd->($vmid, $resume_cmd);
5815     });
5816 }
5817
5818 sub vm_sendkey {
5819     my ($vmid, $skiplock, $key) = @_;
5820
5821     PVE::QemuConfig->lock_config($vmid, sub {
5822
5823         my $conf = PVE::QemuConfig->load_config($vmid);
5824
5825         # there is no qmp command, so we use the human monitor command
5826         vm_human_monitor_command($vmid, "sendkey $key");
5827     });
5828 }
5829
5830 sub vm_destroy {
5831     my ($storecfg, $vmid, $skiplock) = @_;
5832
5833     PVE::QemuConfig->lock_config($vmid, sub {
5834
5835         my $conf = PVE::QemuConfig->load_config($vmid);
5836
5837         if (!check_running($vmid)) {
5838             destroy_vm($storecfg, $vmid, undef, $skiplock);
5839         } else {
5840             die "VM $vmid is running - destroy failed\n";
5841         }
5842     });
5843 }
5844
5845 # vzdump restore implementaion
5846
5847 sub tar_archive_read_firstfile {
5848     my $archive = shift;
5849
5850     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5851
5852     # try to detect archive type first
5853     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5854         die "unable to open file '$archive'\n";
5855     my $firstfile = <$fh>;
5856     kill 15, $pid;
5857     close $fh;
5858
5859     die "ERROR: archive contaions no data\n" if !$firstfile;
5860     chomp $firstfile;
5861
5862     return $firstfile;
5863 }
5864
5865 sub tar_restore_cleanup {
5866     my ($storecfg, $statfile) = @_;
5867
5868     print STDERR "starting cleanup\n";
5869
5870     if (my $fd = IO::File->new($statfile, "r")) {
5871         while (defined(my $line = <$fd>)) {
5872             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5873                 my $volid = $2;
5874                 eval {
5875                     if ($volid =~ m|^/|) {
5876                         unlink $volid || die 'unlink failed\n';
5877                     } else {
5878                         PVE::Storage::vdisk_free($storecfg, $volid);
5879                     }
5880                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5881                 };
5882                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5883             } else {
5884                 print STDERR "unable to parse line in statfile - $line";
5885             }
5886         }
5887         $fd->close();
5888     }
5889 }
5890
5891 sub restore_archive {
5892     my ($archive, $vmid, $user, $opts) = @_;
5893
5894     my $format = $opts->{format};
5895     my $comp;
5896
5897     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5898         $format = 'tar' if !$format;
5899         $comp = 'gzip';
5900     } elsif ($archive =~ m/\.tar$/) {
5901         $format = 'tar' if !$format;
5902     } elsif ($archive =~ m/.tar.lzo$/) {
5903         $format = 'tar' if !$format;
5904         $comp = 'lzop';
5905     } elsif ($archive =~ m/\.vma$/) {
5906         $format = 'vma' if !$format;
5907     } elsif ($archive =~ m/\.vma\.gz$/) {
5908         $format = 'vma' if !$format;
5909         $comp = 'gzip';
5910     } elsif ($archive =~ m/\.vma\.lzo$/) {
5911         $format = 'vma' if !$format;
5912         $comp = 'lzop';
5913     } else {
5914         $format = 'vma' if !$format; # default
5915     }
5916
5917     # try to detect archive format
5918     if ($format eq 'tar') {
5919         return restore_tar_archive($archive, $vmid, $user, $opts);
5920     } else {
5921         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5922     }
5923 }
5924
5925 sub restore_update_config_line {
5926     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5927
5928     return if $line =~ m/^\#qmdump\#/;
5929     return if $line =~ m/^\#vzdump\#/;
5930     return if $line =~ m/^lock:/;
5931     return if $line =~ m/^unused\d+:/;
5932     return if $line =~ m/^parent:/;
5933
5934     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5935     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5936         # try to convert old 1.X settings
5937         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5938         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5939             my ($model, $macaddr) = split(/\=/, $devconfig);
5940             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5941             my $net = {
5942                 model => $model,
5943                 bridge => "vmbr$ind",
5944                 macaddr => $macaddr,
5945             };
5946             my $netstr = print_net($net);
5947
5948             print $outfd "net$cookie->{netcount}: $netstr\n";
5949             $cookie->{netcount}++;
5950         }
5951     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5952         my ($id, $netstr) = ($1, $2);
5953         my $net = parse_net($netstr);
5954         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5955         $netstr = print_net($net);
5956         print $outfd "$id: $netstr\n";
5957     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5958         my $virtdev = $1;
5959         my $value = $3;
5960         my $di = parse_drive($virtdev, $value);
5961         if (defined($di->{backup}) && !$di->{backup}) {
5962             print $outfd "#$line";
5963         } elsif ($map->{$virtdev}) {
5964             delete $di->{format}; # format can change on restore
5965             $di->{file} = $map->{$virtdev};
5966             $value = print_drive($vmid, $di);
5967             print $outfd "$virtdev: $value\n";
5968         } else {
5969             print $outfd $line;
5970         }
5971     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5972         my $vmgenid = $1;
5973         if ($vmgenid ne '0') {
5974             # always generate a new vmgenid if there was a valid one setup
5975             $vmgenid = generate_uuid();
5976         }
5977         print $outfd "vmgenid: $vmgenid\n";
5978     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5979         my ($uuid, $uuid_str);
5980         UUID::generate($uuid);
5981         UUID::unparse($uuid, $uuid_str);
5982         my $smbios1 = parse_smbios1($2);
5983         $smbios1->{uuid} = $uuid_str;
5984         print $outfd $1.print_smbios1($smbios1)."\n";
5985     } else {
5986         print $outfd $line;
5987     }
5988 }
5989
5990 sub scan_volids {
5991     my ($cfg, $vmid) = @_;
5992
5993     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5994
5995     my $volid_hash = {};
5996     foreach my $storeid (keys %$info) {
5997         foreach my $item (@{$info->{$storeid}}) {
5998             next if !($item->{volid} && $item->{size});
5999             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6000             $volid_hash->{$item->{volid}} = $item;
6001         }
6002     }
6003
6004     return $volid_hash;
6005 }
6006
6007 sub is_volume_in_use {
6008     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6009
6010     my $path = PVE::Storage::path($storecfg, $volid);
6011
6012     my $scan_config = sub {
6013         my ($cref, $snapname) = @_;
6014
6015         foreach my $key (keys %$cref) {
6016             my $value = $cref->{$key};
6017             if (is_valid_drivename($key)) {
6018                 next if $skip_drive && $key eq $skip_drive;
6019                 my $drive = parse_drive($key, $value);
6020                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6021                 return 1 if $volid eq $drive->{file};
6022                 if ($drive->{file} =~ m!^/!) {
6023                     return 1 if $drive->{file} eq $path;
6024                 } else {
6025                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6026                     next if !$storeid;
6027                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6028                     next if !$scfg;
6029                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6030                 }
6031             }
6032         }
6033
6034         return 0;
6035     };
6036
6037     return 1 if &$scan_config($conf);
6038
6039     undef $skip_drive;
6040
6041     foreach my $snapname (keys %{$conf->{snapshots}}) {
6042         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6043     }
6044
6045     return 0;
6046 }
6047
6048 sub update_disksize {
6049     my ($vmid, $conf, $volid_hash) = @_;
6050
6051     my $changes;
6052     my $prefix = "VM $vmid:";
6053
6054     # used and unused disks
6055     my $referenced = {};
6056
6057     # Note: it is allowed to define multiple storages with same path (alias), so
6058     # we need to check both 'volid' and real 'path' (two different volid can point
6059     # to the same path).
6060
6061     my $referencedpath = {};
6062
6063     # update size info
6064     foreach my $opt (keys %$conf) {
6065         if (is_valid_drivename($opt)) {
6066             my $drive = parse_drive($opt, $conf->{$opt});
6067             my $volid = $drive->{file};
6068             next if !$volid;
6069
6070             $referenced->{$volid} = 1;
6071             if ($volid_hash->{$volid} &&
6072                 (my $path = $volid_hash->{$volid}->{path})) {
6073                 $referencedpath->{$path} = 1;
6074             }
6075
6076             next if drive_is_cdrom($drive);
6077             next if !$volid_hash->{$volid};
6078
6079             $drive->{size} = $volid_hash->{$volid}->{size};
6080             my $new = print_drive($vmid, $drive);
6081             if ($new ne $conf->{$opt}) {
6082                 $changes = 1;
6083                 $conf->{$opt} = $new;
6084                 print "$prefix update disk '$opt' information.\n";
6085             }
6086         }
6087     }
6088
6089     # remove 'unusedX' entry if volume is used
6090     foreach my $opt (keys %$conf) {
6091         next if $opt !~ m/^unused\d+$/;
6092         my $volid = $conf->{$opt};
6093         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6094         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6095             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6096             $changes = 1;
6097             delete $conf->{$opt};
6098         }
6099
6100         $referenced->{$volid} = 1;
6101         $referencedpath->{$path} = 1 if $path;
6102     }
6103
6104     foreach my $volid (sort keys %$volid_hash) {
6105         next if $volid =~ m/vm-$vmid-state-/;
6106         next if $referenced->{$volid};
6107         my $path = $volid_hash->{$volid}->{path};
6108         next if !$path; # just to be sure
6109         next if $referencedpath->{$path};
6110         $changes = 1;
6111         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6112         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6113         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6114     }
6115
6116     return $changes;
6117 }
6118
6119 sub rescan {
6120     my ($vmid, $nolock, $dryrun) = @_;
6121
6122     my $cfg = PVE::Storage::config();
6123
6124     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6125     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6126     foreach my $stor (keys %{$cfg->{ids}}) {
6127         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6128     }
6129
6130     print "rescan volumes...\n";
6131     my $volid_hash = scan_volids($cfg, $vmid);
6132
6133     my $updatefn =  sub {
6134         my ($vmid) = @_;
6135
6136         my $conf = PVE::QemuConfig->load_config($vmid);
6137
6138         PVE::QemuConfig->check_lock($conf);
6139
6140         my $vm_volids = {};
6141         foreach my $volid (keys %$volid_hash) {
6142             my $info = $volid_hash->{$volid};
6143             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6144         }
6145
6146         my $changes = update_disksize($vmid, $conf, $vm_volids);
6147
6148         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6149     };
6150
6151     if (defined($vmid)) {
6152         if ($nolock) {
6153             &$updatefn($vmid);
6154         } else {
6155             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6156         }
6157     } else {
6158         my $vmlist = config_list();
6159         foreach my $vmid (keys %$vmlist) {
6160             if ($nolock) {
6161                 &$updatefn($vmid);
6162             } else {
6163                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6164             }
6165         }
6166     }
6167 }
6168
6169 sub restore_vma_archive {
6170     my ($archive, $vmid, $user, $opts, $comp) = @_;
6171
6172     my $readfrom = $archive;
6173
6174     my $cfg = PVE::Storage::config();
6175     my $commands = [];
6176     my $bwlimit = $opts->{bwlimit};
6177
6178     my $dbg_cmdstring = '';
6179     my $add_pipe = sub {
6180         my ($cmd) = @_;
6181         push @$commands, $cmd;
6182         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6183         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6184         $readfrom = '-';
6185     };
6186
6187     my $input = undef;
6188     if ($archive eq '-') {
6189         $input = '<&STDIN';
6190     } else {
6191         # If we use a backup from a PVE defined storage we also consider that
6192         # storage's rate limit:
6193         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6194         if (defined($volid)) {
6195             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6196             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6197             if ($readlimit) {
6198                 print STDERR "applying read rate limit: $readlimit\n";
6199                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6200                 $add_pipe->($cstream);
6201             }
6202         }
6203     }
6204
6205     if ($comp) {
6206         my $cmd;
6207         if ($comp eq 'gzip') {
6208             $cmd = ['zcat', $readfrom];
6209         } elsif ($comp eq 'lzop') {
6210             $cmd = ['lzop', '-d', '-c', $readfrom];
6211         } else {
6212             die "unknown compression method '$comp'\n";
6213         }
6214         $add_pipe->($cmd);
6215     }
6216
6217     my $tmpdir = "/var/tmp/vzdumptmp$$";
6218     rmtree $tmpdir;
6219
6220     # disable interrupts (always do cleanups)
6221     local $SIG{INT} =
6222         local $SIG{TERM} =
6223         local $SIG{QUIT} =
6224         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6225
6226     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6227     POSIX::mkfifo($mapfifo, 0600);
6228     my $fifofh;
6229
6230     my $openfifo = sub {
6231         open($fifofh, '>', $mapfifo) || die $!;
6232     };
6233
6234     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6235
6236     my $oldtimeout;
6237     my $timeout = 5;
6238
6239     my $devinfo = {};
6240
6241     my $rpcenv = PVE::RPCEnvironment::get();
6242
6243     my $conffile = PVE::QemuConfig->config_file($vmid);
6244     my $tmpfn = "$conffile.$$.tmp";
6245
6246     # Note: $oldconf is undef if VM does not exists
6247     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6248     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6249
6250     my %storage_limits;
6251
6252     my $print_devmap = sub {
6253         my $virtdev_hash = {};
6254
6255         my $cfgfn = "$tmpdir/qemu-server.conf";
6256
6257         # we can read the config - that is already extracted
6258         my $fh = IO::File->new($cfgfn, "r") ||
6259             "unable to read qemu-server.conf - $!\n";
6260
6261         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6262         if (-f $fwcfgfn) {
6263             my $pve_firewall_dir = '/etc/pve/firewall';
6264             mkdir $pve_firewall_dir; # make sure the dir exists
6265             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6266         }
6267
6268         while (defined(my $line = <$fh>)) {
6269             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6270                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6271                 die "archive does not contain data for drive '$virtdev'\n"
6272                     if !$devinfo->{$devname};
6273                 if (defined($opts->{storage})) {
6274                     $storeid = $opts->{storage} || 'local';
6275                 } elsif (!$storeid) {
6276                     $storeid = 'local';
6277                 }
6278                 $format = 'raw' if !$format;
6279                 $devinfo->{$devname}->{devname} = $devname;
6280                 $devinfo->{$devname}->{virtdev} = $virtdev;
6281                 $devinfo->{$devname}->{format} = $format;
6282                 $devinfo->{$devname}->{storeid} = $storeid;
6283
6284                 # check permission on storage
6285                 my $pool = $opts->{pool}; # todo: do we need that?
6286                 if ($user ne 'root@pam') {
6287                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6288                 }
6289
6290                 $storage_limits{$storeid} = $bwlimit;
6291
6292                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6293             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6294                 my $virtdev = $1;
6295                 my $drive = parse_drive($virtdev, $2);
6296                 if (drive_is_cloudinit($drive)) {
6297                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6298                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6299                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6300
6301                     my $d = {
6302                         format => $format,
6303                         storeid => $opts->{storage} // $storeid,
6304                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6305                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6306                         name => "vm-$vmid-cloudinit",
6307                         is_cloudinit => 1,
6308                     };
6309                     $virtdev_hash->{$virtdev} = $d;
6310                 }
6311             }
6312         }
6313
6314         foreach my $key (keys %storage_limits) {
6315             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6316             next if !$limit;
6317             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6318             $storage_limits{$key} = $limit * 1024;
6319         }
6320
6321         foreach my $devname (keys %$devinfo) {
6322             die "found no device mapping information for device '$devname'\n"
6323                 if !$devinfo->{$devname}->{virtdev};
6324         }
6325
6326         # create empty/temp config
6327         if ($oldconf) {
6328             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6329             foreach_drive($oldconf, sub {
6330                 my ($ds, $drive) = @_;
6331
6332                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6333
6334                 my $volid = $drive->{file};
6335                 return if !$volid || $volid =~ m|^/|;
6336
6337                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6338                 return if !$path || !$owner || ($owner != $vmid);
6339
6340                 # Note: only delete disk we want to restore
6341                 # other volumes will become unused
6342                 if ($virtdev_hash->{$ds}) {
6343                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6344                     if (my $err = $@) {
6345                         warn $err;
6346                     }
6347                 }
6348             });
6349
6350             # delete vmstate files, after the restore we have no snapshots anymore
6351             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6352                 my $snap = $oldconf->{snapshots}->{$snapname};
6353                 if ($snap->{vmstate}) {
6354                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6355                     if (my $err = $@) {
6356                         warn $err;
6357                     }
6358                 }
6359             }
6360         }
6361
6362         my $map = {};
6363         foreach my $virtdev (sort keys %$virtdev_hash) {
6364             my $d = $virtdev_hash->{$virtdev};
6365             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6366             my $storeid = $d->{storeid};
6367             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6368
6369             my $map_opts = '';
6370             if (my $limit = $storage_limits{$storeid}) {
6371                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6372             }
6373
6374             # test if requested format is supported
6375             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6376             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6377             $d->{format} = $defFormat if !$supported;
6378
6379             my $name;
6380             if ($d->{is_cloudinit}) {
6381                 $name = $d->{name};
6382                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6383             }
6384
6385             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6386             print STDERR "new volume ID is '$volid'\n";
6387             $d->{volid} = $volid;
6388
6389             PVE::Storage::activate_volumes($cfg, [$volid]);
6390
6391             my $write_zeros = 1;
6392             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6393                 $write_zeros = 0;
6394             }
6395
6396             if (!$d->{is_cloudinit}) {
6397                 my $path = PVE::Storage::path($cfg, $volid);
6398
6399                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6400
6401                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6402             }
6403             $map->{$virtdev} = $volid;
6404         }
6405
6406         $fh->seek(0, 0) || die "seek failed - $!\n";
6407
6408         my $outfd = new IO::File ($tmpfn, "w") ||
6409             die "unable to write config for VM $vmid\n";
6410
6411         my $cookie = { netcount => 0 };
6412         while (defined(my $line = <$fh>)) {
6413             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6414         }
6415
6416         $fh->close();
6417         $outfd->close();
6418     };
6419
6420     eval {
6421         # enable interrupts
6422         local $SIG{INT} =
6423             local $SIG{TERM} =
6424             local $SIG{QUIT} =
6425             local $SIG{HUP} =
6426             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6427         local $SIG{ALRM} = sub { die "got timeout\n"; };
6428
6429         $oldtimeout = alarm($timeout);
6430
6431         my $parser = sub {
6432             my $line = shift;
6433
6434             print "$line\n";
6435
6436             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6437                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6438                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6439             } elsif ($line =~ m/^CTIME: /) {
6440                 # we correctly received the vma config, so we can disable
6441                 # the timeout now for disk allocation (set to 10 minutes, so
6442                 # that we always timeout if something goes wrong)
6443                 alarm(600);
6444                 &$print_devmap();
6445                 print $fifofh "done\n";
6446                 my $tmp = $oldtimeout || 0;
6447                 $oldtimeout = undef;
6448                 alarm($tmp);
6449                 close($fifofh);
6450             }
6451         };
6452
6453         print "restore vma archive: $dbg_cmdstring\n";
6454         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6455     };
6456     my $err = $@;
6457
6458     alarm($oldtimeout) if $oldtimeout;
6459
6460     my $vollist = [];
6461     foreach my $devname (keys %$devinfo) {
6462         my $volid = $devinfo->{$devname}->{volid};
6463         push @$vollist, $volid if $volid;
6464     }
6465
6466     PVE::Storage::deactivate_volumes($cfg, $vollist);
6467
6468     unlink $mapfifo;
6469
6470     if ($err) {
6471         rmtree $tmpdir;
6472         unlink $tmpfn;
6473
6474         foreach my $devname (keys %$devinfo) {
6475             my $volid = $devinfo->{$devname}->{volid};
6476             next if !$volid;
6477             eval {
6478                 if ($volid =~ m|^/|) {
6479                     unlink $volid || die 'unlink failed\n';
6480                 } else {
6481                     PVE::Storage::vdisk_free($cfg, $volid);
6482                 }
6483                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6484             };
6485             print STDERR "unable to cleanup '$volid' - $@" if $@;
6486         }
6487         die $err;
6488     }
6489
6490     rmtree $tmpdir;
6491
6492     rename($tmpfn, $conffile) ||
6493         die "unable to commit configuration file '$conffile'\n";
6494
6495     PVE::Cluster::cfs_update(); # make sure we read new file
6496
6497     eval { rescan($vmid, 1); };
6498     warn $@ if $@;
6499 }
6500
6501 sub restore_tar_archive {
6502     my ($archive, $vmid, $user, $opts) = @_;
6503
6504     if ($archive ne '-') {
6505         my $firstfile = tar_archive_read_firstfile($archive);
6506         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6507             if $firstfile ne 'qemu-server.conf';
6508     }
6509
6510     my $storecfg = PVE::Storage::config();
6511
6512     # destroy existing data - keep empty config
6513     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6514     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6515
6516     my $tocmd = "/usr/lib/qemu-server/qmextract";
6517
6518     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6519     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6520     $tocmd .= ' --prealloc' if $opts->{prealloc};
6521     $tocmd .= ' --info' if $opts->{info};
6522
6523     # tar option "xf" does not autodetect compression when read from STDIN,
6524     # so we pipe to zcat
6525     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6526         PVE::Tools::shellquote("--to-command=$tocmd");
6527
6528     my $tmpdir = "/var/tmp/vzdumptmp$$";
6529     mkpath $tmpdir;
6530
6531     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6532     local $ENV{VZDUMP_VMID} = $vmid;
6533     local $ENV{VZDUMP_USER} = $user;
6534
6535     my $conffile = PVE::QemuConfig->config_file($vmid);
6536     my $tmpfn = "$conffile.$$.tmp";
6537
6538     # disable interrupts (always do cleanups)
6539     local $SIG{INT} =
6540         local $SIG{TERM} =
6541         local $SIG{QUIT} =
6542         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6543
6544     eval {
6545         # enable interrupts
6546         local $SIG{INT} =
6547             local $SIG{TERM} =
6548             local $SIG{QUIT} =
6549             local $SIG{HUP} =
6550             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6551
6552         if ($archive eq '-') {
6553             print "extracting archive from STDIN\n";
6554             run_command($cmd, input => "<&STDIN");
6555         } else {
6556             print "extracting archive '$archive'\n";
6557             run_command($cmd);
6558         }
6559
6560         return if $opts->{info};
6561
6562         # read new mapping
6563         my $map = {};
6564         my $statfile = "$tmpdir/qmrestore.stat";
6565         if (my $fd = IO::File->new($statfile, "r")) {
6566             while (defined (my $line = <$fd>)) {
6567                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6568                     $map->{$1} = $2 if $1;
6569                 } else {
6570                     print STDERR "unable to parse line in statfile - $line\n";
6571                 }
6572             }
6573             $fd->close();
6574         }
6575
6576         my $confsrc = "$tmpdir/qemu-server.conf";
6577
6578         my $srcfd = new IO::File($confsrc, "r") ||
6579             die "unable to open file '$confsrc'\n";
6580
6581         my $outfd = new IO::File ($tmpfn, "w") ||
6582             die "unable to write config for VM $vmid\n";
6583
6584         my $cookie = { netcount => 0 };
6585         while (defined (my $line = <$srcfd>)) {
6586             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6587         }
6588
6589         $srcfd->close();
6590         $outfd->close();
6591     };
6592     my $err = $@;
6593
6594     if ($err) {
6595
6596         unlink $tmpfn;
6597
6598         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6599
6600         die $err;
6601     }
6602
6603     rmtree $tmpdir;
6604
6605     rename $tmpfn, $conffile ||
6606         die "unable to commit configuration file '$conffile'\n";
6607
6608     PVE::Cluster::cfs_update(); # make sure we read new file
6609
6610     eval { rescan($vmid, 1); };
6611     warn $@ if $@;
6612 };
6613
6614 sub foreach_storage_used_by_vm {
6615     my ($conf, $func) = @_;
6616
6617     my $sidhash = {};
6618
6619     foreach_drive($conf, sub {
6620         my ($ds, $drive) = @_;
6621         return if drive_is_cdrom($drive);
6622
6623         my $volid = $drive->{file};
6624
6625         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6626         $sidhash->{$sid} = $sid if $sid;
6627     });
6628
6629     foreach my $sid (sort keys %$sidhash) {
6630         &$func($sid);
6631     }
6632 }
6633
6634 sub do_snapshots_with_qemu {
6635     my ($storecfg, $volid) = @_;
6636
6637     my $storage_name = PVE::Storage::parse_volume_id($volid);
6638     my $scfg = $storecfg->{ids}->{$storage_name};
6639
6640     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6641         return 1;
6642     }
6643
6644     if ($volid =~ m/\.(qcow2|qed)$/){
6645         return 1;
6646     }
6647
6648     return undef;
6649 }
6650
6651 sub qga_check_running {
6652     my ($vmid, $nowarn) = @_;
6653
6654     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6655     if ($@) {
6656         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6657         return 0;
6658     }
6659     return 1;
6660 }
6661
6662 sub template_create {
6663     my ($vmid, $conf, $disk) = @_;
6664
6665     my $storecfg = PVE::Storage::config();
6666
6667     foreach_drive($conf, sub {
6668         my ($ds, $drive) = @_;
6669
6670         return if drive_is_cdrom($drive);
6671         return if $disk && $ds ne $disk;
6672
6673         my $volid = $drive->{file};
6674         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6675
6676         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6677         $drive->{file} = $voliddst;
6678         $conf->{$ds} = print_drive($vmid, $drive);
6679         PVE::QemuConfig->write_config($vmid, $conf);
6680     });
6681 }
6682
6683 sub convert_iscsi_path {
6684     my ($path) = @_;
6685
6686     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6687         my $portal = $1;
6688         my $target = $2;
6689         my $lun = $3;
6690
6691         my $initiator_name = get_initiator_name();
6692
6693         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6694                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6695     }
6696
6697     die "cannot convert iscsi path '$path', unkown format\n";
6698 }
6699
6700 sub qemu_img_convert {
6701     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6702
6703     my $storecfg = PVE::Storage::config();
6704     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6705     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6706
6707     if ($src_storeid && $dst_storeid) {
6708
6709         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6710
6711         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6712         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6713
6714         my $src_format = qemu_img_format($src_scfg, $src_volname);
6715         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6716
6717         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6718         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6719
6720         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6721         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6722
6723         my $cmd = [];
6724         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6725         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6726         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6727         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6728
6729         if ($src_is_iscsi) {
6730             push @$cmd, '--image-opts';
6731             $src_path = convert_iscsi_path($src_path);
6732         } else {
6733             push @$cmd, '-f', $src_format;
6734         }
6735
6736         if ($dst_is_iscsi) {
6737             push @$cmd, '--target-image-opts';
6738             $dst_path = convert_iscsi_path($dst_path);
6739         } else {
6740             push @$cmd, '-O', $dst_format;
6741         }
6742
6743         push @$cmd, $src_path;
6744
6745         if (!$dst_is_iscsi && $is_zero_initialized) {
6746             push @$cmd, "zeroinit:$dst_path";
6747         } else {
6748             push @$cmd, $dst_path;
6749         }
6750
6751         my $parser = sub {
6752             my $line = shift;
6753             if($line =~ m/\((\S+)\/100\%\)/){
6754                 my $percent = $1;
6755                 my $transferred = int($size * $percent / 100);
6756                 my $remaining = $size - $transferred;
6757
6758                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6759             }
6760
6761         };
6762
6763         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6764         my $err = $@;
6765         die "copy failed: $err" if $err;
6766     }
6767 }
6768
6769 sub qemu_img_format {
6770     my ($scfg, $volname) = @_;
6771
6772     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6773         return $1;
6774     } else {
6775         return "raw";
6776     }
6777 }
6778
6779 sub qemu_drive_mirror {
6780     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6781
6782     $jobs = {} if !$jobs;
6783
6784     my $qemu_target;
6785     my $format;
6786     $jobs->{"drive-$drive"} = {};
6787
6788     if ($dst_volid =~ /^nbd:/) {
6789         $qemu_target = $dst_volid;
6790         $format = "nbd";
6791     } else {
6792         my $storecfg = PVE::Storage::config();
6793         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6794
6795         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6796
6797         $format = qemu_img_format($dst_scfg, $dst_volname);
6798
6799         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6800
6801         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6802     }
6803
6804     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6805     $opts->{format} = $format if $format;
6806
6807     if (defined($bwlimit)) {
6808         $opts->{speed} = $bwlimit * 1024;
6809         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6810     } else {
6811         print "drive mirror is starting for drive-$drive\n";
6812     }
6813
6814     # if a job already runs for this device we get an error, catch it for cleanup
6815     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6816     if (my $err = $@) {
6817         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6818         warn "$@\n" if $@;
6819         die "mirroring error: $err\n";
6820     }
6821
6822     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6823 }
6824
6825 sub qemu_drive_mirror_monitor {
6826     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6827
6828     eval {
6829         my $err_complete = 0;
6830
6831         while (1) {
6832             die "storage migration timed out\n" if $err_complete > 300;
6833
6834             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6835
6836             my $running_mirror_jobs = {};
6837             foreach my $stat (@$stats) {
6838                 next if $stat->{type} ne 'mirror';
6839                 $running_mirror_jobs->{$stat->{device}} = $stat;
6840             }
6841
6842             my $readycounter = 0;
6843
6844             foreach my $job (keys %$jobs) {
6845
6846                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6847                     print "$job : finished\n";
6848                     delete $jobs->{$job};
6849                     next;
6850                 }
6851
6852                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6853
6854                 my $busy = $running_mirror_jobs->{$job}->{busy};
6855                 my $ready = $running_mirror_jobs->{$job}->{ready};
6856                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6857                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6858                     my $remaining = $total - $transferred;
6859                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6860
6861                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6862                 }
6863
6864                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6865             }
6866
6867             last if scalar(keys %$jobs) == 0;
6868
6869             if ($readycounter == scalar(keys %$jobs)) {
6870                 print "all mirroring jobs are ready \n";
6871                 last if $skipcomplete; #do the complete later
6872
6873                 if ($vmiddst && $vmiddst != $vmid) {
6874                     my $agent_running = $qga && qga_check_running($vmid);
6875                     if ($agent_running) {
6876                         print "freeze filesystem\n";
6877                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6878                     } else {
6879                         print "suspend vm\n";
6880                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6881                     }
6882
6883                     # if we clone a disk for a new target vm, we don't switch the disk
6884                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6885
6886                     if ($agent_running) {
6887                         print "unfreeze filesystem\n";
6888                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6889                     } else {
6890                         print "resume vm\n";
6891                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6892                     }
6893
6894                     last;
6895                 } else {
6896
6897                     foreach my $job (keys %$jobs) {
6898                         # try to switch the disk if source and destination are on the same guest
6899                         print "$job: Completing block job...\n";
6900
6901                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6902                         if ($@ =~ m/cannot be completed/) {
6903                             print "$job: Block job cannot be completed, try again.\n";
6904                             $err_complete++;
6905                         }else {
6906                             print "$job: Completed successfully.\n";
6907                             $jobs->{$job}->{complete} = 1;
6908                         }
6909                     }
6910                 }
6911             }
6912             sleep 1;
6913         }
6914     };
6915     my $err = $@;
6916
6917     if ($err) {
6918         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6919         die "mirroring error: $err";
6920     }
6921
6922 }
6923
6924 sub qemu_blockjobs_cancel {
6925     my ($vmid, $jobs) = @_;
6926
6927     foreach my $job (keys %$jobs) {
6928         print "$job: Cancelling block job\n";
6929         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6930         $jobs->{$job}->{cancel} = 1;
6931     }
6932
6933     while (1) {
6934         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6935
6936         my $running_jobs = {};
6937         foreach my $stat (@$stats) {
6938             $running_jobs->{$stat->{device}} = $stat;
6939         }
6940
6941         foreach my $job (keys %$jobs) {
6942
6943             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6944                 print "$job: Done.\n";
6945                 delete $jobs->{$job};
6946             }
6947         }
6948
6949         last if scalar(keys %$jobs) == 0;
6950
6951         sleep 1;
6952     }
6953 }
6954
6955 sub clone_disk {
6956     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6957         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6958
6959     my $newvolid;
6960
6961     if (!$full) {
6962         print "create linked clone of drive $drivename ($drive->{file})\n";
6963         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6964         push @$newvollist, $newvolid;
6965     } else {
6966
6967         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6968         $storeid = $storage if $storage;
6969
6970         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6971         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6972
6973         print "create full clone of drive $drivename ($drive->{file})\n";
6974         my $name = undef;
6975         if (drive_is_cloudinit($drive)) {
6976             $name = "vm-$newvmid-cloudinit";
6977             $snapname = undef;
6978             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
6979             if ($dst_format ne 'raw') {
6980                 $name .= ".$dst_format";
6981             }
6982         }
6983         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6984         push @$newvollist, $newvolid;
6985
6986         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6987
6988         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6989         if (!$running || $snapname) {
6990             # TODO: handle bwlimits
6991             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6992         } else {
6993
6994             my $kvmver = get_running_qemu_version ($vmid);
6995             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6996                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6997                     if $drive->{iothread};
6998             }
6999
7000             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7001         }
7002     }
7003
7004     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7005
7006     my $disk = $drive;
7007     $disk->{format} = undef;
7008     $disk->{file} = $newvolid;
7009     $disk->{size} = $size;
7010
7011     return $disk;
7012 }
7013
7014 # this only works if VM is running
7015 sub get_current_qemu_machine {
7016     my ($vmid) = @_;
7017
7018     my $cmd = { execute => 'query-machines', arguments => {} };
7019     my $res = vm_qmp_command($vmid, $cmd);
7020
7021     my ($current, $default);
7022     foreach my $e (@$res) {
7023         $default = $e->{name} if $e->{'is-default'};
7024         $current = $e->{name} if $e->{'is-current'};
7025     }
7026
7027     # fallback to the default machine if current is not supported by qemu
7028     return $current || $default || 'pc';
7029 }
7030
7031 sub get_running_qemu_version {
7032     my ($vmid) = @_;
7033     my $cmd = { execute => 'query-version', arguments => {} };
7034     my $res = vm_qmp_command($vmid, $cmd);
7035     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7036 }
7037
7038 sub qemu_machine_feature_enabled {
7039     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7040
7041     my $current_major;
7042     my $current_minor;
7043
7044     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7045
7046         $current_major = $3;
7047         $current_minor = $4;
7048
7049     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7050
7051         $current_major = $1;
7052         $current_minor = $2;
7053     }
7054
7055     return 1 if $current_major > $version_major ||
7056                 ($current_major == $version_major &&
7057                  $current_minor >= $version_minor);
7058 }
7059
7060 sub qemu_machine_pxe {
7061     my ($vmid, $conf, $machine) = @_;
7062
7063     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7064
7065     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7066         $machine .= '.pxe';
7067     }
7068
7069     return $machine;
7070 }
7071
7072 sub qemu_use_old_bios_files {
7073     my ($machine_type) = @_;
7074
7075     return if !$machine_type;
7076
7077     my $use_old_bios_files = undef;
7078
7079     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7080         $machine_type = $1;
7081         $use_old_bios_files = 1;
7082     } else {
7083         my $kvmver = kvm_user_version();
7084         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7085         # load new efi bios files on migration. So this hack is required to allow
7086         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7087         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7088         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7089     }
7090
7091     return ($use_old_bios_files, $machine_type);
7092 }
7093
7094 sub create_efidisk($$$$$) {
7095     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7096
7097     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7098     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7099
7100     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7101     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7102     PVE::Storage::activate_volumes($storecfg, [$volid]);
7103
7104     my $path = PVE::Storage::path($storecfg, $volid);
7105     eval {
7106         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7107     };
7108     die "Copying EFI vars image failed: $@" if $@;
7109
7110     return ($volid, $vars_size);
7111 }
7112
7113 sub vm_iothreads_list {
7114     my ($vmid) = @_;
7115
7116     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7117
7118     my $iothreads = {};
7119     foreach my $iothread (@$res) {
7120         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7121     }
7122
7123     return $iothreads;
7124 }
7125
7126 sub scsihw_infos {
7127     my ($conf, $drive) = @_;
7128
7129     my $maxdev = 0;
7130
7131     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7132         $maxdev = 7;
7133     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7134         $maxdev = 1;
7135     } else {
7136         $maxdev = 256;
7137     }
7138
7139     my $controller = int($drive->{index} / $maxdev);
7140     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7141
7142     return ($maxdev, $controller, $controller_prefix);
7143 }
7144
7145 sub add_hyperv_enlightenments {
7146     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7147
7148     return if $winversion < 6;
7149     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7150
7151     if ($gpu_passthrough || defined($hv_vendor_id)) {
7152         $hv_vendor_id //= 'proxmox';
7153         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7154     }
7155
7156     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7157         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7158         push @$cpuFlags , 'hv_vapic';
7159         push @$cpuFlags , 'hv_time';
7160     } else {
7161         push @$cpuFlags , 'hv_spinlocks=0xffff';
7162     }
7163
7164     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7165         push @$cpuFlags , 'hv_reset';
7166         push @$cpuFlags , 'hv_vpindex';
7167         push @$cpuFlags , 'hv_runtime';
7168     }
7169
7170     if ($winversion >= 7) {
7171         push @$cpuFlags , 'hv_relaxed';
7172
7173         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7174             push @$cpuFlags , 'hv_synic';
7175             push @$cpuFlags , 'hv_stimer';
7176         }
7177
7178         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 0)) {
7179             push @$cpuFlags , 'hv_tlbflush';
7180         }
7181
7182         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7183             push @$cpuFlags , 'hv_ipi';
7184             push @$cpuFlags , 'hv_evmcs';
7185         }
7186     }
7187 }
7188
7189 sub windows_version {
7190     my ($ostype) = @_;
7191
7192     return 0 if !$ostype;
7193
7194     my $winversion = 0;
7195
7196     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7197         $winversion = 5;
7198     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7199         $winversion = 6;
7200     } elsif ($ostype =~ m/^win(\d+)$/) {
7201         $winversion = $1;
7202     }
7203
7204     return $winversion;
7205 }
7206
7207 sub resolve_dst_disk_format {
7208         my ($storecfg, $storeid, $src_volname, $format) = @_;
7209         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7210
7211         if (!$format) {
7212             # if no target format is specified, use the source disk format as hint
7213             if ($src_volname) {
7214                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7215                 $format = qemu_img_format($scfg, $src_volname);
7216             } else {
7217                 return $defFormat;
7218             }
7219         }
7220
7221         # test if requested format is supported - else use default
7222         my $supported = grep { $_ eq $format } @$validFormats;
7223         $format = $defFormat if !$supported;
7224         return $format;
7225 }
7226
7227 sub resolve_first_disk {
7228     my $conf = shift;
7229     my @disks = PVE::QemuServer::valid_drive_names();
7230     my $firstdisk;
7231     foreach my $ds (reverse @disks) {
7232         next if !$conf->{$ds};
7233         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7234         next if PVE::QemuServer::drive_is_cdrom($disk);
7235         $firstdisk = $ds;
7236     }
7237     return $firstdisk;
7238 }
7239
7240 sub generate_uuid {
7241     my ($uuid, $uuid_str);
7242     UUID::generate($uuid);
7243     UUID::unparse($uuid, $uuid_str);
7244     return $uuid_str;
7245 }
7246
7247 sub generate_smbios1_uuid {
7248     return "uuid=".generate_uuid();
7249 }
7250
7251 sub nbd_stop {
7252     my ($vmid) = @_;
7253
7254     vm_mon_cmd($vmid, 'nbd-server-stop');
7255 }
7256
7257 # bash completion helper
7258
7259 sub complete_backup_archives {
7260     my ($cmdname, $pname, $cvalue) = @_;
7261
7262     my $cfg = PVE::Storage::config();
7263
7264     my $storeid;
7265
7266     if ($cvalue =~ m/^([^:]+):/) {
7267         $storeid = $1;
7268     }
7269
7270     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7271
7272     my $res = [];
7273     foreach my $id (keys %$data) {
7274         foreach my $item (@{$data->{$id}}) {
7275             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7276             push @$res, $item->{volid} if defined($item->{volid});
7277         }
7278     }
7279
7280     return $res;
7281 }
7282
7283 my $complete_vmid_full = sub {
7284     my ($running) = @_;
7285
7286     my $idlist = vmstatus();
7287
7288     my $res = [];
7289
7290     foreach my $id (keys %$idlist) {
7291         my $d = $idlist->{$id};
7292         if (defined($running)) {
7293             next if $d->{template};
7294             next if $running && $d->{status} ne 'running';
7295             next if !$running && $d->{status} eq 'running';
7296         }
7297         push @$res, $id;
7298
7299     }
7300     return $res;
7301 };
7302
7303 sub complete_vmid {
7304     return &$complete_vmid_full();
7305 }
7306
7307 sub complete_vmid_stopped {
7308     return &$complete_vmid_full(0);
7309 }
7310
7311 sub complete_vmid_running {
7312     return &$complete_vmid_full(1);
7313 }
7314
7315 sub complete_storage {
7316
7317     my $cfg = PVE::Storage::config();
7318     my $ids = $cfg->{ids};
7319
7320     my $res = [];
7321     foreach my $sid (keys %$ids) {
7322         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7323         next if !$ids->{$sid}->{content}->{images};
7324         push @$res, $sid;
7325     }
7326
7327     return $res;
7328 }
7329
7330 1;