PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     }
 641 };
 642
 643 my $cicustom_fmt = {
 644     meta => {
 645         type => 'string',
 646         optional => 1,
 647         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 648         format => 'pve-volume-id',
 649         format_description => 'volume',
 650     },
 651     network => {
 652         type => 'string',
 653         optional => 1,
 654         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 655         format => 'pve-volume-id',
 656         format_description => 'volume',
 657     },
 658     user => {
 659         type => 'string',
 660         optional => 1,
 661         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 662         format => 'pve-volume-id',
 663         format_description => 'volume',
 664     },
 665 };
 666 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 667
 668 my $confdesc_cloudinit = {
 669     citype => {
 670         optional => 1,
 671         type => 'string',
 672         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 673         enum => ['configdrive2', 'nocloud'],
 674     },
 675     ciuser => {
 676         optional => 1,
 677         type => 'string',
 678         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 679     },
 680     cipassword => {
 681         optional => 1,
 682         type => 'string',
 683         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 684     },
 685     cicustom => {
 686         optional => 1,
 687         type => 'string',
 688         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 689         format => 'pve-qm-cicustom',
 690     },
 691     searchdomain => {
 692         optional => 1,
 693         type => 'string',
 694         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 695     },
 696     nameserver => {
 697         optional => 1,
 698         type => 'string', format => 'address-list',
 699         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 700     },
 701     sshkeys => {
 702         optional => 1,
 703         type => 'string',
 704         format => 'urlencoded',
 705         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 706     },
 707 };
 708
 709 # what about other qemu settings ?
 710 #cpu => 'string',
 711 #machine => 'string',
 712 #fda => 'file',
 713 #fdb => 'file',
 714 #mtdblock => 'file',
 715 #sd => 'file',
 716 #pflash => 'file',
 717 #snapshot => 'bool',
 718 #bootp => 'file',
 719 ##tftp => 'dir',
 720 ##smb => 'dir',
 721 #kernel => 'file',
 722 #append => 'string',
 723 #initrd => 'file',
 724 ##soundhw => 'string',
 725
 726 while (my ($k, $v) = each %$confdesc) {
 727     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 728 }
 729
 730 my $MAX_IDE_DISKS = 4;
 731 my $MAX_SCSI_DISKS = 14;
 732 my $MAX_VIRTIO_DISKS = 16;
 733 my $MAX_SATA_DISKS = 6;
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_UNUSED_DISKS = 256;
 737 my $MAX_HOSTPCI_DEVICES = 4;
 738 my $MAX_SERIAL_PORTS = 4;
 739 my $MAX_PARALLEL_PORTS = 3;
 740 my $MAX_NUMA = 8;
 741
 742 my $numa_fmt = {
 743     cpus => {
 744         type => "string",
 745         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 746         description => "CPUs accessing this NUMA node.",
 747         format_description => "id[-id];...",
 748     },
 749     memory => {
 750         type => "number",
 751         description => "Amount of memory this NUMA node provides.",
 752         optional => 1,
 753     },
 754     hostnodes => {
 755         type => "string",
 756         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 757         description => "Host NUMA nodes to use.",
 758         format_description => "id[-id];...",
 759         optional => 1,
 760     },
 761     policy => {
 762         type => 'string',
 763         enum => [qw(preferred bind interleave)],
 764         description => "NUMA allocation policy.",
 765         optional => 1,
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 769 my $numadesc = {
 770     optional => 1,
 771     type => 'string', format => $numa_fmt,
 772     description => "NUMA topology.",
 773 };
 774 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 775
 776 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 777     $confdesc->{"numa$i"} = $numadesc;
 778 }
 779
 780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 781                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 782                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 784
 785 my $net_fmt_bridge_descr = <<__EOD__;
 786 Bridge to attach the network device to. The Proxmox VE standard bridge
 787 is called 'vmbr0'.
 788
 789 If you do not specify a bridge, we create a kvm user (NATed) network
 790 device, which provides DHCP and DNS services. The following addresses
 791 are used:
 792
 793  10.0.2.2   Gateway
 794  10.0.2.3   DNS Server
 795  10.0.2.4   SMB Server
 796
 797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 798 __EOD__
 799
 800 my $net_fmt = {
 801     macaddr  => get_standard_option('mac-addr', {
 802         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 803     }),
 804     model => {
 805         type => 'string',
 806         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 807         enum => $nic_model_list,
 808         default_key => 1,
 809     },
 810     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 811     bridge => {
 812         type => 'string',
 813         description => $net_fmt_bridge_descr,
 814         format_description => 'bridge',
 815         optional => 1,
 816     },
 817     queues => {
 818         type => 'integer',
 819         minimum => 0, maximum => 16,
 820         description => 'Number of packet queues to be used on the device.',
 821         optional => 1,
 822     },
 823     rate => {
 824         type => 'number',
 825         minimum => 0,
 826         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 827         optional => 1,
 828     },
 829     tag => {
 830         type => 'integer',
 831         minimum => 1, maximum => 4094,
 832         description => 'VLAN tag to apply to packets on this interface.',
 833         optional => 1,
 834     },
 835     trunks => {
 836         type => 'string',
 837         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 838         description => 'VLAN trunks to pass through this interface.',
 839         format_description => 'vlanid[;vlanid...]',
 840         optional => 1,
 841     },
 842     firewall => {
 843         type => 'boolean',
 844         description => 'Whether this interface should be protected by the firewall.',
 845         optional => 1,
 846     },
 847     link_down => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be disconnected (like pulling the plug).',
 850         optional => 1,
 851     },
 852 };
 853
 854 my $netdesc = {
 855     optional => 1,
 856     type => 'string', format => $net_fmt,
 857     description => "Specify network devices.",
 858 };
 859
 860 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 861
 862 my $ipconfig_fmt = {
 863     ip => {
 864         type => 'string',
 865         format => 'pve-ipv4-config',
 866         format_description => 'IPv4Format/CIDR',
 867         description => 'IPv4 address in CIDR format.',
 868         optional => 1,
 869         default => 'dhcp',
 870     },
 871     gw => {
 872         type => 'string',
 873         format => 'ipv4',
 874         format_description => 'GatewayIPv4',
 875         description => 'Default gateway for IPv4 traffic.',
 876         optional => 1,
 877         requires => 'ip',
 878     },
 879     ip6 => {
 880         type => 'string',
 881         format => 'pve-ipv6-config',
 882         format_description => 'IPv6Format/CIDR',
 883         description => 'IPv6 address in CIDR format.',
 884         optional => 1,
 885         default => 'dhcp',
 886     },
 887     gw6 => {
 888         type => 'string',
 889         format => 'ipv6',
 890         format_description => 'GatewayIPv6',
 891         description => 'Default gateway for IPv6 traffic.',
 892         optional => 1,
 893         requires => 'ip6',
 894     },
 895 };
 896 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 897 my $ipconfigdesc = {
 898     optional => 1,
 899     type => 'string', format => 'pve-qm-ipconfig',
 900     description => <<'EODESCR',
 901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 902
 903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 904
 905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 907
 908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 909 EODESCR
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 912
 913 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 914     $confdesc->{"net$i"} = $netdesc;
 915     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 916 }
 917
 918 foreach my $key (keys %$confdesc_cloudinit) {
 919     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 920 }
 921
 922 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 923 sub verify_volume_id_or_qm_path {
 924     my ($volid, $noerr) = @_;
 925
 926     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 927         return $volid;
 928     }
 929
 930     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 931     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 932     if ($@) {
 933         return undef if $noerr;
 934         die $@;
 935     }
 936     return $volid;
 937 }
 938
 939 my $drivename_hash;
 940
 941 my %drivedesc_base = (
 942     volume => { alias => 'file' },
 943     file => {
 944         type => 'string',
 945         format => 'pve-volume-id-or-qm-path',
 946         default_key => 1,
 947         format_description => 'volume',
 948         description => "The drive's backing volume.",
 949     },
 950     media => {
 951         type => 'string',
 952         enum => [qw(cdrom disk)],
 953         description => "The drive's media type.",
 954         default => 'disk',
 955         optional => 1
 956     },
 957     cyls => {
 958         type => 'integer',
 959         description => "Force the drive's physical geometry to have a specific cylinder count.",
 960         optional => 1
 961     },
 962     heads => {
 963         type => 'integer',
 964         description => "Force the drive's physical geometry to have a specific head count.",
 965         optional => 1
 966     },
 967     secs => {
 968         type => 'integer',
 969         description => "Force the drive's physical geometry to have a specific sector count.",
 970         optional => 1
 971     },
 972     trans => {
 973         type => 'string',
 974         enum => [qw(none lba auto)],
 975         description => "Force disk geometry bios translation mode.",
 976         optional => 1,
 977     },
 978     snapshot => {
 979         type => 'boolean',
 980         description => "Controls qemu's snapshot mode feature."
 981             . " If activated, changes made to the disk are temporary and will"
 982             . " be discarded when the VM is shutdown.",
 983         optional => 1,
 984     },
 985     cache => {
 986         type => 'string',
 987         enum => [qw(none writethrough writeback unsafe directsync)],
 988         description => "The drive's cache mode",
 989         optional => 1,
 990     },
 991     format => get_standard_option('pve-qm-image-format'),
 992     size => {
 993         type => 'string',
 994         format => 'disk-size',
 995         format_description => 'DiskSize',
 996         description => "Disk size. This is purely informational and has no effect.",
 997         optional => 1,
 998     },
 999     backup => {
1000         type => 'boolean',
1001         description => "Whether the drive should be included when making backups.",
1002         optional => 1,
1003     },
1004     replicate => {
1005         type => 'boolean',
1006         description => 'Whether the drive should considered for replication jobs.',
1007         optional => 1,
1008         default => 1,
1009     },
1010     rerror => {
1011         type => 'string',
1012         enum => [qw(ignore report stop)],
1013         description => 'Read error action.',
1014         optional => 1,
1015     },
1016     werror => {
1017         type => 'string',
1018         enum => [qw(enospc ignore report stop)],
1019         description => 'Write error action.',
1020         optional => 1,
1021     },
1022     aio => {
1023         type => 'string',
1024         enum => [qw(native threads)],
1025         description => 'AIO type to use.',
1026         optional => 1,
1027     },
1028     discard => {
1029         type => 'string',
1030         enum => [qw(ignore on)],
1031         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1032         optional => 1,
1033     },
1034     detect_zeroes => {
1035         type => 'boolean',
1036         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1037         optional => 1,
1038     },
1039     serial => {
1040         type => 'string',
1041         format => 'urlencoded',
1042         format_description => 'serial',
1043         maxLength => 20*3, # *3 since it's %xx url enoded
1044         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1045         optional => 1,
1046     },
1047     shared => {
1048         type => 'boolean',
1049         description => 'Mark this locally-managed volume as available on all nodes',
1050         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1051         optional => 1,
1052         default => 0,
1053     }
1054 );
1055
1056 my %iothread_fmt = ( iothread => {
1057         type => 'boolean',
1058         description => "Whether to use iothreads for this drive",
1059         optional => 1,
1060 });
1061
1062 my %model_fmt = (
1063     model => {
1064         type => 'string',
1065         format => 'urlencoded',
1066         format_description => 'model',
1067         maxLength => 40*3, # *3 since it's %xx url enoded
1068         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1069         optional => 1,
1070     },
1071 );
1072
1073 my %queues_fmt = (
1074     queues => {
1075         type => 'integer',
1076         description => "Number of queues.",
1077         minimum => 2,
1078         optional => 1
1079     }
1080 );
1081
1082 my %scsiblock_fmt = (
1083     scsiblock => {
1084         type => 'boolean',
1085         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1086         optional => 1,
1087         default => 0,
1088     },
1089 );
1090
1091 my %ssd_fmt = (
1092     ssd => {
1093         type => 'boolean',
1094         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1095         optional => 1,
1096     },
1097 );
1098
1099 my %wwn_fmt = (
1100     wwn => {
1101         type => 'string',
1102         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1103         format_description => 'wwn',
1104         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1105         optional => 1,
1106     },
1107 );
1108
1109 my $add_throttle_desc = sub {
1110     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1111     my $d = {
1112         type => $type,
1113         format_description => $unit,
1114         description => "Maximum $what in $longunit.",
1115         optional => 1,
1116     };
1117     $d->{minimum} = $minimum if defined($minimum);
1118     $drivedesc_base{$key} = $d;
1119 };
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1122 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1123 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1124 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1130
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1138
1139 # burst lengths
1140 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1146
1147 # legacy support
1148 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1152
1153 my $ide_fmt = {
1154     %drivedesc_base,
1155     %model_fmt,
1156     %ssd_fmt,
1157     %wwn_fmt,
1158 };
1159 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1160
1161 my $idedesc = {
1162     optional => 1,
1163     type => 'string', format => $ide_fmt,
1164     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1165 };
1166 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1167
1168 my $scsi_fmt = {
1169     %drivedesc_base,
1170     %iothread_fmt,
1171     %queues_fmt,
1172     %scsiblock_fmt,
1173     %ssd_fmt,
1174     %wwn_fmt,
1175 };
1176 my $scsidesc = {
1177     optional => 1,
1178     type => 'string', format => $scsi_fmt,
1179     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1182
1183 my $sata_fmt = {
1184     %drivedesc_base,
1185     %ssd_fmt,
1186     %wwn_fmt,
1187 };
1188 my $satadesc = {
1189     optional => 1,
1190     type => 'string', format => $sata_fmt,
1191     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1192 };
1193 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1194
1195 my $virtio_fmt = {
1196     %drivedesc_base,
1197     %iothread_fmt,
1198 };
1199 my $virtiodesc = {
1200     optional => 1,
1201     type => 'string', format => $virtio_fmt,
1202     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1205
1206 my $alldrive_fmt = {
1207     %drivedesc_base,
1208     %iothread_fmt,
1209     %model_fmt,
1210     %queues_fmt,
1211     %scsiblock_fmt,
1212     %ssd_fmt,
1213     %wwn_fmt,
1214 };
1215
1216 my $efidisk_fmt = {
1217     volume => { alias => 'file' },
1218     file => {
1219         type => 'string',
1220         format => 'pve-volume-id-or-qm-path',
1221         default_key => 1,
1222         format_description => 'volume',
1223         description => "The drive's backing volume.",
1224     },
1225     format => get_standard_option('pve-qm-image-format'),
1226     size => {
1227         type => 'string',
1228         format => 'disk-size',
1229         format_description => 'DiskSize',
1230         description => "Disk size. This is purely informational and has no effect.",
1231         optional => 1,
1232     },
1233 };
1234
1235 my $efidisk_desc = {
1236     optional => 1,
1237     type => 'string', format => $efidisk_fmt,
1238     description => "Configure a Disk for storing EFI vars",
1239 };
1240
1241 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1242
1243 my $usb_fmt = {
1244     host => {
1245         default_key => 1,
1246         type => 'string', format => 'pve-qm-usb-device',
1247         format_description => 'HOSTUSBDEVICE|spice',
1248         description => <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1250
1251  'bus-port(.port)*' (decimal numbers) or
1252  'vendor_id:product_id' (hexadeciaml numbers) or
1253  'spice'
1254
1255 You can use the 'lsusb -t' command to list existing usb devices.
1256
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1258
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1260 EODESCR
1261     },
1262     usb3 => {
1263         optional => 1,
1264         type => 'boolean',
1265         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1266         default => 0,
1267     },
1268 };
1269
1270 my $usbdesc = {
1271     optional => 1,
1272     type => 'string', format => $usb_fmt,
1273     description => "Configure an USB device (n is 0 to 4).",
1274 };
1275 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1276
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1278 my $hostpci_fmt = {
1279     host => {
1280         default_key => 1,
1281         type => 'string',
1282         pattern => qr/$PCIRE(;$PCIRE)*/,
1283         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1284         description => <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1287
1288 'bus:dev.func' (hexadecimal numbers)
1289
1290 You can us the 'lspci' command to list existing PCI devices.
1291 EODESCR
1292     },
1293     rombar => {
1294         type => 'boolean',
1295         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1296         optional => 1,
1297         default => 1,
1298     },
1299     romfile => {
1300         type => 'string',
1301         pattern => '[^,;]+',
1302         format_description => 'string',
1303         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1304         optional => 1,
1305     },
1306     pcie => {
1307         type => 'boolean',
1308         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1309         optional => 1,
1310         default => 0,
1311     },
1312     'x-vga' => {
1313         type => 'boolean',
1314         description =>  "Enable vfio-vga device support.",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'mdev' => {
1319         type => 'string',
1320         format_description => 'string',
1321         pattern => '[^/\.:]+',
1322         optional => 1,
1323         description => <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1327 EODESCR
1328     }
1329 };
1330 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1331
1332 my $hostpcidesc = {
1333         optional => 1,
1334         type => 'string', format => 'pve-qm-hostpci',
1335         description => "Map host PCI devices into guest.",
1336         verbose_description =>  <<EODESCR,
1337 Map host PCI devices into guest.
1338
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1346
1347 my $serialdesc = {
1348         optional => 1,
1349         type => 'string',
1350         pattern => '(/dev/.+|socket)',
1351         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1352         verbose_description =>  <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1356
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1358
1359 CAUTION: Experimental! User reported problems with this option.
1360 EODESCR
1361 };
1362
1363 my $paralleldesc= {
1364         optional => 1,
1365         type => 'string',
1366         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1367         description =>  "Map host parallel devices (n is 0 to 2).",
1368         verbose_description =>  <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1370
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1372
1373 CAUTION: Experimental! User reported problems with this option.
1374 EODESCR
1375 };
1376
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1378     $confdesc->{"parallel$i"} = $paralleldesc;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1382     $confdesc->{"serial$i"} = $serialdesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1386     $confdesc->{"hostpci$i"} = $hostpcidesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1390     $drivename_hash->{"ide$i"} = 1;
1391     $confdesc->{"ide$i"} = $idedesc;
1392 }
1393
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1395     $drivename_hash->{"sata$i"} = 1;
1396     $confdesc->{"sata$i"} = $satadesc;
1397 }
1398
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1400     $drivename_hash->{"scsi$i"} = 1;
1401     $confdesc->{"scsi$i"} = $scsidesc ;
1402 }
1403
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1405     $drivename_hash->{"virtio$i"} = 1;
1406     $confdesc->{"virtio$i"} = $virtiodesc;
1407 }
1408
1409 $drivename_hash->{efidisk0} = 1;
1410 $confdesc->{efidisk0} = $efidisk_desc;
1411
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1413     $confdesc->{"usb$i"} = $usbdesc;
1414 }
1415
1416 my $unuseddesc = {
1417     optional => 1,
1418     type => 'string', format => 'pve-volume-id',
1419     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1420 };
1421
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1423     $confdesc->{"unused$i"} = $unuseddesc;
1424 }
1425
1426 my $kvm_api_version = 0;
1427
1428 sub kvm_version {
1429     return $kvm_api_version if $kvm_api_version;
1430
1431     open my $fh, '<', '/dev/kvm'
1432         or return undef;
1433
1434     # 0xae00 => KVM_GET_API_VERSION
1435     $kvm_api_version = ioctl($fh, 0xae00, 0);
1436
1437     return $kvm_api_version;
1438 }
1439
1440 my $kvm_user_version;
1441
1442 sub kvm_user_version {
1443
1444     return $kvm_user_version if $kvm_user_version;
1445
1446     $kvm_user_version = 'unknown';
1447
1448     my $code = sub {
1449         my $line = shift;
1450         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451             $kvm_user_version = $2;
1452         }
1453     };
1454
1455     eval { run_command("kvm -version", outfunc => $code); };
1456     warn $@ if $@;
1457
1458     return $kvm_user_version;
1459
1460 }
1461
1462 sub kernel_has_vhost_net {
1463     return -c '/dev/vhost-net';
1464 }
1465
1466 sub valid_drive_names {
1467     # order is important - used to autoselect boot disk
1468     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1472             'efidisk0');
1473 }
1474
1475 sub is_valid_drivename {
1476     my $dev = shift;
1477
1478     return defined($drivename_hash->{$dev});
1479 }
1480
1481 sub option_exists {
1482     my $key = shift;
1483     return defined($confdesc->{$key});
1484 }
1485
1486 sub nic_models {
1487     return $nic_model_list;
1488 }
1489
1490 sub os_list_description {
1491
1492     return {
1493         other => 'Other',
1494         wxp => 'Windows XP',
1495         w2k => 'Windows 2000',
1496         w2k3 =>, 'Windows 2003',
1497         w2k8 => 'Windows 2008',
1498         wvista => 'Windows Vista',
1499         win7 => 'Windows 7',
1500         win8 => 'Windows 8/2012',
1501         win10 => 'Windows 10/2016',
1502         l24 => 'Linux 2.4',
1503         l26 => 'Linux 2.6',
1504     };
1505 }
1506
1507 my $cdrom_path;
1508
1509 sub get_cdrom_path {
1510
1511     return  $cdrom_path if $cdrom_path;
1512
1513     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1514     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1515     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1516 }
1517
1518 sub get_iso_path {
1519     my ($storecfg, $vmid, $cdrom) = @_;
1520
1521     if ($cdrom eq 'cdrom') {
1522         return get_cdrom_path();
1523     } elsif ($cdrom eq 'none') {
1524         return '';
1525     } elsif ($cdrom =~ m|^/|) {
1526         return $cdrom;
1527     } else {
1528         return PVE::Storage::path($storecfg, $cdrom);
1529     }
1530 }
1531
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id {
1534     my ($vmid, $file, $media) = @_;
1535
1536      if (!($file eq 'none' || $file eq 'cdrom' ||
1537           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1538
1539         return undef if $file =~ m|/|;
1540
1541         if ($media && $media eq 'cdrom') {
1542             $file = "local:iso/$file";
1543         } else {
1544             $file = "local:$vmid/$file";
1545         }
1546     }
1547
1548     return $file;
1549 }
1550
1551 sub verify_media_type {
1552     my ($opt, $vtype, $media) = @_;
1553
1554     return if !$media;
1555
1556     my $etype;
1557     if ($media eq 'disk') {
1558         $etype = 'images';
1559     } elsif ($media eq 'cdrom') {
1560         $etype = 'iso';
1561     } else {
1562         die "internal error";
1563     }
1564
1565     return if ($vtype eq $etype);
1566
1567     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1568 }
1569
1570 sub cleanup_drive_path {
1571     my ($opt, $storecfg, $drive) = @_;
1572
1573     # try to convert filesystem paths to volume IDs
1574
1575     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1576         ($drive->{file} !~ m|^/dev/.+|) &&
1577         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1578         ($drive->{file} !~ m/^\d+$/)) {
1579         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1580         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1582         verify_media_type($opt, $vtype, $drive->{media});
1583         $drive->{file} = $volid;
1584     }
1585
1586     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1587 }
1588
1589 sub parse_hotplug_features {
1590     my ($data) = @_;
1591
1592     my $res = {};
1593
1594     return $res if $data eq '0';
1595
1596     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1597
1598     foreach my $feature (PVE::Tools::split_list($data)) {
1599         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1600             $res->{$1} = 1;
1601         } else {
1602             die "invalid hotplug feature '$feature'\n";
1603         }
1604     }
1605     return $res;
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1609 sub pve_verify_hotplug_features {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_hotplug_features($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse hotplug option\n";
1617 }
1618
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 #        [,iothread=on][,serial=serial][,model=model]
1624
1625 sub parse_drive {
1626     my ($key, $data) = @_;
1627
1628     my ($interface, $index);
1629
1630     if ($key =~ m/^([^\d]+)(\d+)$/) {
1631         $interface = $1;
1632         $index = $2;
1633     } else {
1634         return undef;
1635     }
1636
1637     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1638                                      : $confdesc->{$key}->{format};
1639     if (!$desc) {
1640         warn "invalid drive key: $key\n";
1641         return undef;
1642     }
1643     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1644     return undef if !$res;
1645     $res->{interface} = $interface;
1646     $res->{index} = $index;
1647
1648     my $error = 0;
1649     foreach my $opt (qw(bps bps_rd bps_wr)) {
1650         if (my $bps = defined(delete $res->{$opt})) {
1651             if (defined($res->{"m$opt"})) {
1652                 warn "both $opt and m$opt specified\n";
1653                 ++$error;
1654                 next;
1655             }
1656             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1657         }
1658     }
1659
1660     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661     for my $requirement (
1662         [mbps_max => 'mbps'],
1663         [mbps_rd_max => 'mbps_rd'],
1664         [mbps_wr_max => 'mbps_wr'],
1665         [miops_max => 'miops'],
1666         [miops_rd_max => 'miops_rd'],
1667         [miops_wr_max => 'miops_wr'],
1668         [bps_max_length => 'mbps_max'],
1669         [bps_rd_max_length => 'mbps_rd_max'],
1670         [bps_wr_max_length => 'mbps_wr_max'],
1671         [iops_max_length => 'iops_max'],
1672         [iops_rd_max_length => 'iops_rd_max'],
1673         [iops_wr_max_length => 'iops_wr_max']) {
1674         my ($option, $requires) = @$requirement;
1675         if ($res->{$option} && !$res->{$requires}) {
1676             warn "$option requires $requires\n";
1677             ++$error;
1678         }
1679     }
1680
1681     return undef if $error;
1682
1683     return undef if $res->{mbps_rd} && $res->{mbps};
1684     return undef if $res->{mbps_wr} && $res->{mbps};
1685     return undef if $res->{iops_rd} && $res->{iops};
1686     return undef if $res->{iops_wr} && $res->{iops};
1687
1688     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1689         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1690         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1691         return undef if $res->{interface} eq 'virtio';
1692     }
1693
1694     if (my $size = $res->{size}) {
1695         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1696     }
1697
1698     return $res;
1699 }
1700
1701 sub print_drive {
1702     my ($vmid, $drive) = @_;
1703     my $data = { %$drive };
1704     delete $data->{$_} for qw(index interface);
1705     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1706 }
1707
1708 sub scsi_inquiry {
1709     my($fh, $noerr) = @_;
1710
1711     my $SG_IO = 0x2285;
1712     my $SG_GET_VERSION_NUM = 0x2282;
1713
1714     my $versionbuf = "\x00" x 8;
1715     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1716     if (!$ret) {
1717         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1718         return undef;
1719     }
1720     my $version = unpack("I", $versionbuf);
1721     if ($version < 30000) {
1722         die "scsi generic interface too old\n"  if !$noerr;
1723         return undef;
1724     }
1725
1726     my $buf = "\x00" x 36;
1727     my $sensebuf = "\x00" x 8;
1728     my $cmd = pack("C x3 C x1", 0x12, 36);
1729
1730     # see /usr/include/scsi/sg.h
1731     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1732
1733     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734                       length($sensebuf), 0, length($buf), $buf,
1735                       $cmd, $sensebuf, 6000);
1736
1737     $ret = ioctl($fh, $SG_IO, $packet);
1738     if (!$ret) {
1739         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1740         return undef;
1741     }
1742
1743     my @res = unpack($sg_io_hdr_t, $packet);
1744     if ($res[17] || $res[18]) {
1745         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my $res = {};
1750     (my $byte0, my $byte1, $res->{vendor},
1751      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1752
1753     $res->{removable} = $byte1 & 128 ? 1 : 0;
1754     $res->{type} = $byte0 & 31;
1755
1756     return $res;
1757 }
1758
1759 sub path_is_scsi {
1760     my ($path) = @_;
1761
1762     my $fh = IO::File->new("+<$path") || return undef;
1763     my $res = scsi_inquiry($fh, 1);
1764     close($fh);
1765
1766     return $res;
1767 }
1768
1769 sub machine_type_is_q35 {
1770     my ($conf) = @_;
1771
1772     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1773 }
1774
1775 sub print_tabletdevice_full {
1776     my ($conf, $arch) = @_;
1777
1778     my $q35 = machine_type_is_q35($conf);
1779
1780     # we use uhci for old VMs because tablet driver was buggy in older qemu
1781     my $usbbus;
1782     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1783         $usbbus = 'ehci';
1784     } else {
1785         $usbbus = 'uhci';
1786     }
1787
1788     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1789 }
1790
1791 sub print_keyboarddevice_full {
1792     my ($conf, $arch, $machine) = @_;
1793
1794     return undef if $arch ne 'aarch64';
1795
1796     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1797 }
1798
1799 sub print_drivedevice_full {
1800     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1801
1802     my $device = '';
1803     my $maxdev = 0;
1804
1805     if ($drive->{interface} eq 'virtio') {
1806         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1809     } elsif ($drive->{interface} eq 'scsi') {
1810
1811         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1812         my $unit = $drive->{index} % $maxdev;
1813         my $devicetype = 'hd';
1814         my $path = '';
1815         if (drive_is_cdrom($drive)) {
1816             $devicetype = 'cd';
1817         } else {
1818             if ($drive->{file} =~ m|^/|) {
1819                 $path = $drive->{file};
1820                 if (my $info = path_is_scsi($path)) {
1821                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1822                         $devicetype = 'block';
1823                     } elsif ($info->{type} == 1) { # tape
1824                         $devicetype = 'generic';
1825                     }
1826                 }
1827             } else {
1828                  $path = PVE::Storage::path($storecfg, $drive->{file});
1829             }
1830
1831             if($path =~ m/^iscsi\:\/\//){
1832                 $devicetype = 'generic';
1833             }
1834         }
1835
1836         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1837             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1838         } else {
1839             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1840         }
1841
1842         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843             $device .= ",rotation_rate=1";
1844         }
1845         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1846
1847     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1848         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1849         my $controller = int($drive->{index} / $maxdev);
1850         my $unit = $drive->{index} % $maxdev;
1851         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1852
1853         $device = "ide-$devicetype";
1854         if ($drive->{interface} eq 'ide') {
1855             $device .= ",bus=ide.$controller,unit=$unit";
1856         } else {
1857             $device .= ",bus=ahci$controller.$unit";
1858         }
1859         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1860
1861         if ($devicetype eq 'hd') {
1862             if (my $model = $drive->{model}) {
1863                 $model = URI::Escape::uri_unescape($model);
1864                 $device .= ",model=$model";
1865             }
1866             if ($drive->{ssd}) {
1867                 $device .= ",rotation_rate=1";
1868             }
1869         }
1870         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1871     } elsif ($drive->{interface} eq 'usb') {
1872         die "implement me";
1873         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1874     } else {
1875         die "unsupported interface type";
1876     }
1877
1878     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1879
1880     if (my $serial = $drive->{serial}) {
1881         $serial = URI::Escape::uri_unescape($serial);
1882         $device .= ",serial=$serial";
1883     }
1884
1885
1886     return $device;
1887 }
1888
1889 sub get_initiator_name {
1890     my $initiator;
1891
1892     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893     while (defined(my $line = <$fh>)) {
1894         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1895         $initiator = $1;
1896         last;
1897     }
1898     $fh->close();
1899
1900     return $initiator;
1901 }
1902
1903 sub print_drive_full {
1904     my ($storecfg, $vmid, $drive) = @_;
1905
1906     my $path;
1907     my $volid = $drive->{file};
1908     my $format;
1909
1910     if (drive_is_cdrom($drive)) {
1911         $path = get_iso_path($storecfg, $vmid, $volid);
1912     } else {
1913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1914         if ($storeid) {
1915             $path = PVE::Storage::path($storecfg, $volid);
1916             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1917             $format = qemu_img_format($scfg, $volname);
1918         } else {
1919             $path = $volid;
1920             $format = "raw";
1921         }
1922    }
1923
1924     my $opts = '';
1925     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926     foreach my $o (@qemu_drive_options) {
1927         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1928     }
1929
1930     # snapshot only accepts on|off
1931     if (defined($drive->{snapshot})) {
1932         my $v = $drive->{snapshot} ? 'on' : 'off';
1933         $opts .= ",snapshot=$v";
1934     }
1935
1936     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1937         my ($dir, $qmpname) = @$type;
1938         if (my $v = $drive->{"mbps$dir"}) {
1939             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1940         }
1941         if (my $v = $drive->{"mbps${dir}_max"}) {
1942             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1943         }
1944         if (my $v = $drive->{"bps${dir}_max_length"}) {
1945             $opts .= ",throttling.bps$qmpname-max-length=$v";
1946         }
1947         if (my $v = $drive->{"iops${dir}"}) {
1948             $opts .= ",throttling.iops$qmpname=$v";
1949         }
1950         if (my $v = $drive->{"iops${dir}_max"}) {
1951             $opts .= ",throttling.iops$qmpname-max=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}_max_length"}) {
1954             $opts .= ",throttling.iops$qmpname-max-length=$v";
1955         }
1956     }
1957
1958     $opts .= ",format=$format" if $format && !$drive->{format};
1959
1960     my $cache_direct = 0;
1961
1962     if (my $cache = $drive->{cache}) {
1963         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964     } elsif (!drive_is_cdrom($drive)) {
1965         $opts .= ",cache=none";
1966         $cache_direct = 1;
1967     }
1968
1969     # aio native works only with O_DIRECT
1970     if (!$drive->{aio}) {
1971         if($cache_direct) {
1972             $opts .= ",aio=native";
1973         } else {
1974             $opts .= ",aio=threads";
1975         }
1976     }
1977
1978     if (!drive_is_cdrom($drive)) {
1979         my $detectzeroes;
1980         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1981             $detectzeroes = 'off';
1982         } elsif ($drive->{discard}) {
1983             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1984         } else {
1985             # This used to be our default with discard not being specified:
1986             $detectzeroes = 'on';
1987         }
1988         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1989     }
1990
1991     my $pathinfo = $path ? "file=$path," : '';
1992
1993     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1994 }
1995
1996 sub print_netdevice_full {
1997     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1998
1999     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2000
2001     my $device = $net->{model};
2002     if ($net->{model} eq 'virtio') {
2003          $device = 'virtio-net-pci';
2004      };
2005
2006     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2007     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2009         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010         my $vectors = $net->{queues} * 2 + 2;
2011         $tmpstr .= ",vectors=$vectors,mq=on";
2012     }
2013     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2014
2015     if ($use_old_bios_files) {
2016         my $romfile;
2017         if ($device eq 'virtio-net-pci') {
2018             $romfile = 'pxe-virtio.rom';
2019         } elsif ($device eq 'e1000') {
2020             $romfile = 'pxe-e1000.rom';
2021         } elsif ($device eq 'ne2k') {
2022             $romfile = 'pxe-ne2k_pci.rom';
2023         } elsif ($device eq 'pcnet') {
2024             $romfile = 'pxe-pcnet.rom';
2025         } elsif ($device eq 'rtl8139') {
2026             $romfile = 'pxe-rtl8139.rom';
2027         }
2028         $tmpstr .= ",romfile=$romfile" if $romfile;
2029     }
2030
2031     return $tmpstr;
2032 }
2033
2034 sub print_netdev_full {
2035     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2036
2037     my $i = '';
2038     if ($netid =~ m/^net(\d+)$/) {
2039         $i = int($1);
2040     }
2041
2042     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2043
2044     my $ifname = "tap${vmid}i$i";
2045
2046     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047     die "interface name '$ifname' is too long (max 15 character)\n"
2048         if length($ifname) >= 16;
2049
2050     my $vhostparam = '';
2051     if (is_native($arch)) {
2052         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2053     }
2054
2055     my $vmname = $conf->{name} || "vm$vmid";
2056
2057     my $netdev = "";
2058     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2059
2060     if ($net->{bridge}) {
2061         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2062     } else {
2063         $netdev = "type=user,id=$netid,hostname=$vmname";
2064     }
2065
2066     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2067
2068     return $netdev;
2069 }
2070
2071
2072 sub print_cpu_device {
2073     my ($conf, $id) = @_;
2074
2075     my $kvm = $conf->{kvm} // 1;
2076     my $cpu = $kvm ? "kvm64" : "qemu64";
2077     if (my $cputype = $conf->{cpu}) {
2078         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2079             or die "Cannot parse cpu description: $cputype\n";
2080         $cpu = $cpuconf->{cputype};
2081     }
2082
2083     my $cores = $conf->{cores} || 1;
2084
2085     my $current_core = ($id - 1) % $cores;
2086     my $current_socket = int(($id - 1 - $current_core)/$cores);
2087
2088     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2089 }
2090
2091 my $vga_map = {
2092     'cirrus' => 'cirrus-vga',
2093     'std' => 'VGA',
2094     'vmware' => 'vmware-svga',
2095     'virtio' => 'virtio-vga',
2096 };
2097
2098 sub print_vga_device {
2099     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2100
2101     my $type = $vga_map->{$vga->{type}};
2102     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103         $type = 'virtio-gpu';
2104     }
2105     my $vgamem_mb = $vga->{memory};
2106     if ($qxlnum) {
2107         $type = $id ? 'qxl' : 'qxl-vga';
2108     }
2109     die "no devicetype for $vga->{type}\n" if !$type;
2110
2111     my $memory = "";
2112     if ($vgamem_mb) {
2113         if ($vga->{type} eq 'virtio') {
2114             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2115             $memory = ",max_hostmem=$bytes";
2116         } elsif ($qxlnum) {
2117             # from https://www.spice-space.org/multiple-monitors.html
2118             $memory = ",vgamem_mb=$vga->{memory}";
2119             my $ram = $vgamem_mb * 4;
2120             my $vram = $vgamem_mb * 2;
2121             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2122         } else {
2123             $memory = ",vgamem_mb=$vga->{memory}";
2124         }
2125     } elsif ($qxlnum && $id) {
2126         $memory = ",ram_size=67108864,vram_size=33554432";
2127     }
2128
2129     my $q35 = machine_type_is_q35($conf);
2130     my $vgaid = "vga" . ($id // '');
2131     my $pciaddr;
2132
2133     if ($q35 && $vgaid eq 'vga') {
2134         # the first display uses pcie.0 bus on q35 machines
2135         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2136     } else {
2137         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2138     }
2139
2140     return "$type,id=${vgaid}${memory}${pciaddr}";
2141 }
2142
2143 sub drive_is_cloudinit {
2144     my ($drive) = @_;
2145     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2146 }
2147
2148 sub drive_is_cdrom {
2149     my ($drive, $exclude_cloudinit) = @_;
2150
2151     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2152
2153     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2154
2155 }
2156
2157 sub parse_number_sets {
2158     my ($set) = @_;
2159     my $res = [];
2160     foreach my $part (split(/;/, $set)) {
2161         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163             push @$res, [ $1, $2 ];
2164         } else {
2165             die "invalid range: $part\n";
2166         }
2167     }
2168     return $res;
2169 }
2170
2171 sub parse_numa {
2172     my ($data) = @_;
2173
2174     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2175     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2176     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2177     return $res;
2178 }
2179
2180 sub parse_hostpci {
2181     my ($value) = @_;
2182
2183     return undef if !$value;
2184
2185     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2186
2187     my @idlist = split(/;/, $res->{host});
2188     delete $res->{host};
2189     foreach my $id (@idlist) {
2190         if ($id =~ m/\./) { # full id 00:00.1
2191             push @{$res->{pciid}}, {
2192                 id => $id,
2193             };
2194         } else { # partial id 00:00
2195             $res->{pciid} = PVE::SysFSTools::lspci($id);
2196         }
2197     }
2198     return $res;
2199 }
2200
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2202 sub parse_net {
2203     my ($data) = @_;
2204
2205     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2206     if ($@) {
2207         warn $@;
2208         return undef;
2209     }
2210     if (!defined($res->{macaddr})) {
2211         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2212         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2213     }
2214     return $res;
2215 }
2216
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig {
2219     my ($data) = @_;
2220
2221     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2222     if ($@) {
2223         warn $@;
2224         return undef;
2225     }
2226
2227     if ($res->{gw} && !$res->{ip}) {
2228         warn 'gateway specified without specifying an IP address';
2229         return undef;
2230     }
2231     if ($res->{gw6} && !$res->{ip6}) {
2232         warn 'IPv6 gateway specified without specifying an IPv6 address';
2233         return undef;
2234     }
2235     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2236         warn 'gateway specified together with DHCP';
2237         return undef;
2238     }
2239     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2240         # gw6 + auto/dhcp
2241         warn "IPv6 gateway specified together with $res->{ip6} address";
2242         return undef;
2243     }
2244
2245     if (!$res->{ip} && !$res->{ip6}) {
2246         return { ip => 'dhcp', ip6 => 'dhcp' };
2247     }
2248
2249     return $res;
2250 }
2251
2252 sub print_net {
2253     my $net = shift;
2254
2255     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2256 }
2257
2258 sub add_random_macs {
2259     my ($settings) = @_;
2260
2261     foreach my $opt (keys %$settings) {
2262         next if $opt !~ m/^net(\d+)$/;
2263         my $net = parse_net($settings->{$opt});
2264         next if !$net;
2265         $settings->{$opt} = print_net($net);
2266     }
2267 }
2268
2269 sub vm_is_volid_owner {
2270     my ($storecfg, $vmid, $volid) = @_;
2271
2272     if ($volid !~  m|^/|) {
2273         my ($path, $owner);
2274         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2275         if ($owner && ($owner == $vmid)) {
2276             return 1;
2277         }
2278     }
2279
2280     return undef;
2281 }
2282
2283 sub split_flagged_list {
2284     my $text = shift || '';
2285     $text =~ s/[,;]/ /g;
2286     $text =~ s/^\s+//;
2287     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2288 }
2289
2290 sub join_flagged_list {
2291     my ($how, $lst) = @_;
2292     join $how, map { $lst->{$_} . $_ } keys %$lst;
2293 }
2294
2295 sub vmconfig_delete_pending_option {
2296     my ($conf, $key, $force) = @_;
2297
2298     delete $conf->{pending}->{$key};
2299     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2300     $pending_delete_hash->{$key} = $force ? '!' : '';
2301     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2302 }
2303
2304 sub vmconfig_undelete_pending_option {
2305     my ($conf, $key) = @_;
2306
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     delete $pending_delete_hash->{$key};
2309
2310     if (%$pending_delete_hash) {
2311         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2312     } else {
2313         delete $conf->{pending}->{delete};
2314     }
2315 }
2316
2317 sub vmconfig_register_unused_drive {
2318     my ($storecfg, $vmid, $conf, $drive) = @_;
2319
2320     if (drive_is_cloudinit($drive)) {
2321         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2322         warn $@ if $@;
2323     } elsif (!drive_is_cdrom($drive)) {
2324         my $volid = $drive->{file};
2325         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2326             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2327         }
2328     }
2329 }
2330
2331 sub vmconfig_cleanup_pending {
2332     my ($conf) = @_;
2333
2334     # remove pending changes when nothing changed
2335     my $changes;
2336     foreach my $opt (keys %{$conf->{pending}}) {
2337         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2338             $changes = 1;
2339             delete $conf->{pending}->{$opt};
2340         }
2341     }
2342
2343     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2344     my $pending_delete_hash = {};
2345     while (my ($opt, $force) = each %$current_delete_hash) {
2346         if (defined($conf->{$opt})) {
2347             $pending_delete_hash->{$opt} = $force;
2348         } else {
2349             $changes = 1;
2350         }
2351     }
2352
2353     if (%$pending_delete_hash) {
2354         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2355     } else {
2356         delete $conf->{pending}->{delete};
2357     }
2358
2359     return $changes;
2360 }
2361
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2363 my $smbios1_fmt = {
2364     uuid => {
2365         type => 'string',
2366         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367         format_description => 'UUID',
2368         description => "Set SMBIOS1 UUID.",
2369         optional => 1,
2370     },
2371     version => {
2372         type => 'string',
2373         pattern => '[A-Za-z0-9+\/]+={0,2}',
2374         format_description => 'Base64 encoded string',
2375         description => "Set SMBIOS1 version.",
2376         optional => 1,
2377     },
2378     serial => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 serial number.",
2383         optional => 1,
2384     },
2385     manufacturer => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 manufacturer.",
2390         optional => 1,
2391     },
2392     product => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 product ID.",
2397         optional => 1,
2398     },
2399     sku => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 SKU string.",
2404         optional => 1,
2405     },
2406     family => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 family string.",
2411         optional => 1,
2412     },
2413     base64 => {
2414         type => 'boolean',
2415         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2416         optional => 1,
2417     },
2418 };
2419
2420 sub parse_smbios1 {
2421     my ($data) = @_;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub print_smbios1 {
2429     my ($smbios1) = @_;
2430     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2431 }
2432
2433 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2434
2435 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2436 sub verify_bootdisk {
2437     my ($value, $noerr) = @_;
2438
2439     return $value if is_valid_drivename($value);
2440
2441     return undef if $noerr;
2442
2443     die "invalid boot disk '$value'\n";
2444 }
2445
2446 sub parse_watchdog {
2447     my ($value) = @_;
2448
2449     return undef if !$value;
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2452     warn $@ if $@;
2453     return $res;
2454 }
2455
2456 sub parse_guest_agent {
2457     my ($value) = @_;
2458
2459     return {} if !defined($value->{agent});
2460
2461     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2462     warn $@ if $@;
2463
2464     # if the agent is disabled ignore the other potentially set properties
2465     return {} if !$res->{enabled};
2466     return $res;
2467 }
2468
2469 sub parse_vga {
2470     my ($value) = @_;
2471
2472     return {} if !$value;
2473     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2474     warn $@ if $@;
2475     return $res;
2476 }
2477
2478 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2479 sub verify_usb_device {
2480     my ($value, $noerr) = @_;
2481
2482     return $value if parse_usb_device($value);
2483
2484     return undef if $noerr;
2485
2486     die "unable to parse usb device\n";
2487 }
2488
2489 # add JSON properties for create and set function
2490 sub json_config_properties {
2491     my $prop = shift;
2492
2493     foreach my $opt (keys %$confdesc) {
2494         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495         $prop->{$opt} = $confdesc->{$opt};
2496     }
2497
2498     return $prop;
2499 }
2500
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties {
2503
2504     return dclone($confdesc_cloudinit);
2505 }
2506
2507 sub check_type {
2508     my ($key, $value) = @_;
2509
2510     die "unknown setting '$key'\n" if !$confdesc->{$key};
2511
2512     my $type = $confdesc->{$key}->{type};
2513
2514     if (!defined($value)) {
2515         die "got undefined value\n";
2516     }
2517
2518     if ($value =~ m/[\n\r]/) {
2519         die "property contains a line feed\n";
2520     }
2521
2522     if ($type eq 'boolean') {
2523         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525         die "type check ('boolean') failed - got '$value'\n";
2526     } elsif ($type eq 'integer') {
2527         return int($1) if $value =~ m/^(\d+)$/;
2528         die "type check ('integer') failed - got '$value'\n";
2529     } elsif ($type eq 'number') {
2530         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531         die "type check ('number') failed - got '$value'\n";
2532     } elsif ($type eq 'string') {
2533         if (my $fmt = $confdesc->{$key}->{format}) {
2534             PVE::JSONSchema::check_format($fmt, $value);
2535             return $value;
2536         }
2537         $value =~ s/^\"(.*)\"$/$1/;
2538         return $value;
2539     } else {
2540         die "internal error"
2541     }
2542 }
2543
2544 sub touch_config {
2545     my ($vmid) = @_;
2546
2547     my $conf = PVE::QemuConfig->config_file($vmid);
2548     utime undef, undef, $conf;
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conffile = PVE::QemuConfig->config_file($vmid);
2555
2556     my $conf = PVE::QemuConfig->load_config($vmid);
2557
2558     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2559
2560     if ($conf->{template}) {
2561         # check if any base image is still used by a linked clone
2562         foreach_drive($conf, sub {
2563                 my ($ds, $drive) = @_;
2564
2565                 return if drive_is_cdrom($drive);
2566
2567                 my $volid = $drive->{file};
2568
2569                 return if !$volid || $volid =~ m|^/|;
2570
2571                 die "base volume '$volid' is still in use by linked cloned\n"
2572                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2573
2574         });
2575     }
2576
2577     # only remove disks owned by this VM
2578     foreach_drive($conf, sub {
2579         my ($ds, $drive) = @_;
2580
2581         return if drive_is_cdrom($drive, 1);
2582
2583         my $volid = $drive->{file};
2584
2585         return if !$volid || $volid =~ m|^/|;
2586
2587         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2588         return if !$path || !$owner || ($owner != $vmid);
2589
2590         eval {
2591             PVE::Storage::vdisk_free($storecfg, $volid);
2592         };
2593         warn "Could not remove disk '$volid', check manually: $@" if $@;
2594
2595     });
2596
2597     if ($keep_empty_config) {
2598         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2599     } else {
2600         unlink $conffile;
2601     }
2602
2603     # also remove unused disk
2604     eval {
2605         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2606
2607         eval {
2608             PVE::Storage::foreach_volid($dl, sub {
2609                 my ($volid, $sid, $volname, $d) = @_;
2610                 PVE::Storage::vdisk_free($storecfg, $volid);
2611             });
2612         };
2613         warn $@ if $@;
2614
2615     };
2616     warn $@ if $@;
2617 }
2618
2619 sub parse_vm_config {
2620     my ($filename, $raw) = @_;
2621
2622     return undef if !defined($raw);
2623
2624     my $res = {
2625         digest => Digest::SHA::sha1_hex($raw),
2626         snapshots => {},
2627         pending => {},
2628     };
2629
2630     $filename =~ m|/qemu-server/(\d+)\.conf$|
2631         || die "got strange filename '$filename'";
2632
2633     my $vmid = $1;
2634
2635     my $conf = $res;
2636     my $descr;
2637     my $section = '';
2638
2639     my @lines = split(/\n/, $raw);
2640     foreach my $line (@lines) {
2641         next if $line =~ m/^\s*$/;
2642
2643         if ($line =~ m/^\[PENDING\]\s*$/i) {
2644             $section = 'pending';
2645             if (defined($descr)) {
2646                 $descr =~ s/\s+$//;
2647                 $conf->{description} = $descr;
2648             }
2649             $descr = undef;
2650             $conf = $res->{$section} = {};
2651             next;
2652
2653         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2654             $section = $1;
2655             if (defined($descr)) {
2656                 $descr =~ s/\s+$//;
2657                 $conf->{description} = $descr;
2658             }
2659             $descr = undef;
2660             $conf = $res->{snapshots}->{$section} = {};
2661             next;
2662         }
2663
2664         if ($line =~ m/^\#(.*)\s*$/) {
2665             $descr = '' if !defined($descr);
2666             $descr .= PVE::Tools::decode_text($1) . "\n";
2667             next;
2668         }
2669
2670         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($2);
2673         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674             $conf->{snapstate} = $1;
2675         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2676             my $key = $1;
2677             my $value = $2;
2678             $conf->{$key} = $value;
2679         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2680             my $value = $1;
2681             if ($section eq 'pending') {
2682                 $conf->{delete} = $value; # we parse this later
2683             } else {
2684                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2685             }
2686         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2687             my $key = $1;
2688             my $value = $2;
2689             eval { $value = check_type($key, $value); };
2690             if ($@) {
2691                 warn "vm $vmid - unable to parse value of '$key' - $@";
2692             } else {
2693                 $key = 'ide2' if $key eq 'cdrom';
2694                 my $fmt = $confdesc->{$key}->{format};
2695                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696                     my $v = parse_drive($key, $value);
2697                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2698                         $v->{file} = $volid;
2699                         $value = print_drive($vmid, $v);
2700                     } else {
2701                         warn "vm $vmid - unable to parse value of '$key'\n";
2702                         next;
2703                     }
2704                 }
2705
2706                 $conf->{$key} = $value;
2707             }
2708         }
2709     }
2710
2711     if (defined($descr)) {
2712         $descr =~ s/\s+$//;
2713         $conf->{description} = $descr;
2714     }
2715     delete $res->{snapstate}; # just to be sure
2716
2717     return $res;
2718 }
2719
2720 sub write_vm_config {
2721     my ($filename, $conf) = @_;
2722
2723     delete $conf->{snapstate}; # just to be sure
2724
2725     if ($conf->{cdrom}) {
2726         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2727         $conf->{ide2} = $conf->{cdrom};
2728         delete $conf->{cdrom};
2729     }
2730
2731     # we do not use 'smp' any longer
2732     if ($conf->{sockets}) {
2733         delete $conf->{smp};
2734     } elsif ($conf->{smp}) {
2735         $conf->{sockets} = $conf->{smp};
2736         delete $conf->{cores};
2737         delete $conf->{smp};
2738     }
2739
2740     my $used_volids = {};
2741
2742     my $cleanup_config = sub {
2743         my ($cref, $pending, $snapname) = @_;
2744
2745         foreach my $key (keys %$cref) {
2746             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747                 $key eq 'snapstate' || $key eq 'pending';
2748             my $value = $cref->{$key};
2749             if ($key eq 'delete') {
2750                 die "propertry 'delete' is only allowed in [PENDING]\n"
2751                     if !$pending;
2752                 # fixme: check syntax?
2753                 next;
2754             }
2755             eval { $value = check_type($key, $value); };
2756             die "unable to parse value of '$key' - $@" if $@;
2757
2758             $cref->{$key} = $value;
2759
2760             if (!$snapname && is_valid_drivename($key)) {
2761                 my $drive = parse_drive($key, $value);
2762                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2763             }
2764         }
2765     };
2766
2767     &$cleanup_config($conf);
2768
2769     &$cleanup_config($conf->{pending}, 1);
2770
2771     foreach my $snapname (keys %{$conf->{snapshots}}) {
2772         die "internal error" if $snapname eq 'pending';
2773         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2774     }
2775
2776     # remove 'unusedX' settings if we re-add a volume
2777     foreach my $key (keys %$conf) {
2778         my $value = $conf->{$key};
2779         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780             delete $conf->{$key};
2781         }
2782     }
2783
2784     my $generate_raw_config = sub {
2785         my ($conf, $pending) = @_;
2786
2787         my $raw = '';
2788
2789         # add description as comment to top of file
2790         if (defined(my $descr = $conf->{description})) {
2791             if ($descr) {
2792                 foreach my $cl (split(/\n/, $descr)) {
2793                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2794                 }
2795             } else {
2796                 $raw .= "#\n" if $pending;
2797             }
2798         }
2799
2800         foreach my $key (sort keys %$conf) {
2801             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802             $raw .= "$key: $conf->{$key}\n";
2803         }
2804         return $raw;
2805     };
2806
2807     my $raw = &$generate_raw_config($conf);
2808
2809     if (scalar(keys %{$conf->{pending}})){
2810         $raw .= "\n[PENDING]\n";
2811         $raw .= &$generate_raw_config($conf->{pending}, 1);
2812     }
2813
2814     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2815         $raw .= "\n[$snapname]\n";
2816         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2817     }
2818
2819     return $raw;
2820 }
2821
2822 sub load_defaults {
2823
2824     my $res = {};
2825
2826     # we use static defaults from our JSON schema configuration
2827     foreach my $key (keys %$confdesc) {
2828         if (defined(my $default = $confdesc->{$key}->{default})) {
2829             $res->{$key} = $default;
2830         }
2831     }
2832
2833     return $res;
2834 }
2835
2836 sub config_list {
2837     my $vmlist = PVE::Cluster::get_vmlist();
2838     my $res = {};
2839     return $res if !$vmlist || !$vmlist->{ids};
2840     my $ids = $vmlist->{ids};
2841
2842     foreach my $vmid (keys %$ids) {
2843         my $d = $ids->{$vmid};
2844         next if !$d->{node} || $d->{node} ne $nodename;
2845         next if !$d->{type} || $d->{type} ne 'qemu';
2846         $res->{$vmid}->{exists} = 1;
2847     }
2848     return $res;
2849 }
2850
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources {
2853     my ($conf, $noerr) = @_;
2854
2855     my @loc_res = ();
2856
2857     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2858     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2859
2860     push @loc_res, "ivshmem" if $conf->{ivshmem};
2861
2862     foreach my $k (keys %$conf) {
2863         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864         # sockets are safe: they will recreated be on the target side post-migrate
2865         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2867     }
2868
2869     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2870
2871     return \@loc_res;
2872 }
2873
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability {
2876     my ($storecfg, $conf, $node) = @_;
2877
2878     foreach_drive($conf, sub {
2879         my ($ds, $drive) = @_;
2880
2881         my $volid = $drive->{file};
2882         return if !$volid;
2883
2884         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2885         return if !$sid;
2886
2887         # check if storage is available on both nodes
2888         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2889         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2890    });
2891 }
2892
2893 # list nodes where all VM images are available (used by has_feature API)
2894 sub shared_nodes {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => 1 } @$nodelist };
2899     my $nodename = PVE::INotify::nodename();
2900
2901     foreach_drive($conf, sub {
2902         my ($ds, $drive) = @_;
2903
2904         my $volid = $drive->{file};
2905         return if !$volid;
2906
2907         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2908         if ($storeid) {
2909             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2910             if ($scfg->{disable}) {
2911                 $nodehash = {};
2912             } elsif (my $avail = $scfg->{nodes}) {
2913                 foreach my $node (keys %$nodehash) {
2914                     delete $nodehash->{$node} if !$avail->{$node};
2915                 }
2916             } elsif (!$scfg->{shared}) {
2917                 foreach my $node (keys %$nodehash) {
2918                     delete $nodehash->{$node} if $node ne $nodename
2919                 }
2920             }
2921         }
2922     });
2923
2924     return $nodehash
2925 }
2926
2927 sub check_local_storage_availability {
2928     my ($conf, $storecfg) = @_;
2929
2930     my $nodelist = PVE::Cluster::get_nodelist();
2931     my $nodehash = { map { $_ => {} } @$nodelist };
2932
2933     foreach_drive($conf, sub {
2934         my ($ds, $drive) = @_;
2935
2936         my $volid = $drive->{file};
2937         return if !$volid;
2938
2939         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2940         if ($storeid) {
2941             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2942
2943             if ($scfg->{disable}) {
2944                 foreach my $node (keys %$nodehash) {
2945                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2946                 }
2947             } elsif (my $avail = $scfg->{nodes}) {
2948                 foreach my $node (keys %$nodehash) {
2949                     if (!$avail->{$node}) {
2950                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2951                     }
2952                 }
2953             }
2954         }
2955     });
2956
2957     foreach my $node (values %$nodehash) {
2958         if (my $unavail = $node->{unavailable_storages}) {
2959             $node->{unavailable_storages} = [ sort keys %$unavail ];
2960         }
2961     }
2962
2963     return $nodehash
2964 }
2965
2966 sub check_cmdline {
2967     my ($pidfile, $pid) = @_;
2968
2969     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2970     if (defined($fh)) {
2971         my $line = <$fh>;
2972         $fh->close;
2973         return undef if !$line;
2974         my @param = split(/\0/, $line);
2975
2976         my $cmd = $param[0];
2977         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2978
2979         for (my $i = 0; $i < scalar (@param); $i++) {
2980             my $p = $param[$i];
2981             next if !$p;
2982             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2983                 my $p = $param[$i+1];
2984                 return 1 if $p && ($p eq $pidfile);
2985                 return undef;
2986             }
2987         }
2988     }
2989     return undef;
2990 }
2991
2992 sub check_running {
2993     my ($vmid, $nocheck, $node) = @_;
2994
2995     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2996
2997     die "unable to find configuration file for VM $vmid - no such machine\n"
2998         if !$nocheck && ! -f $filename;
2999
3000     my $pidfile = pidfile_name($vmid);
3001
3002     if (my $fd = IO::File->new("<$pidfile")) {
3003         my $st = stat($fd);
3004         my $line = <$fd>;
3005         close($fd);
3006
3007         my $mtime = $st->mtime;
3008         if ($mtime > time()) {
3009             warn "file '$filename' modified in future\n";
3010         }
3011
3012         if ($line =~ m/^(\d+)$/) {
3013             my $pid = $1;
3014             if (check_cmdline($pidfile, $pid)) {
3015                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3016                     return $pid;
3017                 }
3018             }
3019         }
3020     }
3021
3022     return undef;
3023 }
3024
3025 sub vzlist {
3026
3027     my $vzlist = config_list();
3028
3029     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3030
3031     while (defined(my $de = $fd->read)) {
3032         next if $de !~ m/^(\d+)\.pid$/;
3033         my $vmid = $1;
3034         next if !defined($vzlist->{$vmid});
3035         if (my $pid = check_running($vmid)) {
3036             $vzlist->{$vmid}->{pid} = $pid;
3037         }
3038     }
3039
3040     return $vzlist;
3041 }
3042
3043 sub disksize {
3044     my ($storecfg, $conf) = @_;
3045
3046     my $bootdisk = $conf->{bootdisk};
3047     return undef if !$bootdisk;
3048     return undef if !is_valid_drivename($bootdisk);
3049
3050     return undef if !$conf->{$bootdisk};
3051
3052     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3053     return undef if !defined($drive);
3054
3055     return undef if drive_is_cdrom($drive);
3056
3057     my $volid = $drive->{file};
3058     return undef if !$volid;
3059
3060     return $drive->{size};
3061 }
3062
3063 our $vmstatus_return_properties = {
3064     vmid => get_standard_option('pve-vmid'),
3065     status => {
3066         description => "Qemu process status.",
3067         type => 'string',
3068         enum => ['stopped', 'running'],
3069     },
3070     maxmem => {
3071         description => "Maximum memory in bytes.",
3072         type => 'integer',
3073         optional => 1,
3074         renderer => 'bytes',
3075     },
3076     maxdisk => {
3077         description => "Root disk size in bytes.",
3078         type => 'integer',
3079         optional => 1,
3080         renderer => 'bytes',
3081     },
3082     name => {
3083         description => "VM name.",
3084         type => 'string',
3085         optional => 1,
3086     },
3087     qmpstatus => {
3088         description => "Qemu QMP agent status.",
3089         type => 'string',
3090         optional => 1,
3091     },
3092     pid => {
3093         description => "PID of running qemu process.",
3094         type => 'integer',
3095         optional => 1,
3096     },
3097     uptime => {
3098         description => "Uptime.",
3099         type => 'integer',
3100         optional => 1,
3101         renderer => 'duration',
3102     },
3103     cpus => {
3104         description => "Maximum usable CPUs.",
3105         type => 'number',
3106         optional => 1,
3107     },
3108     lock => {
3109         description => "The current config lock, if any.",
3110         type => 'string',
3111         optional => 1,
3112     }
3113 };
3114
3115 my $last_proc_pid_stat;
3116
3117 # get VM status information
3118 # This must be fast and should not block ($full == false)
3119 # We only query KVM using QMP if $full == true (this can be slow)
3120 sub vmstatus {
3121     my ($opt_vmid, $full) = @_;
3122
3123     my $res = {};
3124
3125     my $storecfg = PVE::Storage::config();
3126
3127     my $list = vzlist();
3128     my $defaults = load_defaults();
3129
3130     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3131
3132     my $cpucount = $cpuinfo->{cpus} || 1;
3133
3134     foreach my $vmid (keys %$list) {
3135         next if $opt_vmid && ($vmid ne $opt_vmid);
3136
3137         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3138         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3139
3140         my $d = { vmid => $vmid };
3141         $d->{pid} = $list->{$vmid}->{pid};
3142
3143         # fixme: better status?
3144         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3145
3146         my $size = disksize($storecfg, $conf);
3147         if (defined($size)) {
3148             $d->{disk} = 0; # no info available
3149             $d->{maxdisk} = $size;
3150         } else {
3151             $d->{disk} = 0;
3152             $d->{maxdisk} = 0;
3153         }
3154
3155         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3156             * ($conf->{cores} || $defaults->{cores});
3157         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3158         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3159
3160         $d->{name} = $conf->{name} || "VM $vmid";
3161         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3162             : $defaults->{memory}*(1024*1024);
3163
3164         if ($conf->{balloon}) {
3165             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3166             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3167                 : $defaults->{shares};
3168         }
3169
3170         $d->{uptime} = 0;
3171         $d->{cpu} = 0;
3172         $d->{mem} = 0;
3173
3174         $d->{netout} = 0;
3175         $d->{netin} = 0;
3176
3177         $d->{diskread} = 0;
3178         $d->{diskwrite} = 0;
3179
3180         $d->{template} = PVE::QemuConfig->is_template($conf);
3181
3182         $d->{serial} = 1 if conf_has_serial($conf);
3183         $d->{lock} = $conf->{lock} if $conf->{lock};
3184
3185         $res->{$vmid} = $d;
3186     }
3187
3188     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3189     foreach my $dev (keys %$netdev) {
3190         next if $dev !~ m/^tap([1-9]\d*)i/;
3191         my $vmid = $1;
3192         my $d = $res->{$vmid};
3193         next if !$d;
3194
3195         $d->{netout} += $netdev->{$dev}->{receive};
3196         $d->{netin} += $netdev->{$dev}->{transmit};
3197
3198         if ($full) {
3199             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3200             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3201         }
3202
3203     }
3204
3205     my $ctime = gettimeofday;
3206
3207     foreach my $vmid (keys %$list) {
3208
3209         my $d = $res->{$vmid};
3210         my $pid = $d->{pid};
3211         next if !$pid;
3212
3213         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3214         next if !$pstat; # not running
3215
3216         my $used = $pstat->{utime} + $pstat->{stime};
3217
3218         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3219
3220         if ($pstat->{vsize}) {
3221             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3222         }
3223
3224         my $old = $last_proc_pid_stat->{$pid};
3225         if (!$old) {
3226             $last_proc_pid_stat->{$pid} = {
3227                 time => $ctime,
3228                 used => $used,
3229                 cpu => 0,
3230             };
3231             next;
3232         }
3233
3234         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3235
3236         if ($dtime > 1000) {
3237             my $dutime = $used -  $old->{used};
3238
3239             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3240             $last_proc_pid_stat->{$pid} = {
3241                 time => $ctime,
3242                 used => $used,
3243                 cpu => $d->{cpu},
3244             };
3245         } else {
3246             $d->{cpu} = $old->{cpu};
3247         }
3248     }
3249
3250     return $res if !$full;
3251
3252     my $qmpclient = PVE::QMPClient->new();
3253
3254     my $ballooncb = sub {
3255         my ($vmid, $resp) = @_;
3256
3257         my $info = $resp->{'return'};
3258         return if !$info->{max_mem};
3259
3260         my $d = $res->{$vmid};
3261
3262         # use memory assigned to VM
3263         $d->{maxmem} = $info->{max_mem};
3264         $d->{balloon} = $info->{actual};
3265
3266         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3267             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3268             $d->{freemem} = $info->{free_mem};
3269         }
3270
3271         $d->{ballooninfo} = $info;
3272     };
3273
3274     my $blockstatscb = sub {
3275         my ($vmid, $resp) = @_;
3276         my $data = $resp->{'return'} || [];
3277         my $totalrdbytes = 0;
3278         my $totalwrbytes = 0;
3279
3280         for my $blockstat (@$data) {
3281             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3282             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3283
3284             $blockstat->{device} =~ s/drive-//;
3285             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3286         }
3287         $res->{$vmid}->{diskread} = $totalrdbytes;
3288         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3289     };
3290
3291     my $statuscb = sub {
3292         my ($vmid, $resp) = @_;
3293
3294         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3295         # this fails if ballon driver is not loaded, so this must be
3296         # the last commnand (following command are aborted if this fails).
3297         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3298
3299         my $status = 'unknown';
3300         if (!defined($status = $resp->{'return'}->{status})) {
3301             warn "unable to get VM status\n";
3302             return;
3303         }
3304
3305         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3306     };
3307
3308     foreach my $vmid (keys %$list) {
3309         next if $opt_vmid && ($vmid ne $opt_vmid);
3310         next if !$res->{$vmid}->{pid}; # not running
3311         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3312     }
3313
3314     $qmpclient->queue_execute(undef, 2);
3315
3316     foreach my $vmid (keys %$list) {
3317         next if $opt_vmid && ($vmid ne $opt_vmid);
3318         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3319     }
3320
3321     return $res;
3322 }
3323
3324 sub foreach_drive {
3325     my ($conf, $func, @param) = @_;
3326
3327     foreach my $ds (valid_drive_names()) {
3328         next if !defined($conf->{$ds});
3329
3330         my $drive = parse_drive($ds, $conf->{$ds});
3331         next if !$drive;
3332
3333         &$func($ds, $drive, @param);
3334     }
3335 }
3336
3337 sub foreach_volid {
3338     my ($conf, $func, @param) = @_;
3339
3340     my $volhash = {};
3341
3342     my $test_volid = sub {
3343         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3344
3345         return if !$volid;
3346
3347         $volhash->{$volid}->{cdrom} //= 1;
3348         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3349
3350         $volhash->{$volid}->{replicate} //= 0;
3351         $volhash->{$volid}->{replicate} = 1 if $replicate;
3352
3353         $volhash->{$volid}->{shared} //= 0;
3354         $volhash->{$volid}->{shared} = 1 if $shared;
3355
3356         $volhash->{$volid}->{referenced_in_config} //= 0;
3357         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3358
3359         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3360             if defined($snapname);
3361         $volhash->{$volid}->{size} = $size if $size;
3362     };
3363
3364     foreach_drive($conf, sub {
3365         my ($ds, $drive) = @_;
3366         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3367     });
3368
3369     foreach my $snapname (keys %{$conf->{snapshots}}) {
3370         my $snap = $conf->{snapshots}->{$snapname};
3371         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3372         foreach_drive($snap, sub {
3373             my ($ds, $drive) = @_;
3374             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3375         });
3376     }
3377
3378     foreach my $volid (keys %$volhash) {
3379         &$func($volid, $volhash->{$volid}, @param);
3380     }
3381 }
3382
3383 sub conf_has_serial {
3384     my ($conf) = @_;
3385
3386     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3387         if ($conf->{"serial$i"}) {
3388             return 1;
3389         }
3390     }
3391
3392     return 0;
3393 }
3394
3395 sub vga_conf_has_spice {
3396     my ($vga) = @_;
3397
3398     my $vgaconf = parse_vga($vga);
3399     my $vgatype = $vgaconf->{type};
3400     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3401
3402     return $1 || 1;
3403 }
3404
3405 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3406 sub get_host_arch() {
3407     $host_arch = (POSIX::uname())[4] if !$host_arch;
3408     return $host_arch;
3409 }
3410
3411 sub is_native($) {
3412     my ($arch) = @_;
3413     return get_host_arch() eq $arch;
3414 }
3415
3416 my $default_machines = {
3417     x86_64 => 'pc',
3418     aarch64 => 'virt',
3419 };
3420
3421 sub get_basic_machine_info {
3422     my ($conf, $forcemachine) = @_;
3423
3424     my $arch = $conf->{arch} // get_host_arch();
3425     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3426     return ($arch, $machine);
3427 }
3428
3429 sub get_ovmf_files($) {
3430     my ($arch) = @_;
3431
3432     my $ovmf = $OVMF->{$arch}
3433         or die "no OVMF images known for architecture '$arch'\n";
3434
3435     return @$ovmf;
3436 }
3437
3438 my $Arch2Qemu = {
3439     aarch64 => '/usr/bin/qemu-system-aarch64',
3440     x86_64 => '/usr/bin/qemu-system-x86_64',
3441 };
3442 sub get_command_for_arch($) {
3443     my ($arch) = @_;
3444     return '/usr/bin/kvm' if is_native($arch);
3445
3446     my $cmd = $Arch2Qemu->{$arch}
3447         or die "don't know how to emulate architecture '$arch'\n";
3448     return $cmd;
3449 }
3450
3451 sub get_cpu_options {
3452     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3453
3454     my $cpuFlags = [];
3455     my $ostype = $conf->{ostype};
3456
3457     my $cpu = $kvm ? "kvm64" : "qemu64";
3458     if ($arch eq 'aarch64') {
3459         $cpu = 'cortex-a57';
3460     }
3461     my $hv_vendor_id;
3462     if (my $cputype = $conf->{cpu}) {
3463         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3464             or die "Cannot parse cpu description: $cputype\n";
3465         $cpu = $cpuconf->{cputype};
3466         $kvm_off = 1 if $cpuconf->{hidden};
3467         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3468
3469         if (defined(my $flags = $cpuconf->{flags})) {
3470             push @$cpuFlags, split(";", $flags);
3471         }
3472     }
3473
3474     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3475
3476     push @$cpuFlags , '-x2apic'
3477         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3478
3479     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3480
3481     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3482
3483     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3484
3485         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3486         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3487     }
3488
3489     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3490
3491     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3492
3493     push @$cpuFlags, 'kvm=off' if $kvm_off;
3494
3495     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3496         push @$cpuFlags, "vendor=${cpu_vendor}"
3497             if $cpu_vendor ne 'default';
3498     } elsif ($arch ne 'aarch64') {
3499         die "internal error"; # should not happen
3500     }
3501
3502     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3503
3504     return ('-cpu', $cpu);
3505 }
3506
3507 sub config_to_command {
3508     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3509
3510     my $cmd = [];
3511     my $globalFlags = [];
3512     my $machineFlags = [];
3513     my $rtcFlags = [];
3514     my $devices = [];
3515     my $pciaddr = '';
3516     my $bridges = {};
3517     my $kvmver = kvm_user_version();
3518     my $vernum = 0; # unknown
3519     my $ostype = $conf->{ostype};
3520     my $winversion = windows_version($ostype);
3521     my $kvm = $conf->{kvm};
3522
3523     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3524     $kvm //= 1 if is_native($arch);
3525
3526     if ($kvm) {
3527         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3528             if !defined kvm_version();
3529     }
3530
3531     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3532         $vernum = $1*1000000+$2*1000;
3533     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3534         $vernum = $1*1000000+$2*1000+$3;
3535     }
3536
3537     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3538
3539     my $have_ovz = -f '/proc/vz/vestat';
3540
3541     my $q35 = machine_type_is_q35($conf);
3542     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3543     my $use_old_bios_files = undef;
3544     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3545
3546     my $cpuunits = defined($conf->{cpuunits}) ?
3547             $conf->{cpuunits} : $defaults->{cpuunits};
3548
3549     push @$cmd, get_command_for_arch($arch);
3550
3551     push @$cmd, '-id', $vmid;
3552
3553     my $vmname = $conf->{name} || "vm$vmid";
3554
3555     push @$cmd, '-name', $vmname;
3556
3557     my $use_virtio = 0;
3558
3559     my $qmpsocket = qmp_socket($vmid);
3560     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3561     push @$cmd, '-mon', "chardev=qmp,mode=control";
3562
3563     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3564         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3565         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3566     }
3567
3568     push @$cmd, '-pidfile' , pidfile_name($vmid);
3569
3570     push @$cmd, '-daemonize';
3571
3572     if ($conf->{smbios1}) {
3573         my $smbios_conf = parse_smbios1($conf->{smbios1});
3574         if ($smbios_conf->{base64}) {
3575             # Do not pass base64 flag to qemu
3576             delete $smbios_conf->{base64};
3577             my $smbios_string = "";
3578             foreach my $key (keys %$smbios_conf) {
3579                 my $value;
3580                 if ($key eq "uuid") {
3581                     $value = $smbios_conf->{uuid}
3582                 } else {
3583                     $value = decode_base64($smbios_conf->{$key});
3584                 }
3585                 # qemu accepts any binary data, only commas need escaping by double comma
3586                 $value =~ s/,/,,/g;
3587                 $smbios_string .= "," . $key . "=" . $value if $value;
3588             }
3589             push @$cmd, '-smbios', "type=1" . $smbios_string;
3590         } else {
3591             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3592         }
3593     }
3594
3595     if ($conf->{vmgenid}) {
3596         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3597     }
3598
3599     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3600     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3601         die "uefi base image not found\n" if ! -f $ovmf_code;
3602
3603         my $path;
3604         my $format;
3605         if (my $efidisk = $conf->{efidisk0}) {
3606             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3607             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3608             $format = $d->{format};
3609             if ($storeid) {
3610                 $path = PVE::Storage::path($storecfg, $d->{file});
3611                 if (!defined($format)) {
3612                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3613                     $format = qemu_img_format($scfg, $volname);
3614                 }
3615             } else {
3616                 $path = $d->{file};
3617                 die "efidisk format must be specified\n"
3618                     if !defined($format);
3619             }
3620         } else {
3621             warn "no efidisk configured! Using temporary efivars disk.\n";
3622             $path = "/tmp/$vmid-ovmf.fd";
3623             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3624             $format = 'raw';
3625         }
3626
3627         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3628         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3629     }
3630
3631
3632     # add usb controllers
3633     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3634     push @$devices, @usbcontrollers if @usbcontrollers;
3635     my $vga = parse_vga($conf->{vga});
3636
3637     my $qxlnum = vga_conf_has_spice($conf->{vga});
3638     $vga->{type} = 'qxl' if $qxlnum;
3639
3640     if (!$vga->{type}) {
3641         if ($arch eq 'aarch64') {
3642             $vga->{type} = 'virtio';
3643         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3644             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3645         } else {
3646             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3647         }
3648     }
3649
3650     # enable absolute mouse coordinates (needed by vnc)
3651     my $tablet;
3652     if (defined($conf->{tablet})) {
3653         $tablet = $conf->{tablet};
3654     } else {
3655         $tablet = $defaults->{tablet};
3656         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3657         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3658     }
3659
3660     if ($tablet) {
3661         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3662         my $kbd = print_keyboarddevice_full($conf, $arch);
3663         push @$devices, '-device', $kbd if defined($kbd);
3664     }
3665
3666     my $kvm_off = 0;
3667     my $gpu_passthrough;
3668
3669     # host pci devices
3670     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3671         my $d = parse_hostpci($conf->{"hostpci$i"});
3672         next if !$d;
3673
3674         my $pcie = $d->{pcie};
3675         if ($pcie) {
3676             die "q35 machine model is not enabled" if !$q35;
3677             # win7 wants to have the pcie devices directly on the pcie bus
3678             # instead of in the root port
3679             if ($winversion == 7) {
3680                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3681             } else {
3682                 $pciaddr = print_pcie_addr("hostpci$i");
3683             }
3684         } else {
3685             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3686         }
3687
3688         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3689         my $romfile = $d->{romfile};
3690
3691         my $xvga = '';
3692         if ($d->{'x-vga'}) {
3693             $xvga = ',x-vga=on';
3694             $kvm_off = 1;
3695             $vga->{type} = 'none' if !defined($conf->{vga});
3696             $gpu_passthrough = 1;
3697
3698             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3699                 $xvga = "";
3700             }
3701         }
3702         my $pcidevices = $d->{pciid};
3703         my $multifunction = 1 if @$pcidevices > 1;
3704         my $sysfspath;
3705         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3706             my $id = $pcidevices->[0]->{id};
3707             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3708             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3709         } elsif ($d->{mdev}) {
3710             warn "ignoring mediated device with multifunction device\n";
3711         }
3712
3713         my $j=0;
3714         foreach my $pcidevice (@$pcidevices) {
3715
3716             my $id = "hostpci$i";
3717             $id .= ".$j" if $multifunction;
3718             my $addr = $pciaddr;
3719             $addr .= ".$j" if $multifunction;
3720             my $devicestr = "vfio-pci";
3721             if ($sysfspath) {
3722                 $devicestr .= ",sysfsdev=$sysfspath";
3723             } else {
3724                 $devicestr .= ",host=$pcidevice->{id}";
3725             }
3726             $devicestr .= ",id=$id$addr";
3727
3728             if($j == 0){
3729                 $devicestr .= "$rombar$xvga";
3730                 $devicestr .= ",multifunction=on" if $multifunction;
3731                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3732             }
3733
3734             push @$devices, '-device', $devicestr;
3735             $j++;
3736         }
3737     }
3738
3739     # usb devices
3740     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3741     push @$devices, @usbdevices if @usbdevices;
3742     # serial devices
3743     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3744         if (my $path = $conf->{"serial$i"}) {
3745             if ($path eq 'socket') {
3746                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3747                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3748                 # On aarch64, serial0 is the UART device. Qemu only allows
3749                 # connecting UART devices via the '-serial' command line, as
3750                 # the device has a fixed slot on the hardware...
3751                 if ($arch eq 'aarch64' && $i == 0) {
3752                     push @$devices, '-serial', "chardev:serial$i";
3753                 } else {
3754                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3755                 }
3756             } else {
3757                 die "no such serial device\n" if ! -c $path;
3758                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3759                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3760             }
3761         }
3762     }
3763
3764     # parallel devices
3765     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3766         if (my $path = $conf->{"parallel$i"}) {
3767             die "no such parallel device\n" if ! -c $path;
3768             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3769             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3770             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3771         }
3772     }
3773
3774
3775     my $sockets = 1;
3776     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3777     $sockets = $conf->{sockets} if  $conf->{sockets};
3778
3779     my $cores = $conf->{cores} || 1;
3780
3781     my $maxcpus = $sockets * $cores;
3782
3783     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3784
3785     my $allowed_vcpus = $cpuinfo->{cpus};
3786
3787     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3788         if ($allowed_vcpus < $maxcpus);
3789
3790     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3791
3792         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3793         for (my $i = 2; $i <= $vcpus; $i++)  {
3794             my $cpustr = print_cpu_device($conf,$i);
3795             push @$cmd, '-device', $cpustr;
3796         }
3797
3798     } else {
3799
3800         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3801     }
3802     push @$cmd, '-nodefaults';
3803
3804     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3805
3806     my $bootindex_hash = {};
3807     my $i = 1;
3808     foreach my $o (split(//, $bootorder)) {
3809         $bootindex_hash->{$o} = $i*100;
3810         $i++;
3811     }
3812
3813     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3814
3815     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3816
3817     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3818
3819     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3820         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3821         my $socket = vnc_socket($vmid);
3822         push @$cmd,  '-vnc', "unix:$socket,password";
3823     } else {
3824         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3825         push @$cmd, '-nographic';
3826     }
3827
3828     # time drift fix
3829     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3830
3831     my $useLocaltime = $conf->{localtime};
3832
3833     if ($winversion >= 5) { # windows
3834         $useLocaltime = 1 if !defined($conf->{localtime});
3835
3836         # use time drift fix when acpi is enabled
3837         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3838             $tdf = 1 if !defined($conf->{tdf});
3839         }
3840     }
3841
3842     if ($winversion >= 6) {
3843         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3844         push @$cmd, '-no-hpet';
3845     }
3846
3847     push @$rtcFlags, 'driftfix=slew' if $tdf;
3848
3849     if (!$kvm) {
3850         push @$machineFlags, 'accel=tcg';
3851     }
3852
3853     if ($machine_type) {
3854         push @$machineFlags, "type=${machine_type}";
3855     }
3856
3857     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3858         push @$rtcFlags, "base=$conf->{startdate}";
3859     } elsif ($useLocaltime) {
3860         push @$rtcFlags, 'base=localtime';
3861     }
3862
3863     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3864
3865     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3866
3867     push @$cmd, '-S' if $conf->{freeze};
3868
3869     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3870
3871     # enable sound
3872     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3873     #push @$cmd, '-soundhw', 'es1370';
3874     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3875
3876     if (parse_guest_agent($conf)->{enabled}) {
3877         my $qgasocket = qmp_socket($vmid, 1);
3878         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3879         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3880         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3881         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3882     }
3883
3884     my $spice_port;
3885
3886     if ($qxlnum) {
3887         if ($qxlnum > 1) {
3888             if ($winversion){
3889                 for(my $i = 1; $i < $qxlnum; $i++){
3890                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3891                 }
3892             } else {
3893                 # assume other OS works like Linux
3894                 my ($ram, $vram) = ("134217728", "67108864");
3895                 if ($vga->{memory}) {
3896                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3897                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3898                 }
3899                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3900                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3901             }
3902         }
3903
3904         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3905
3906         my $nodename = PVE::INotify::nodename();
3907         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3908         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3909         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3910         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3911         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3912
3913         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3914
3915         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3916         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3917         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3918     }
3919
3920     # enable balloon by default, unless explicitly disabled
3921     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3922         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3923         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3924     }
3925
3926     if ($conf->{watchdog}) {
3927         my $wdopts = parse_watchdog($conf->{watchdog});
3928         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3929         my $watchdog = $wdopts->{model} || 'i6300esb';
3930         push @$devices, '-device', "$watchdog$pciaddr";
3931         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3932     }
3933
3934     my $vollist = [];
3935     my $scsicontroller = {};
3936     my $ahcicontroller = {};
3937     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3938
3939     # Add iscsi initiator name if available
3940     if (my $initiator = get_initiator_name()) {
3941         push @$devices, '-iscsi', "initiator-name=$initiator";
3942     }
3943
3944     foreach_drive($conf, sub {
3945         my ($ds, $drive) = @_;
3946
3947         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3948             push @$vollist, $drive->{file};
3949         }
3950
3951         # ignore efidisk here, already added in bios/fw handling code above
3952         return if $drive->{interface} eq 'efidisk';
3953
3954         $use_virtio = 1 if $ds =~ m/^virtio/;
3955
3956         if (drive_is_cdrom ($drive)) {
3957             if ($bootindex_hash->{d}) {
3958                 $drive->{bootindex} = $bootindex_hash->{d};
3959                 $bootindex_hash->{d} += 1;
3960             }
3961         } else {
3962             if ($bootindex_hash->{c}) {
3963                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3964                 $bootindex_hash->{c} += 1;
3965             }
3966         }
3967
3968         if($drive->{interface} eq 'virtio'){
3969            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3970         }
3971
3972         if ($drive->{interface} eq 'scsi') {
3973
3974             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3975
3976             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3977             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3978
3979             my $iothread = '';
3980             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3981                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3982                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3983             } elsif ($drive->{iothread}) {
3984                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3985             }
3986
3987             my $queues = '';
3988             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3989                 $queues = ",num_queues=$drive->{queues}";
3990             }
3991
3992             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3993             $scsicontroller->{$controller}=1;
3994         }
3995
3996         if ($drive->{interface} eq 'sata') {
3997            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3998            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3999            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4000            $ahcicontroller->{$controller}=1;
4001         }
4002
4003         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4004         push @$devices, '-drive',$drive_cmd;
4005         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4006     });
4007
4008     for (my $i = 0; $i < $MAX_NETS; $i++) {
4009          next if !$conf->{"net$i"};
4010          my $d = parse_net($conf->{"net$i"});
4011          next if !$d;
4012
4013          $use_virtio = 1 if $d->{model} eq 'virtio';
4014
4015          if ($bootindex_hash->{n}) {
4016             $d->{bootindex} = $bootindex_hash->{n};
4017             $bootindex_hash->{n} += 1;
4018          }
4019
4020          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4021          push @$devices, '-netdev', $netdevfull;
4022
4023          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4024          push @$devices, '-device', $netdevicefull;
4025     }
4026
4027     if ($conf->{ivshmem}) {
4028         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4029
4030         my $bus;
4031         if ($q35) {
4032             $bus = print_pcie_addr("ivshmem");
4033         } else {
4034             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4035         }
4036
4037         my $ivshmem_name = $ivshmem->{name} // $vmid;
4038         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4039
4040         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4042     }
4043
4044     if (!$q35) {
4045         # add pci bridges
4046         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4047            $bridges->{1} = 1;
4048            $bridges->{2} = 1;
4049         }
4050
4051         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4052
4053         while (my ($k, $v) = each %$bridges) {
4054             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4055             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4056         }
4057     }
4058
4059     push @$cmd, @$devices;
4060     push @$cmd, '-rtc', join(',', @$rtcFlags)
4061         if scalar(@$rtcFlags);
4062     push @$cmd, '-machine', join(',', @$machineFlags)
4063         if scalar(@$machineFlags);
4064     push @$cmd, '-global', join(',', @$globalFlags)
4065         if scalar(@$globalFlags);
4066
4067     if (my $vmstate = $conf->{vmstate}) {
4068         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4069         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4070         push @$cmd, '-loadstate', $statepath;
4071     }
4072
4073     # add custom args
4074     if ($conf->{args}) {
4075         my $aa = PVE::Tools::split_args($conf->{args});
4076         push @$cmd, @$aa;
4077     }
4078
4079     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4080 }
4081
4082 sub vnc_socket {
4083     my ($vmid) = @_;
4084     return "${var_run_tmpdir}/$vmid.vnc";
4085 }
4086
4087 sub spice_port {
4088     my ($vmid) = @_;
4089
4090     my $res = vm_mon_cmd($vmid, 'query-spice');
4091
4092     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4093 }
4094
4095 sub qmp_socket {
4096     my ($vmid, $qga, $name) = @_;
4097     my $sockettype = $qga ? 'qga' : 'qmp';
4098     my $ext = $name ? '-'.$name : '';
4099     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4100 }
4101
4102 sub pidfile_name {
4103     my ($vmid) = @_;
4104     return "${var_run_tmpdir}/$vmid.pid";
4105 }
4106
4107 sub vm_devices_list {
4108     my ($vmid) = @_;
4109
4110     my $res = vm_mon_cmd($vmid, 'query-pci');
4111     my $devices_to_check = [];
4112     my $devices = {};
4113     foreach my $pcibus (@$res) {
4114         push @$devices_to_check, @{$pcibus->{devices}},
4115     }
4116
4117     while (@$devices_to_check) {
4118         my $to_check = [];
4119         for my $d (@$devices_to_check) {
4120             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4121             next if !$d->{'pci_bridge'};
4122
4123             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4124             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4125         }
4126         $devices_to_check = $to_check;
4127     }
4128
4129     my $resblock = vm_mon_cmd($vmid, 'query-block');
4130     foreach my $block (@$resblock) {
4131         if($block->{device} =~ m/^drive-(\S+)/){
4132                 $devices->{$1} = 1;
4133         }
4134     }
4135
4136     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4137     foreach my $mice (@$resmice) {
4138         if ($mice->{name} eq 'QEMU HID Tablet') {
4139             $devices->{tablet} = 1;
4140             last;
4141         }
4142     }
4143
4144     # for usb devices there is no query-usb
4145     # but we can iterate over the entries in
4146     # qom-list path=/machine/peripheral
4147     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4148     foreach my $per (@$resperipheral) {
4149         if ($per->{name} =~ m/^usb\d+$/) {
4150             $devices->{$per->{name}} = 1;
4151         }
4152     }
4153
4154     return $devices;
4155 }
4156
4157 sub vm_deviceplug {
4158     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4159
4160     my $q35 = machine_type_is_q35($conf);
4161
4162     my $devices_list = vm_devices_list($vmid);
4163     return 1 if defined($devices_list->{$deviceid});
4164
4165     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4166
4167     if ($deviceid eq 'tablet') {
4168
4169         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4170
4171     } elsif ($deviceid eq 'keyboard') {
4172
4173         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4174
4175     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4176
4177         die "usb hotplug currently not reliable\n";
4178         # since we can't reliably hot unplug all added usb devices
4179         # and usb passthrough disables live migration
4180         # we disable usb hotplugging for now
4181         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4182
4183     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4184
4185         qemu_iothread_add($vmid, $deviceid, $device);
4186
4187         qemu_driveadd($storecfg, $vmid, $device);
4188         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4189
4190         qemu_deviceadd($vmid, $devicefull);
4191         eval { qemu_deviceaddverify($vmid, $deviceid); };
4192         if (my $err = $@) {
4193             eval { qemu_drivedel($vmid, $deviceid); };
4194             warn $@ if $@;
4195             die $err;
4196         }
4197
4198     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4199
4200
4201         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4202         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4203         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4204
4205         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4206
4207         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4208             qemu_iothread_add($vmid, $deviceid, $device);
4209             $devicefull .= ",iothread=iothread-$deviceid";
4210         }
4211
4212         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4213             $devicefull .= ",num_queues=$device->{queues}";
4214         }
4215
4216         qemu_deviceadd($vmid, $devicefull);
4217         qemu_deviceaddverify($vmid, $deviceid);
4218
4219     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4220
4221         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4222         qemu_driveadd($storecfg, $vmid, $device);
4223
4224         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4225         eval { qemu_deviceadd($vmid, $devicefull); };
4226         if (my $err = $@) {
4227             eval { qemu_drivedel($vmid, $deviceid); };
4228             warn $@ if $@;
4229             die $err;
4230         }
4231
4232     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4233
4234         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4235
4236         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4237         my $use_old_bios_files = undef;
4238         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4239
4240         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4241         qemu_deviceadd($vmid, $netdevicefull);
4242         eval {
4243             qemu_deviceaddverify($vmid, $deviceid);
4244             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4245         };
4246         if (my $err = $@) {
4247             eval { qemu_netdevdel($vmid, $deviceid); };
4248             warn $@ if $@;
4249             die $err;
4250         }
4251
4252     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4253
4254         my $bridgeid = $2;
4255         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4256         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4257
4258         qemu_deviceadd($vmid, $devicefull);
4259         qemu_deviceaddverify($vmid, $deviceid);
4260
4261     } else {
4262         die "can't hotplug device '$deviceid'\n";
4263     }
4264
4265     return 1;
4266 }
4267
4268 # fixme: this should raise exceptions on error!
4269 sub vm_deviceunplug {
4270     my ($vmid, $conf, $deviceid) = @_;
4271
4272     my $devices_list = vm_devices_list($vmid);
4273     return 1 if !defined($devices_list->{$deviceid});
4274
4275     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4276
4277     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4278
4279         qemu_devicedel($vmid, $deviceid);
4280
4281     } elsif ($deviceid =~ m/^usb\d+$/) {
4282
4283         die "usb hotplug currently not reliable\n";
4284         # when unplugging usb devices this way,
4285         # there may be remaining usb controllers/hubs
4286         # so we disable it for now
4287         qemu_devicedel($vmid, $deviceid);
4288         qemu_devicedelverify($vmid, $deviceid);
4289
4290     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4291
4292         qemu_devicedel($vmid, $deviceid);
4293         qemu_devicedelverify($vmid, $deviceid);
4294         qemu_drivedel($vmid, $deviceid);
4295         qemu_iothread_del($conf, $vmid, $deviceid);
4296
4297     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4298
4299         qemu_devicedel($vmid, $deviceid);
4300         qemu_devicedelverify($vmid, $deviceid);
4301         qemu_iothread_del($conf, $vmid, $deviceid);
4302
4303     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4304
4305         qemu_devicedel($vmid, $deviceid);
4306         qemu_drivedel($vmid, $deviceid);
4307         qemu_deletescsihw($conf, $vmid, $deviceid);
4308
4309     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4310
4311         qemu_devicedel($vmid, $deviceid);
4312         qemu_devicedelverify($vmid, $deviceid);
4313         qemu_netdevdel($vmid, $deviceid);
4314
4315     } else {
4316         die "can't unplug device '$deviceid'\n";
4317     }
4318
4319     return 1;
4320 }
4321
4322 sub qemu_deviceadd {
4323     my ($vmid, $devicefull) = @_;
4324
4325     $devicefull = "driver=".$devicefull;
4326     my %options =  split(/[=,]/, $devicefull);
4327
4328     vm_mon_cmd($vmid, "device_add" , %options);
4329 }
4330
4331 sub qemu_devicedel {
4332     my ($vmid, $deviceid) = @_;
4333
4334     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4335 }
4336
4337 sub qemu_iothread_add {
4338     my($vmid, $deviceid, $device) = @_;
4339
4340     if ($device->{iothread}) {
4341         my $iothreads = vm_iothreads_list($vmid);
4342         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4343     }
4344 }
4345
4346 sub qemu_iothread_del {
4347     my($conf, $vmid, $deviceid) = @_;
4348
4349     my $confid = $deviceid;
4350     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4351         $confid = 'scsi' . $1;
4352     }
4353     my $device = parse_drive($confid, $conf->{$confid});
4354     if ($device->{iothread}) {
4355         my $iothreads = vm_iothreads_list($vmid);
4356         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4357     }
4358 }
4359
4360 sub qemu_objectadd {
4361     my($vmid, $objectid, $qomtype) = @_;
4362
4363     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4364
4365     return 1;
4366 }
4367
4368 sub qemu_objectdel {
4369     my($vmid, $objectid) = @_;
4370
4371     vm_mon_cmd($vmid, "object-del", id => $objectid);
4372
4373     return 1;
4374 }
4375
4376 sub qemu_driveadd {
4377     my ($storecfg, $vmid, $device) = @_;
4378
4379     my $drive = print_drive_full($storecfg, $vmid, $device);
4380     $drive =~ s/\\/\\\\/g;
4381     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4382
4383     # If the command succeeds qemu prints: "OK"
4384     return 1 if $ret =~ m/OK/s;
4385
4386     die "adding drive failed: $ret\n";
4387 }
4388
4389 sub qemu_drivedel {
4390     my($vmid, $deviceid) = @_;
4391
4392     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4393     $ret =~ s/^\s+//;
4394
4395     return 1 if $ret eq "";
4396
4397     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4398     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4399
4400     die "deleting drive $deviceid failed : $ret\n";
4401 }
4402
4403 sub qemu_deviceaddverify {
4404     my ($vmid, $deviceid) = @_;
4405
4406     for (my $i = 0; $i <= 5; $i++) {
4407          my $devices_list = vm_devices_list($vmid);
4408          return 1 if defined($devices_list->{$deviceid});
4409          sleep 1;
4410     }
4411
4412     die "error on hotplug device '$deviceid'\n";
4413 }
4414
4415
4416 sub qemu_devicedelverify {
4417     my ($vmid, $deviceid) = @_;
4418
4419     # need to verify that the device is correctly removed as device_del
4420     # is async and empty return is not reliable
4421
4422     for (my $i = 0; $i <= 5; $i++) {
4423          my $devices_list = vm_devices_list($vmid);
4424          return 1 if !defined($devices_list->{$deviceid});
4425          sleep 1;
4426     }
4427
4428     die "error on hot-unplugging device '$deviceid'\n";
4429 }
4430
4431 sub qemu_findorcreatescsihw {
4432     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4433
4434     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4435
4436     my $scsihwid="$controller_prefix$controller";
4437     my $devices_list = vm_devices_list($vmid);
4438
4439     if(!defined($devices_list->{$scsihwid})) {
4440         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4441     }
4442
4443     return 1;
4444 }
4445
4446 sub qemu_deletescsihw {
4447     my ($conf, $vmid, $opt) = @_;
4448
4449     my $device = parse_drive($opt, $conf->{$opt});
4450
4451     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4452         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4453         return 1;
4454     }
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $devices_list = vm_devices_list($vmid);
4459     foreach my $opt (keys %{$devices_list}) {
4460         if (PVE::QemuServer::is_valid_drivename($opt)) {
4461             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4462             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4463                 return 1;
4464             }
4465         }
4466     }
4467
4468     my $scsihwid="scsihw$controller";
4469
4470     vm_deviceunplug($vmid, $conf, $scsihwid);
4471
4472     return 1;
4473 }
4474
4475 sub qemu_add_pci_bridge {
4476     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4477
4478     my $bridges = {};
4479
4480     my $bridgeid;
4481
4482     print_pci_addr($device, $bridges, $arch, $machine_type);
4483
4484     while (my ($k, $v) = each %$bridges) {
4485         $bridgeid = $k;
4486     }
4487     return 1 if !defined($bridgeid) || $bridgeid < 1;
4488
4489     my $bridge = "pci.$bridgeid";
4490     my $devices_list = vm_devices_list($vmid);
4491
4492     if (!defined($devices_list->{$bridge})) {
4493         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4494     }
4495
4496     return 1;
4497 }
4498
4499 sub qemu_set_link_status {
4500     my ($vmid, $device, $up) = @_;
4501
4502     vm_mon_cmd($vmid, "set_link", name => $device,
4503                up => $up ? JSON::true : JSON::false);
4504 }
4505
4506 sub qemu_netdevadd {
4507     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4508
4509     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4510     my %options =  split(/[=,]/, $netdev);
4511
4512     vm_mon_cmd($vmid, "netdev_add",  %options);
4513     return 1;
4514 }
4515
4516 sub qemu_netdevdel {
4517     my ($vmid, $deviceid) = @_;
4518
4519     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4520 }
4521
4522 sub qemu_usb_hotplug {
4523     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4524
4525     return if !$device;
4526
4527     # remove the old one first
4528     vm_deviceunplug($vmid, $conf, $deviceid);
4529
4530     # check if xhci controller is necessary and available
4531     if ($device->{usb3}) {
4532
4533         my $devicelist = vm_devices_list($vmid);
4534
4535         if (!$devicelist->{xhci}) {
4536             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4537             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4538         }
4539     }
4540     my $d = parse_usb_device($device->{host});
4541     $d->{usb3} = $device->{usb3};
4542
4543     # add the new one
4544     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4545 }
4546
4547 sub qemu_cpu_hotplug {
4548     my ($vmid, $conf, $vcpus) = @_;
4549
4550     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4551
4552     my $sockets = 1;
4553     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4554     $sockets = $conf->{sockets} if  $conf->{sockets};
4555     my $cores = $conf->{cores} || 1;
4556     my $maxcpus = $sockets * $cores;
4557
4558     $vcpus = $maxcpus if !$vcpus;
4559
4560     die "you can't add more vcpus than maxcpus\n"
4561         if $vcpus > $maxcpus;
4562
4563     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4564
4565     if ($vcpus < $currentvcpus) {
4566
4567         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4568
4569             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4570                 qemu_devicedel($vmid, "cpu$i");
4571                 my $retry = 0;
4572                 my $currentrunningvcpus = undef;
4573                 while (1) {
4574                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4575                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4576                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4577                     $retry++;
4578                     sleep 1;
4579                 }
4580                 #update conf after each succesfull cpu unplug
4581                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4582                 PVE::QemuConfig->write_config($vmid, $conf);
4583             }
4584         } else {
4585             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4586         }
4587
4588         return;
4589     }
4590
4591     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4592     die "vcpus in running vm does not match its configuration\n"
4593         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4594
4595     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4596
4597         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4598             my $cpustr = print_cpu_device($conf, $i);
4599             qemu_deviceadd($vmid, $cpustr);
4600
4601             my $retry = 0;
4602             my $currentrunningvcpus = undef;
4603             while (1) {
4604                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4605                 last if scalar(@{$currentrunningvcpus}) == $i;
4606                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4607                 sleep 1;
4608                 $retry++;
4609             }
4610             #update conf after each succesfull cpu hotplug
4611             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4612             PVE::QemuConfig->write_config($vmid, $conf);
4613         }
4614     } else {
4615
4616         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4617             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4618         }
4619     }
4620 }
4621
4622 sub qemu_block_set_io_throttle {
4623     my ($vmid, $deviceid,
4624         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4625         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4626         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4627         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4628
4629     return if !check_running($vmid) ;
4630
4631     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4632         bps => int($bps),
4633         bps_rd => int($bps_rd),
4634         bps_wr => int($bps_wr),
4635         iops => int($iops),
4636         iops_rd => int($iops_rd),
4637         iops_wr => int($iops_wr),
4638         bps_max => int($bps_max),
4639         bps_rd_max => int($bps_rd_max),
4640         bps_wr_max => int($bps_wr_max),
4641         iops_max => int($iops_max),
4642         iops_rd_max => int($iops_rd_max),
4643         iops_wr_max => int($iops_wr_max),
4644         bps_max_length => int($bps_max_length),
4645         bps_rd_max_length => int($bps_rd_max_length),
4646         bps_wr_max_length => int($bps_wr_max_length),
4647         iops_max_length => int($iops_max_length),
4648         iops_rd_max_length => int($iops_rd_max_length),
4649         iops_wr_max_length => int($iops_wr_max_length),
4650     );
4651
4652 }
4653
4654 # old code, only used to shutdown old VM after update
4655 sub __read_avail {
4656     my ($fh, $timeout) = @_;
4657
4658     my $sel = new IO::Select;
4659     $sel->add($fh);
4660
4661     my $res = '';
4662     my $buf;
4663
4664     my @ready;
4665     while (scalar (@ready = $sel->can_read($timeout))) {
4666         my $count;
4667         if ($count = $fh->sysread($buf, 8192)) {
4668             if ($buf =~ /^(.*)\(qemu\) $/s) {
4669                 $res .= $1;
4670                 last;
4671             } else {
4672                 $res .= $buf;
4673             }
4674         } else {
4675             if (!defined($count)) {
4676                 die "$!\n";
4677             }
4678             last;
4679         }
4680     }
4681
4682     die "monitor read timeout\n" if !scalar(@ready);
4683
4684     return $res;
4685 }
4686
4687 sub qemu_block_resize {
4688     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4689
4690     my $running = check_running($vmid);
4691
4692     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4693
4694     return if !$running;
4695
4696     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4697
4698 }
4699
4700 sub qemu_volume_snapshot {
4701     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4702
4703     my $running = check_running($vmid);
4704
4705     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4706         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4707     } else {
4708         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4709     }
4710 }
4711
4712 sub qemu_volume_snapshot_delete {
4713     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4714
4715     my $running = check_running($vmid);
4716
4717     if($running) {
4718
4719         $running = undef;
4720         my $conf = PVE::QemuConfig->load_config($vmid);
4721         foreach_drive($conf, sub {
4722             my ($ds, $drive) = @_;
4723             $running = 1 if $drive->{file} eq $volid;
4724         });
4725     }
4726
4727     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4728         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4729     } else {
4730         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4731     }
4732 }
4733
4734 sub set_migration_caps {
4735     my ($vmid) = @_;
4736
4737     my $cap_ref = [];
4738
4739     my $enabled_cap = {
4740         "auto-converge" => 1,
4741         "xbzrle" => 1,
4742         "x-rdma-pin-all" => 0,
4743         "zero-blocks" => 0,
4744         "compress" => 0
4745     };
4746
4747     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4748
4749     for my $supported_capability (@$supported_capabilities) {
4750         push @$cap_ref, {
4751             capability => $supported_capability->{capability},
4752             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4753         };
4754     }
4755
4756     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4757 }
4758
4759 my $fast_plug_option = {
4760     'lock' => 1,
4761     'name' => 1,
4762     'onboot' => 1,
4763     'shares' => 1,
4764     'startup' => 1,
4765     'description' => 1,
4766     'protection' => 1,
4767     'vmstatestorage' => 1,
4768     'hookscript' => 1,
4769 };
4770
4771 # hotplug changes in [PENDING]
4772 # $selection hash can be used to only apply specified options, for
4773 # example: { cores => 1 } (only apply changed 'cores')
4774 # $errors ref is used to return error messages
4775 sub vmconfig_hotplug_pending {
4776     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4777
4778     my $defaults = load_defaults();
4779     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4780
4781     # commit values which do not have any impact on running VM first
4782     # Note: those option cannot raise errors, we we do not care about
4783     # $selection and always apply them.
4784
4785     my $add_error = sub {
4786         my ($opt, $msg) = @_;
4787         $errors->{$opt} = "hotplug problem - $msg";
4788     };
4789
4790     my $changes = 0;
4791     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4792         if ($fast_plug_option->{$opt}) {
4793             $conf->{$opt} = $conf->{pending}->{$opt};
4794             delete $conf->{pending}->{$opt};
4795             $changes = 1;
4796         }
4797     }
4798
4799     if ($changes) {
4800         PVE::QemuConfig->write_config($vmid, $conf);
4801         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4802     }
4803
4804     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4805
4806     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4807     while (my ($opt, $force) = each %$pending_delete_hash) {
4808         next if $selection && !$selection->{$opt};
4809         eval {
4810             if ($opt eq 'hotplug') {
4811                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4812             } elsif ($opt eq 'tablet') {
4813                 die "skip\n" if !$hotplug_features->{usb};
4814                 if ($defaults->{tablet}) {
4815                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4816                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4817                         if $arch eq 'aarch64';
4818                 } else {
4819                     vm_deviceunplug($vmid, $conf, 'tablet');
4820                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4821                 }
4822             } elsif ($opt =~ m/^usb\d+/) {
4823                 die "skip\n";
4824                 # since we cannot reliably hot unplug usb devices
4825                 # we are disabling it
4826                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4827                 vm_deviceunplug($vmid, $conf, $opt);
4828             } elsif ($opt eq 'vcpus') {
4829                 die "skip\n" if !$hotplug_features->{cpu};
4830                 qemu_cpu_hotplug($vmid, $conf, undef);
4831             } elsif ($opt eq 'balloon') {
4832                 # enable balloon device is not hotpluggable
4833                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4834                 # here we reset the ballooning value to memory
4835                 my $balloon = $conf->{memory} || $defaults->{memory};
4836                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4837             } elsif ($fast_plug_option->{$opt}) {
4838                 # do nothing
4839             } elsif ($opt =~ m/^net(\d+)$/) {
4840                 die "skip\n" if !$hotplug_features->{network};
4841                 vm_deviceunplug($vmid, $conf, $opt);
4842             } elsif (is_valid_drivename($opt)) {
4843                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4844                 vm_deviceunplug($vmid, $conf, $opt);
4845                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4846             } elsif ($opt =~ m/^memory$/) {
4847                 die "skip\n" if !$hotplug_features->{memory};
4848                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4849             } elsif ($opt eq 'cpuunits') {
4850                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4851             } elsif ($opt eq 'cpulimit') {
4852                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4853             } else {
4854                 die "skip\n";
4855             }
4856         };
4857         if (my $err = $@) {
4858             &$add_error($opt, $err) if $err ne "skip\n";
4859         } else {
4860             # save new config if hotplug was successful
4861             delete $conf->{$opt};
4862             vmconfig_undelete_pending_option($conf, $opt);
4863             PVE::QemuConfig->write_config($vmid, $conf);
4864             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4865         }
4866     }
4867
4868     my $apply_pending_cloudinit;
4869     $apply_pending_cloudinit = sub {
4870         my ($key, $value) = @_;
4871         $apply_pending_cloudinit = sub {}; # once is enough
4872
4873         my @cloudinit_opts = keys %$confdesc_cloudinit;
4874         foreach my $opt (keys %{$conf->{pending}}) {
4875             next if !grep { $_ eq $opt } @cloudinit_opts;
4876             $conf->{$opt} = delete $conf->{pending}->{$opt};
4877         }
4878
4879         my $new_conf = { %$conf };
4880         $new_conf->{$key} = $value;
4881         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4882     };
4883
4884     foreach my $opt (keys %{$conf->{pending}}) {
4885         next if $selection && !$selection->{$opt};
4886         my $value = $conf->{pending}->{$opt};
4887         eval {
4888             if ($opt eq 'hotplug') {
4889                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4890             } elsif ($opt eq 'tablet') {
4891                 die "skip\n" if !$hotplug_features->{usb};
4892                 if ($value == 1) {
4893                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4894                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4895                         if $arch eq 'aarch64';
4896                 } elsif ($value == 0) {
4897                     vm_deviceunplug($vmid, $conf, 'tablet');
4898                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4899                 }
4900             } elsif ($opt =~ m/^usb\d+$/) {
4901                 die "skip\n";
4902                 # since we cannot reliably hot unplug usb devices
4903                 # we are disabling it
4904                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4905                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4906                 die "skip\n" if !$d;
4907                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4908             } elsif ($opt eq 'vcpus') {
4909                 die "skip\n" if !$hotplug_features->{cpu};
4910                 qemu_cpu_hotplug($vmid, $conf, $value);
4911             } elsif ($opt eq 'balloon') {
4912                 # enable/disable balloning device is not hotpluggable
4913                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4914                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4915                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4916
4917                 # allow manual ballooning if shares is set to zero
4918                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4919                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4920                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4921                 }
4922             } elsif ($opt =~ m/^net(\d+)$/) {
4923                 # some changes can be done without hotplug
4924                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4925                                     $vmid, $opt, $value, $arch, $machine_type);
4926             } elsif (is_valid_drivename($opt)) {
4927                 # some changes can be done without hotplug
4928                 my $drive = parse_drive($opt, $value);
4929                 if (drive_is_cloudinit($drive)) {
4930                     &$apply_pending_cloudinit($opt, $value);
4931                 }
4932                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4933                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4934             } elsif ($opt =~ m/^memory$/) { #dimms
4935                 die "skip\n" if !$hotplug_features->{memory};
4936                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4937             } elsif ($opt eq 'cpuunits') {
4938                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4939             } elsif ($opt eq 'cpulimit') {
4940                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4941                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4942             } else {
4943                 die "skip\n";  # skip non-hot-pluggable options
4944             }
4945         };
4946         if (my $err = $@) {
4947             &$add_error($opt, $err) if $err ne "skip\n";
4948         } else {
4949             # save new config if hotplug was successful
4950             $conf->{$opt} = $value;
4951             delete $conf->{pending}->{$opt};
4952             PVE::QemuConfig->write_config($vmid, $conf);
4953             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4954         }
4955     }
4956 }
4957
4958 sub try_deallocate_drive {
4959     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4960
4961     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4962         my $volid = $drive->{file};
4963         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4964             my $sid = PVE::Storage::parse_volume_id($volid);
4965             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4966
4967             # check if the disk is really unused
4968             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4969                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4970             PVE::Storage::vdisk_free($storecfg, $volid);
4971             return 1;
4972         } else {
4973             # If vm is not owner of this disk remove from config
4974             return 1;
4975         }
4976     }
4977
4978     return undef;
4979 }
4980
4981 sub vmconfig_delete_or_detach_drive {
4982     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4983
4984     my $drive = parse_drive($opt, $conf->{$opt});
4985
4986     my $rpcenv = PVE::RPCEnvironment::get();
4987     my $authuser = $rpcenv->get_user();
4988
4989     if ($force) {
4990         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4991         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4992     } else {
4993         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4994     }
4995 }
4996
4997 sub vmconfig_apply_pending {
4998     my ($vmid, $conf, $storecfg) = @_;
4999
5000     # cold plug
5001
5002     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5003     while (my ($opt, $force) = each %$pending_delete_hash) {
5004         die "internal error" if $opt =~ m/^unused/;
5005         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5006         if (!defined($conf->{$opt})) {
5007             vmconfig_undelete_pending_option($conf, $opt);
5008             PVE::QemuConfig->write_config($vmid, $conf);
5009         } elsif (is_valid_drivename($opt)) {
5010             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5011             vmconfig_undelete_pending_option($conf, $opt);
5012             delete $conf->{$opt};
5013             PVE::QemuConfig->write_config($vmid, $conf);
5014         } else {
5015             vmconfig_undelete_pending_option($conf, $opt);
5016             delete $conf->{$opt};
5017             PVE::QemuConfig->write_config($vmid, $conf);
5018         }
5019     }
5020
5021     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5022
5023     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5024         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5025
5026         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5027             # skip if nothing changed
5028         } elsif (is_valid_drivename($opt)) {
5029             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5030                 if defined($conf->{$opt});
5031             $conf->{$opt} = $conf->{pending}->{$opt};
5032         } else {
5033             $conf->{$opt} = $conf->{pending}->{$opt};
5034         }
5035
5036         delete $conf->{pending}->{$opt};
5037         PVE::QemuConfig->write_config($vmid, $conf);
5038     }
5039 }
5040
5041 my $safe_num_ne = sub {
5042     my ($a, $b) = @_;
5043
5044     return 0 if !defined($a) && !defined($b);
5045     return 1 if !defined($a);
5046     return 1 if !defined($b);
5047
5048     return $a != $b;
5049 };
5050
5051 my $safe_string_ne = sub {
5052     my ($a, $b) = @_;
5053
5054     return 0 if !defined($a) && !defined($b);
5055     return 1 if !defined($a);
5056     return 1 if !defined($b);
5057
5058     return $a ne $b;
5059 };
5060
5061 sub vmconfig_update_net {
5062     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5063
5064     my $newnet = parse_net($value);
5065
5066     if ($conf->{$opt}) {
5067         my $oldnet = parse_net($conf->{$opt});
5068
5069         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5070             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5071             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5072             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5073
5074             # for non online change, we try to hot-unplug
5075             die "skip\n" if !$hotplug;
5076             vm_deviceunplug($vmid, $conf, $opt);
5077         } else {
5078
5079             die "internal error" if $opt !~ m/net(\d+)/;
5080             my $iface = "tap${vmid}i$1";
5081
5082             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5083                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5084                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5085                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5086                 PVE::Network::tap_unplug($iface);
5087                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5088             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5089                 # Rate can be applied on its own but any change above needs to
5090                 # include the rate in tap_plug since OVS resets everything.
5091                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5092             }
5093
5094             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5095                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5096             }
5097
5098             return 1;
5099         }
5100     }
5101
5102     if ($hotplug) {
5103         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5104     } else {
5105         die "skip\n";
5106     }
5107 }
5108
5109 sub vmconfig_update_disk {
5110     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5111
5112     # fixme: do we need force?
5113
5114     my $drive = parse_drive($opt, $value);
5115
5116     if ($conf->{$opt}) {
5117
5118         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5119
5120             my $media = $drive->{media} || 'disk';
5121             my $oldmedia = $old_drive->{media} || 'disk';
5122             die "unable to change media type\n" if $media ne $oldmedia;
5123
5124             if (!drive_is_cdrom($old_drive)) {
5125
5126                 if ($drive->{file} ne $old_drive->{file}) {
5127
5128                     die "skip\n" if !$hotplug;
5129
5130                     # unplug and register as unused
5131                     vm_deviceunplug($vmid, $conf, $opt);
5132                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5133
5134                 } else {
5135                     # update existing disk
5136
5137                     # skip non hotpluggable value
5138                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5139                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5140                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5141                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5142                         die "skip\n";
5143                     }
5144
5145                     # apply throttle
5146                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5147                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5148                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5149                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5150                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5151                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5152                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5153                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5154                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5155                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5156                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5157                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5158                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5159                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5160                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5161                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5162                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5163                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5164
5165                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5166                                                    ($drive->{mbps} || 0)*1024*1024,
5167                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5168                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5169                                                    $drive->{iops} || 0,
5170                                                    $drive->{iops_rd} || 0,
5171                                                    $drive->{iops_wr} || 0,
5172                                                    ($drive->{mbps_max} || 0)*1024*1024,
5173                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5174                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5175                                                    $drive->{iops_max} || 0,
5176                                                    $drive->{iops_rd_max} || 0,
5177                                                    $drive->{iops_wr_max} || 0,
5178                                                    $drive->{bps_max_length} || 1,
5179                                                    $drive->{bps_rd_max_length} || 1,
5180                                                    $drive->{bps_wr_max_length} || 1,
5181                                                    $drive->{iops_max_length} || 1,
5182                                                    $drive->{iops_rd_max_length} || 1,
5183                                                    $drive->{iops_wr_max_length} || 1);
5184
5185                     }
5186
5187                     return 1;
5188                 }
5189
5190             } else { # cdrom
5191
5192                 if ($drive->{file} eq 'none') {
5193                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5194                     if (drive_is_cloudinit($old_drive)) {
5195                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5196                     }
5197                 } else {
5198                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5199                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5200                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5201                 }
5202
5203                 return 1;
5204             }
5205         }
5206     }
5207
5208     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5209     # hotplug new disks
5210     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5211     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5212 }
5213
5214 sub vm_start {
5215     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5216         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5217
5218     PVE::QemuConfig->lock_config($vmid, sub {
5219         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5220
5221         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5222
5223         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5224
5225         PVE::QemuConfig->check_lock($conf)
5226             if !($skiplock || $is_suspended);
5227
5228         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5229
5230         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5231             vmconfig_apply_pending($vmid, $conf, $storecfg);
5232             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5233         }
5234
5235         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5236
5237         my $defaults = load_defaults();
5238
5239         # set environment variable useful inside network script
5240         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5241
5242         my $local_volumes = {};
5243
5244         if ($targetstorage) {
5245             foreach_drive($conf, sub {
5246                 my ($ds, $drive) = @_;
5247
5248                 return if drive_is_cdrom($drive);
5249
5250                 my $volid = $drive->{file};
5251
5252                 return if !$volid;
5253
5254                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5255
5256                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5257                 return if $scfg->{shared};
5258                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5259             });
5260
5261             my $format = undef;
5262
5263             foreach my $opt (sort keys %$local_volumes) {
5264
5265                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5266                 my $drive = parse_drive($opt, $conf->{$opt});
5267
5268                 #if remote storage is specified, use default format
5269                 if ($targetstorage && $targetstorage ne "1") {
5270                     $storeid = $targetstorage;
5271                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5272                     $format = $defFormat;
5273                 } else {
5274                     #else we use same format than original
5275                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5276                     $format = qemu_img_format($scfg, $volid);
5277                 }
5278
5279                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5280                 my $newdrive = $drive;
5281                 $newdrive->{format} = $format;
5282                 $newdrive->{file} = $newvolid;
5283                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5284                 $local_volumes->{$opt} = $drivestr;
5285                 #pass drive to conf for command line
5286                 $conf->{$opt} = $drivestr;
5287             }
5288         }
5289
5290         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5291
5292         if ($is_suspended) {
5293             # enforce machine type on suspended vm to ensure HW compatibility
5294             $forcemachine = $conf->{runningmachine};
5295             print "Resuming suspended VM\n";
5296         }
5297
5298         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5299
5300         my $migrate_port = 0;
5301         my $migrate_uri;
5302         if ($statefile) {
5303             if ($statefile eq 'tcp') {
5304                 my $localip = "localhost";
5305                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5306                 my $nodename = PVE::INotify::nodename();
5307
5308                 if (!defined($migration_type)) {
5309                     if (defined($datacenterconf->{migration}->{type})) {
5310                         $migration_type = $datacenterconf->{migration}->{type};
5311                     } else {
5312                         $migration_type = 'secure';
5313                     }
5314                 }
5315
5316                 if ($migration_type eq 'insecure') {
5317                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5318                     if ($migrate_network_addr) {
5319                         $localip = $migrate_network_addr;
5320                     } else {
5321                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5322                     }
5323
5324                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5325                 }
5326
5327                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5328                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5329                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5330                 push @$cmd, '-incoming', $migrate_uri;
5331                 push @$cmd, '-S';
5332
5333             } elsif ($statefile eq 'unix') {
5334                 # should be default for secure migrations as a ssh TCP forward
5335                 # tunnel is not deterministic reliable ready and fails regurarly
5336                 # to set up in time, so use UNIX socket forwards
5337                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5338                 unlink $socket_addr;
5339
5340                 $migrate_uri = "unix:$socket_addr";
5341
5342                 push @$cmd, '-incoming', $migrate_uri;
5343                 push @$cmd, '-S';
5344
5345             } else {
5346                 push @$cmd, '-loadstate', $statefile;
5347             }
5348         } elsif ($paused) {
5349             push @$cmd, '-S';
5350         }
5351
5352         # host pci devices
5353         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5354           my $d = parse_hostpci($conf->{"hostpci$i"});
5355           next if !$d;
5356           my $pcidevices = $d->{pciid};
5357           foreach my $pcidevice (@$pcidevices) {
5358                 my $pciid = $pcidevice->{id};
5359
5360                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5361                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5362                 die "no pci device info for device '$pciid'\n" if !$info;
5363
5364                 if ($d->{mdev}) {
5365                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5366                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5367                 } else {
5368                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5369                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5370                     die "can't reset pci device '$pciid'\n"
5371                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5372                 }
5373           }
5374         }
5375
5376         PVE::Storage::activate_volumes($storecfg, $vollist);
5377
5378         eval {
5379             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5380                 outfunc => sub {}, errfunc => sub {});
5381         };
5382         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5383         # timeout should be more than enough here...
5384         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5385
5386         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5387                                                   : $defaults->{cpuunits};
5388
5389         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5390         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5391
5392         my %properties = (
5393             Slice => 'qemu.slice',
5394             KillMode => 'none',
5395             CPUShares => $cpuunits
5396         );
5397
5398         if (my $cpulimit = $conf->{cpulimit}) {
5399             $properties{CPUQuota} = int($cpulimit * 100);
5400         }
5401         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5402
5403         my $run_qemu = sub {
5404             PVE::Tools::run_fork sub {
5405                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5406                 run_command($cmd, %run_params);
5407             };
5408         };
5409
5410         if ($conf->{hugepages}) {
5411
5412             my $code = sub {
5413                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5414                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5415
5416                 PVE::QemuServer::Memory::hugepages_mount();
5417                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5418
5419                 eval { $run_qemu->() };
5420                 if (my $err = $@) {
5421                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5422                     die $err;
5423                 }
5424
5425                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5426             };
5427             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5428
5429         } else {
5430             eval { $run_qemu->() };
5431         }
5432
5433         if (my $err = $@) {
5434             # deactivate volumes if start fails
5435             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5436             die "start failed: $err";
5437         }
5438
5439         print "migration listens on $migrate_uri\n" if $migrate_uri;
5440
5441         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5442             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5443             warn $@ if $@;
5444         }
5445
5446         #start nbd server for storage migration
5447         if ($targetstorage) {
5448             my $nodename = PVE::INotify::nodename();
5449             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5450             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5451             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5452             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5453
5454             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5455
5456             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5457
5458             foreach my $opt (sort keys %$local_volumes) {
5459                 my $volid = $local_volumes->{$opt};
5460                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5461                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5462                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5463             }
5464         }
5465
5466         if ($migratedfrom) {
5467             eval {
5468                 set_migration_caps($vmid);
5469             };
5470             warn $@ if $@;
5471
5472             if ($spice_port) {
5473                 print "spice listens on port $spice_port\n";
5474                 if ($spice_ticket) {
5475                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5476                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5477                 }
5478             }
5479
5480         } else {
5481             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5482                 if !$statefile && $conf->{balloon};
5483
5484             foreach my $opt (keys %$conf) {
5485                 next if $opt !~  m/^net\d+$/;
5486                 my $nicconf = parse_net($conf->{$opt});
5487                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5488             }
5489         }
5490
5491         vm_mon_cmd_nocheck($vmid, 'qom-set',
5492                     path => "machine/peripheral/balloon0",
5493                     property => "guest-stats-polling-interval",
5494                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5495
5496         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5497             print "Resumed VM, removing state\n";
5498             delete $conf->@{qw(lock vmstate runningmachine)};
5499             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5500             PVE::Storage::vdisk_free($storecfg, $vmstate);
5501             PVE::QemuConfig->write_config($vmid, $conf);
5502         }
5503
5504         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5505     });
5506 }
5507
5508 sub vm_mon_cmd {
5509     my ($vmid, $execute, %params) = @_;
5510
5511     my $cmd = { execute => $execute, arguments => \%params };
5512     vm_qmp_command($vmid, $cmd);
5513 }
5514
5515 sub vm_mon_cmd_nocheck {
5516     my ($vmid, $execute, %params) = @_;
5517
5518     my $cmd = { execute => $execute, arguments => \%params };
5519     vm_qmp_command($vmid, $cmd, 1);
5520 }
5521
5522 sub vm_qmp_command {
5523     my ($vmid, $cmd, $nocheck) = @_;
5524
5525     my $res;
5526
5527     my $timeout;
5528     if ($cmd->{arguments}) {
5529         $timeout = delete $cmd->{arguments}->{timeout};
5530     }
5531
5532     eval {
5533         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5534         my $sname = qmp_socket($vmid);
5535         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5536             my $qmpclient = PVE::QMPClient->new();
5537
5538             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5539         } else {
5540             die "unable to open monitor socket\n";
5541         }
5542     };
5543     if (my $err = $@) {
5544         syslog("err", "VM $vmid qmp command failed - $err");
5545         die $err;
5546     }
5547
5548     return $res;
5549 }
5550
5551 sub vm_human_monitor_command {
5552     my ($vmid, $cmdline) = @_;
5553
5554     my $res;
5555
5556     my $cmd = {
5557         execute => 'human-monitor-command',
5558         arguments => { 'command-line' => $cmdline},
5559     };
5560
5561     return vm_qmp_command($vmid, $cmd);
5562 }
5563
5564 sub vm_commandline {
5565     my ($storecfg, $vmid, $snapname) = @_;
5566
5567     my $conf = PVE::QemuConfig->load_config($vmid);
5568
5569     if ($snapname) {
5570         my $snapshot = $conf->{snapshots}->{$snapname};
5571         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5572
5573         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5574
5575         $conf = $snapshot;
5576     }
5577
5578     my $defaults = load_defaults();
5579
5580     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5581
5582     return PVE::Tools::cmd2string($cmd);
5583 }
5584
5585 sub vm_reset {
5586     my ($vmid, $skiplock) = @_;
5587
5588     PVE::QemuConfig->lock_config($vmid, sub {
5589
5590         my $conf = PVE::QemuConfig->load_config($vmid);
5591
5592         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5593
5594         vm_mon_cmd($vmid, "system_reset");
5595     });
5596 }
5597
5598 sub get_vm_volumes {
5599     my ($conf) = @_;
5600
5601     my $vollist = [];
5602     foreach_volid($conf, sub {
5603         my ($volid, $attr) = @_;
5604
5605         return if $volid =~ m|^/|;
5606
5607         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5608         return if !$sid;
5609
5610         push @$vollist, $volid;
5611     });
5612
5613     return $vollist;
5614 }
5615
5616 sub vm_stop_cleanup {
5617     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5618
5619     eval {
5620
5621         if (!$keepActive) {
5622             my $vollist = get_vm_volumes($conf);
5623             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5624         }
5625
5626         foreach my $ext (qw(mon qmp pid vnc qga)) {
5627             unlink "/var/run/qemu-server/${vmid}.$ext";
5628         }
5629
5630         if ($conf->{ivshmem}) {
5631             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5632             # just delete it for now, VMs which have this already open do not
5633             # are affected, but new VMs will get a separated one. If this
5634             # becomes an issue we either add some sort of ref-counting or just
5635             # add a "don't delete on stop" flag to the ivshmem format.
5636             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5637         }
5638
5639         foreach my $key (keys %$conf) {
5640             next if $key !~ m/^hostpci(\d+)$/;
5641             my $hostpciindex = $1;
5642             my $d = parse_hostpci($conf->{$key});
5643             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5644
5645             foreach my $pci (@{$d->{pciid}}) {
5646                 my $pciid = $pci->{id};
5647                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5648             }
5649         }
5650
5651         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5652     };
5653     warn $@ if $@; # avoid errors - just warn
5654 }
5655
5656 # Note: use $nockeck to skip tests if VM configuration file exists.
5657 # We need that when migration VMs to other nodes (files already moved)
5658 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5659 sub vm_stop {
5660     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5661
5662     $force = 1 if !defined($force) && !$shutdown;
5663
5664     if ($migratedfrom){
5665         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5666         kill 15, $pid if $pid;
5667         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5668         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5669         return;
5670     }
5671
5672     PVE::QemuConfig->lock_config($vmid, sub {
5673
5674         my $pid = check_running($vmid, $nocheck);
5675         return if !$pid;
5676
5677         my $conf;
5678         if (!$nocheck) {
5679             $conf = PVE::QemuConfig->load_config($vmid);
5680             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5681             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5682                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5683                 $timeout = $opts->{down} if $opts->{down};
5684             }
5685             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5686         }
5687
5688         eval {
5689             if ($shutdown) {
5690                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5691                     vm_qmp_command($vmid, {
5692                         execute => "guest-shutdown",
5693                         arguments => { timeout => $timeout }
5694                     }, $nocheck);
5695                 } else {
5696                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5697                 }
5698             } else {
5699                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5700             }
5701         };
5702         my $err = $@;
5703
5704         if (!$err) {
5705             $timeout = 60 if !defined($timeout);
5706
5707             my $count = 0;
5708             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5709                 $count++;
5710                 sleep 1;
5711             }
5712
5713             if ($count >= $timeout) {
5714                 if ($force) {
5715                     warn "VM still running - terminating now with SIGTERM\n";
5716                     kill 15, $pid;
5717                 } else {
5718                     die "VM quit/powerdown failed - got timeout\n";
5719                 }
5720             } else {
5721                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5722                 return;
5723             }
5724         } else {
5725             if ($force) {
5726                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5727                 kill 15, $pid;
5728             } else {
5729                 die "VM quit/powerdown failed\n";
5730             }
5731         }
5732
5733         # wait again
5734         $timeout = 10;
5735
5736         my $count = 0;
5737         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5738             $count++;
5739             sleep 1;
5740         }
5741
5742         if ($count >= $timeout) {
5743             warn "VM still running - terminating now with SIGKILL\n";
5744             kill 9, $pid;
5745             sleep 1;
5746         }
5747
5748         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5749    });
5750 }
5751
5752 sub vm_suspend {
5753     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5754
5755     my $conf;
5756     my $path;
5757     my $storecfg;
5758     my $vmstate;
5759
5760     PVE::QemuConfig->lock_config($vmid, sub {
5761
5762         $conf = PVE::QemuConfig->load_config($vmid);
5763
5764         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5765         PVE::QemuConfig->check_lock($conf)
5766             if !($skiplock || $is_backing_up);
5767
5768         die "cannot suspend to disk during backup\n"
5769             if $is_backing_up && $includestate;
5770
5771         if ($includestate) {
5772             $conf->{lock} = 'suspending';
5773             my $date = strftime("%Y-%m-%d", localtime(time()));
5774             $storecfg = PVE::Storage::config();
5775             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5776             $path = PVE::Storage::path($storecfg, $vmstate);
5777             PVE::QemuConfig->write_config($vmid, $conf);
5778         } else {
5779             vm_mon_cmd($vmid, "stop");
5780         }
5781     });
5782
5783     if ($includestate) {
5784         # save vm state
5785         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5786
5787         eval {
5788             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5789             for(;;) {
5790                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5791                 if (!$state->{status}) {
5792                     die "savevm not active\n";
5793                 } elsif ($state->{status} eq 'active') {
5794                     sleep(1);
5795                     next;
5796                 } elsif ($state->{status} eq 'completed') {
5797                     print "State saved, quitting\n";
5798                     last;
5799                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5800                     die "query-savevm failed with error '$state->{error}'\n"
5801                 } else {
5802                     die "query-savevm returned status '$state->{status}'\n";
5803                 }
5804             }
5805         };
5806         my $err = $@;
5807
5808         PVE::QemuConfig->lock_config($vmid, sub {
5809             $conf = PVE::QemuConfig->load_config($vmid);
5810             if ($err) {
5811                 # cleanup, but leave suspending lock, to indicate something went wrong
5812                 eval {
5813                     vm_mon_cmd($vmid, "savevm-end");
5814                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5815                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5816                     delete $conf->@{qw(vmstate runningmachine)};
5817                     PVE::QemuConfig->write_config($vmid, $conf);
5818                 };
5819                 warn $@ if $@;
5820                 die $err;
5821             }
5822
5823             die "lock changed unexpectedly\n"
5824                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5825
5826             vm_qmp_command($vmid, { execute => "quit" });
5827             $conf->{lock} = 'suspended';
5828             PVE::QemuConfig->write_config($vmid, $conf);
5829         });
5830     }
5831 }
5832
5833 sub vm_resume {
5834     my ($vmid, $skiplock, $nocheck) = @_;
5835
5836     PVE::QemuConfig->lock_config($vmid, sub {
5837         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5838         my $res = $vm_mon_cmd->($vmid, 'query-status');
5839         my $resume_cmd = 'cont';
5840
5841         if ($res->{status} && $res->{status} eq 'suspended') {
5842             $resume_cmd = 'system_wakeup';
5843         }
5844
5845         if (!$nocheck) {
5846
5847             my $conf = PVE::QemuConfig->load_config($vmid);
5848
5849             PVE::QemuConfig->check_lock($conf)
5850                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5851         }
5852
5853         $vm_mon_cmd->($vmid, $resume_cmd);
5854     });
5855 }
5856
5857 sub vm_sendkey {
5858     my ($vmid, $skiplock, $key) = @_;
5859
5860     PVE::QemuConfig->lock_config($vmid, sub {
5861
5862         my $conf = PVE::QemuConfig->load_config($vmid);
5863
5864         # there is no qmp command, so we use the human monitor command
5865         vm_human_monitor_command($vmid, "sendkey $key");
5866     });
5867 }
5868
5869 sub vm_destroy {
5870     my ($storecfg, $vmid, $skiplock) = @_;
5871
5872     PVE::QemuConfig->lock_config($vmid, sub {
5873
5874         my $conf = PVE::QemuConfig->load_config($vmid);
5875
5876         if (!check_running($vmid)) {
5877             destroy_vm($storecfg, $vmid, undef, $skiplock);
5878         } else {
5879             die "VM $vmid is running - destroy failed\n";
5880         }
5881     });
5882 }
5883
5884 # vzdump restore implementaion
5885
5886 sub tar_archive_read_firstfile {
5887     my $archive = shift;
5888
5889     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5890
5891     # try to detect archive type first
5892     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5893         die "unable to open file '$archive'\n";
5894     my $firstfile = <$fh>;
5895     kill 15, $pid;
5896     close $fh;
5897
5898     die "ERROR: archive contaions no data\n" if !$firstfile;
5899     chomp $firstfile;
5900
5901     return $firstfile;
5902 }
5903
5904 sub tar_restore_cleanup {
5905     my ($storecfg, $statfile) = @_;
5906
5907     print STDERR "starting cleanup\n";
5908
5909     if (my $fd = IO::File->new($statfile, "r")) {
5910         while (defined(my $line = <$fd>)) {
5911             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5912                 my $volid = $2;
5913                 eval {
5914                     if ($volid =~ m|^/|) {
5915                         unlink $volid || die 'unlink failed\n';
5916                     } else {
5917                         PVE::Storage::vdisk_free($storecfg, $volid);
5918                     }
5919                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5920                 };
5921                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5922             } else {
5923                 print STDERR "unable to parse line in statfile - $line";
5924             }
5925         }
5926         $fd->close();
5927     }
5928 }
5929
5930 sub restore_archive {
5931     my ($archive, $vmid, $user, $opts) = @_;
5932
5933     my $format = $opts->{format};
5934     my $comp;
5935
5936     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5937         $format = 'tar' if !$format;
5938         $comp = 'gzip';
5939     } elsif ($archive =~ m/\.tar$/) {
5940         $format = 'tar' if !$format;
5941     } elsif ($archive =~ m/.tar.lzo$/) {
5942         $format = 'tar' if !$format;
5943         $comp = 'lzop';
5944     } elsif ($archive =~ m/\.vma$/) {
5945         $format = 'vma' if !$format;
5946     } elsif ($archive =~ m/\.vma\.gz$/) {
5947         $format = 'vma' if !$format;
5948         $comp = 'gzip';
5949     } elsif ($archive =~ m/\.vma\.lzo$/) {
5950         $format = 'vma' if !$format;
5951         $comp = 'lzop';
5952     } else {
5953         $format = 'vma' if !$format; # default
5954     }
5955
5956     # try to detect archive format
5957     if ($format eq 'tar') {
5958         return restore_tar_archive($archive, $vmid, $user, $opts);
5959     } else {
5960         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5961     }
5962 }
5963
5964 sub restore_update_config_line {
5965     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5966
5967     return if $line =~ m/^\#qmdump\#/;
5968     return if $line =~ m/^\#vzdump\#/;
5969     return if $line =~ m/^lock:/;
5970     return if $line =~ m/^unused\d+:/;
5971     return if $line =~ m/^parent:/;
5972
5973     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5974     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5975         # try to convert old 1.X settings
5976         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5977         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5978             my ($model, $macaddr) = split(/\=/, $devconfig);
5979             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5980             my $net = {
5981                 model => $model,
5982                 bridge => "vmbr$ind",
5983                 macaddr => $macaddr,
5984             };
5985             my $netstr = print_net($net);
5986
5987             print $outfd "net$cookie->{netcount}: $netstr\n";
5988             $cookie->{netcount}++;
5989         }
5990     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5991         my ($id, $netstr) = ($1, $2);
5992         my $net = parse_net($netstr);
5993         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5994         $netstr = print_net($net);
5995         print $outfd "$id: $netstr\n";
5996     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5997         my $virtdev = $1;
5998         my $value = $3;
5999         my $di = parse_drive($virtdev, $value);
6000         if (defined($di->{backup}) && !$di->{backup}) {
6001             print $outfd "#$line";
6002         } elsif ($map->{$virtdev}) {
6003             delete $di->{format}; # format can change on restore
6004             $di->{file} = $map->{$virtdev};
6005             $value = print_drive($vmid, $di);
6006             print $outfd "$virtdev: $value\n";
6007         } else {
6008             print $outfd $line;
6009         }
6010     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6011         my $vmgenid = $1;
6012         if ($vmgenid ne '0') {
6013             # always generate a new vmgenid if there was a valid one setup
6014             $vmgenid = generate_uuid();
6015         }
6016         print $outfd "vmgenid: $vmgenid\n";
6017     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6018         my ($uuid, $uuid_str);
6019         UUID::generate($uuid);
6020         UUID::unparse($uuid, $uuid_str);
6021         my $smbios1 = parse_smbios1($2);
6022         $smbios1->{uuid} = $uuid_str;
6023         print $outfd $1.print_smbios1($smbios1)."\n";
6024     } else {
6025         print $outfd $line;
6026     }
6027 }
6028
6029 sub scan_volids {
6030     my ($cfg, $vmid) = @_;
6031
6032     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6033
6034     my $volid_hash = {};
6035     foreach my $storeid (keys %$info) {
6036         foreach my $item (@{$info->{$storeid}}) {
6037             next if !($item->{volid} && $item->{size});
6038             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6039             $volid_hash->{$item->{volid}} = $item;
6040         }
6041     }
6042
6043     return $volid_hash;
6044 }
6045
6046 sub is_volume_in_use {
6047     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6048
6049     my $path = PVE::Storage::path($storecfg, $volid);
6050
6051     my $scan_config = sub {
6052         my ($cref, $snapname) = @_;
6053
6054         foreach my $key (keys %$cref) {
6055             my $value = $cref->{$key};
6056             if (is_valid_drivename($key)) {
6057                 next if $skip_drive && $key eq $skip_drive;
6058                 my $drive = parse_drive($key, $value);
6059                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6060                 return 1 if $volid eq $drive->{file};
6061                 if ($drive->{file} =~ m!^/!) {
6062                     return 1 if $drive->{file} eq $path;
6063                 } else {
6064                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6065                     next if !$storeid;
6066                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6067                     next if !$scfg;
6068                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6069                 }
6070             }
6071         }
6072
6073         return 0;
6074     };
6075
6076     return 1 if &$scan_config($conf);
6077
6078     undef $skip_drive;
6079
6080     foreach my $snapname (keys %{$conf->{snapshots}}) {
6081         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6082     }
6083
6084     return 0;
6085 }
6086
6087 sub update_disksize {
6088     my ($vmid, $conf, $volid_hash) = @_;
6089
6090     my $changes;
6091     my $prefix = "VM $vmid:";
6092
6093     # used and unused disks
6094     my $referenced = {};
6095
6096     # Note: it is allowed to define multiple storages with same path (alias), so
6097     # we need to check both 'volid' and real 'path' (two different volid can point
6098     # to the same path).
6099
6100     my $referencedpath = {};
6101
6102     # update size info
6103     foreach my $opt (keys %$conf) {
6104         if (is_valid_drivename($opt)) {
6105             my $drive = parse_drive($opt, $conf->{$opt});
6106             my $volid = $drive->{file};
6107             next if !$volid;
6108
6109             $referenced->{$volid} = 1;
6110             if ($volid_hash->{$volid} &&
6111                 (my $path = $volid_hash->{$volid}->{path})) {
6112                 $referencedpath->{$path} = 1;
6113             }
6114
6115             next if drive_is_cdrom($drive);
6116             next if !$volid_hash->{$volid};
6117
6118             $drive->{size} = $volid_hash->{$volid}->{size};
6119             my $new = print_drive($vmid, $drive);
6120             if ($new ne $conf->{$opt}) {
6121                 $changes = 1;
6122                 $conf->{$opt} = $new;
6123                 print "$prefix update disk '$opt' information.\n";
6124             }
6125         }
6126     }
6127
6128     # remove 'unusedX' entry if volume is used
6129     foreach my $opt (keys %$conf) {
6130         next if $opt !~ m/^unused\d+$/;
6131         my $volid = $conf->{$opt};
6132         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6133         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6134             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6135             $changes = 1;
6136             delete $conf->{$opt};
6137         }
6138
6139         $referenced->{$volid} = 1;
6140         $referencedpath->{$path} = 1 if $path;
6141     }
6142
6143     foreach my $volid (sort keys %$volid_hash) {
6144         next if $volid =~ m/vm-$vmid-state-/;
6145         next if $referenced->{$volid};
6146         my $path = $volid_hash->{$volid}->{path};
6147         next if !$path; # just to be sure
6148         next if $referencedpath->{$path};
6149         $changes = 1;
6150         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6151         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6152         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6153     }
6154
6155     return $changes;
6156 }
6157
6158 sub rescan {
6159     my ($vmid, $nolock, $dryrun) = @_;
6160
6161     my $cfg = PVE::Storage::config();
6162
6163     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6164     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6165     foreach my $stor (keys %{$cfg->{ids}}) {
6166         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6167     }
6168
6169     print "rescan volumes...\n";
6170     my $volid_hash = scan_volids($cfg, $vmid);
6171
6172     my $updatefn =  sub {
6173         my ($vmid) = @_;
6174
6175         my $conf = PVE::QemuConfig->load_config($vmid);
6176
6177         PVE::QemuConfig->check_lock($conf);
6178
6179         my $vm_volids = {};
6180         foreach my $volid (keys %$volid_hash) {
6181             my $info = $volid_hash->{$volid};
6182             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6183         }
6184
6185         my $changes = update_disksize($vmid, $conf, $vm_volids);
6186
6187         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6188     };
6189
6190     if (defined($vmid)) {
6191         if ($nolock) {
6192             &$updatefn($vmid);
6193         } else {
6194             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6195         }
6196     } else {
6197         my $vmlist = config_list();
6198         foreach my $vmid (keys %$vmlist) {
6199             if ($nolock) {
6200                 &$updatefn($vmid);
6201             } else {
6202                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6203             }
6204         }
6205     }
6206 }
6207
6208 sub restore_vma_archive {
6209     my ($archive, $vmid, $user, $opts, $comp) = @_;
6210
6211     my $readfrom = $archive;
6212
6213     my $cfg = PVE::Storage::config();
6214     my $commands = [];
6215     my $bwlimit = $opts->{bwlimit};
6216
6217     my $dbg_cmdstring = '';
6218     my $add_pipe = sub {
6219         my ($cmd) = @_;
6220         push @$commands, $cmd;
6221         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6222         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6223         $readfrom = '-';
6224     };
6225
6226     my $input = undef;
6227     if ($archive eq '-') {
6228         $input = '<&STDIN';
6229     } else {
6230         # If we use a backup from a PVE defined storage we also consider that
6231         # storage's rate limit:
6232         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6233         if (defined($volid)) {
6234             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6235             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6236             if ($readlimit) {
6237                 print STDERR "applying read rate limit: $readlimit\n";
6238                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6239                 $add_pipe->($cstream);
6240             }
6241         }
6242     }
6243
6244     if ($comp) {
6245         my $cmd;
6246         if ($comp eq 'gzip') {
6247             $cmd = ['zcat', $readfrom];
6248         } elsif ($comp eq 'lzop') {
6249             $cmd = ['lzop', '-d', '-c', $readfrom];
6250         } else {
6251             die "unknown compression method '$comp'\n";
6252         }
6253         $add_pipe->($cmd);
6254     }
6255
6256     my $tmpdir = "/var/tmp/vzdumptmp$$";
6257     rmtree $tmpdir;
6258
6259     # disable interrupts (always do cleanups)
6260     local $SIG{INT} =
6261         local $SIG{TERM} =
6262         local $SIG{QUIT} =
6263         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6264
6265     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6266     POSIX::mkfifo($mapfifo, 0600);
6267     my $fifofh;
6268
6269     my $openfifo = sub {
6270         open($fifofh, '>', $mapfifo) || die $!;
6271     };
6272
6273     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6274
6275     my $oldtimeout;
6276     my $timeout = 5;
6277
6278     my $devinfo = {};
6279
6280     my $rpcenv = PVE::RPCEnvironment::get();
6281
6282     my $conffile = PVE::QemuConfig->config_file($vmid);
6283     my $tmpfn = "$conffile.$$.tmp";
6284
6285     # Note: $oldconf is undef if VM does not exists
6286     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6287     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6288
6289     my %storage_limits;
6290
6291     my $print_devmap = sub {
6292         my $virtdev_hash = {};
6293
6294         my $cfgfn = "$tmpdir/qemu-server.conf";
6295
6296         # we can read the config - that is already extracted
6297         my $fh = IO::File->new($cfgfn, "r") ||
6298             "unable to read qemu-server.conf - $!\n";
6299
6300         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6301         if (-f $fwcfgfn) {
6302             my $pve_firewall_dir = '/etc/pve/firewall';
6303             mkdir $pve_firewall_dir; # make sure the dir exists
6304             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6305         }
6306
6307         while (defined(my $line = <$fh>)) {
6308             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6309                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6310                 die "archive does not contain data for drive '$virtdev'\n"
6311                     if !$devinfo->{$devname};
6312                 if (defined($opts->{storage})) {
6313                     $storeid = $opts->{storage} || 'local';
6314                 } elsif (!$storeid) {
6315                     $storeid = 'local';
6316                 }
6317                 $format = 'raw' if !$format;
6318                 $devinfo->{$devname}->{devname} = $devname;
6319                 $devinfo->{$devname}->{virtdev} = $virtdev;
6320                 $devinfo->{$devname}->{format} = $format;
6321                 $devinfo->{$devname}->{storeid} = $storeid;
6322
6323                 # check permission on storage
6324                 my $pool = $opts->{pool}; # todo: do we need that?
6325                 if ($user ne 'root@pam') {
6326                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6327                 }
6328
6329                 $storage_limits{$storeid} = $bwlimit;
6330
6331                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6332             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6333                 my $virtdev = $1;
6334                 my $drive = parse_drive($virtdev, $2);
6335                 if (drive_is_cloudinit($drive)) {
6336                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6337                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6338                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6339
6340                     my $d = {
6341                         format => $format,
6342                         storeid => $opts->{storage} // $storeid,
6343                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6344                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6345                         name => "vm-$vmid-cloudinit",
6346                         is_cloudinit => 1,
6347                     };
6348                     $virtdev_hash->{$virtdev} = $d;
6349                 }
6350             }
6351         }
6352
6353         foreach my $key (keys %storage_limits) {
6354             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6355             next if !$limit;
6356             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6357             $storage_limits{$key} = $limit * 1024;
6358         }
6359
6360         foreach my $devname (keys %$devinfo) {
6361             die "found no device mapping information for device '$devname'\n"
6362                 if !$devinfo->{$devname}->{virtdev};
6363         }
6364
6365         # create empty/temp config
6366         if ($oldconf) {
6367             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6368             foreach_drive($oldconf, sub {
6369                 my ($ds, $drive) = @_;
6370
6371                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6372
6373                 my $volid = $drive->{file};
6374                 return if !$volid || $volid =~ m|^/|;
6375
6376                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6377                 return if !$path || !$owner || ($owner != $vmid);
6378
6379                 # Note: only delete disk we want to restore
6380                 # other volumes will become unused
6381                 if ($virtdev_hash->{$ds}) {
6382                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6383                     if (my $err = $@) {
6384                         warn $err;
6385                     }
6386                 }
6387             });
6388
6389             # delete vmstate files, after the restore we have no snapshots anymore
6390             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6391                 my $snap = $oldconf->{snapshots}->{$snapname};
6392                 if ($snap->{vmstate}) {
6393                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6394                     if (my $err = $@) {
6395                         warn $err;
6396                     }
6397                 }
6398             }
6399         }
6400
6401         my $map = {};
6402         foreach my $virtdev (sort keys %$virtdev_hash) {
6403             my $d = $virtdev_hash->{$virtdev};
6404             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6405             my $storeid = $d->{storeid};
6406             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6407
6408             my $map_opts = '';
6409             if (my $limit = $storage_limits{$storeid}) {
6410                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6411             }
6412
6413             # test if requested format is supported
6414             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6415             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6416             $d->{format} = $defFormat if !$supported;
6417
6418             my $name;
6419             if ($d->{is_cloudinit}) {
6420                 $name = $d->{name};
6421                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6422             }
6423
6424             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6425             print STDERR "new volume ID is '$volid'\n";
6426             $d->{volid} = $volid;
6427
6428             PVE::Storage::activate_volumes($cfg, [$volid]);
6429
6430             my $write_zeros = 1;
6431             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6432                 $write_zeros = 0;
6433             }
6434
6435             if (!$d->{is_cloudinit}) {
6436                 my $path = PVE::Storage::path($cfg, $volid);
6437
6438                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6439
6440                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6441             }
6442             $map->{$virtdev} = $volid;
6443         }
6444
6445         $fh->seek(0, 0) || die "seek failed - $!\n";
6446
6447         my $outfd = new IO::File ($tmpfn, "w") ||
6448             die "unable to write config for VM $vmid\n";
6449
6450         my $cookie = { netcount => 0 };
6451         while (defined(my $line = <$fh>)) {
6452             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6453         }
6454
6455         $fh->close();
6456         $outfd->close();
6457     };
6458
6459     eval {
6460         # enable interrupts
6461         local $SIG{INT} =
6462             local $SIG{TERM} =
6463             local $SIG{QUIT} =
6464             local $SIG{HUP} =
6465             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6466         local $SIG{ALRM} = sub { die "got timeout\n"; };
6467
6468         $oldtimeout = alarm($timeout);
6469
6470         my $parser = sub {
6471             my $line = shift;
6472
6473             print "$line\n";
6474
6475             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6476                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6477                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6478             } elsif ($line =~ m/^CTIME: /) {
6479                 # we correctly received the vma config, so we can disable
6480                 # the timeout now for disk allocation (set to 10 minutes, so
6481                 # that we always timeout if something goes wrong)
6482                 alarm(600);
6483                 &$print_devmap();
6484                 print $fifofh "done\n";
6485                 my $tmp = $oldtimeout || 0;
6486                 $oldtimeout = undef;
6487                 alarm($tmp);
6488                 close($fifofh);
6489             }
6490         };
6491
6492         print "restore vma archive: $dbg_cmdstring\n";
6493         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6494     };
6495     my $err = $@;
6496
6497     alarm($oldtimeout) if $oldtimeout;
6498
6499     my $vollist = [];
6500     foreach my $devname (keys %$devinfo) {
6501         my $volid = $devinfo->{$devname}->{volid};
6502         push @$vollist, $volid if $volid;
6503     }
6504
6505     PVE::Storage::deactivate_volumes($cfg, $vollist);
6506
6507     unlink $mapfifo;
6508
6509     if ($err) {
6510         rmtree $tmpdir;
6511         unlink $tmpfn;
6512
6513         foreach my $devname (keys %$devinfo) {
6514             my $volid = $devinfo->{$devname}->{volid};
6515             next if !$volid;
6516             eval {
6517                 if ($volid =~ m|^/|) {
6518                     unlink $volid || die 'unlink failed\n';
6519                 } else {
6520                     PVE::Storage::vdisk_free($cfg, $volid);
6521                 }
6522                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6523             };
6524             print STDERR "unable to cleanup '$volid' - $@" if $@;
6525         }
6526         die $err;
6527     }
6528
6529     rmtree $tmpdir;
6530
6531     rename($tmpfn, $conffile) ||
6532         die "unable to commit configuration file '$conffile'\n";
6533
6534     PVE::Cluster::cfs_update(); # make sure we read new file
6535
6536     eval { rescan($vmid, 1); };
6537     warn $@ if $@;
6538 }
6539
6540 sub restore_tar_archive {
6541     my ($archive, $vmid, $user, $opts) = @_;
6542
6543     if ($archive ne '-') {
6544         my $firstfile = tar_archive_read_firstfile($archive);
6545         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6546             if $firstfile ne 'qemu-server.conf';
6547     }
6548
6549     my $storecfg = PVE::Storage::config();
6550
6551     # destroy existing data - keep empty config
6552     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6553     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6554
6555     my $tocmd = "/usr/lib/qemu-server/qmextract";
6556
6557     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6558     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6559     $tocmd .= ' --prealloc' if $opts->{prealloc};
6560     $tocmd .= ' --info' if $opts->{info};
6561
6562     # tar option "xf" does not autodetect compression when read from STDIN,
6563     # so we pipe to zcat
6564     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6565         PVE::Tools::shellquote("--to-command=$tocmd");
6566
6567     my $tmpdir = "/var/tmp/vzdumptmp$$";
6568     mkpath $tmpdir;
6569
6570     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6571     local $ENV{VZDUMP_VMID} = $vmid;
6572     local $ENV{VZDUMP_USER} = $user;
6573
6574     my $conffile = PVE::QemuConfig->config_file($vmid);
6575     my $tmpfn = "$conffile.$$.tmp";
6576
6577     # disable interrupts (always do cleanups)
6578     local $SIG{INT} =
6579         local $SIG{TERM} =
6580         local $SIG{QUIT} =
6581         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6582
6583     eval {
6584         # enable interrupts
6585         local $SIG{INT} =
6586             local $SIG{TERM} =
6587             local $SIG{QUIT} =
6588             local $SIG{HUP} =
6589             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6590
6591         if ($archive eq '-') {
6592             print "extracting archive from STDIN\n";
6593             run_command($cmd, input => "<&STDIN");
6594         } else {
6595             print "extracting archive '$archive'\n";
6596             run_command($cmd);
6597         }
6598
6599         return if $opts->{info};
6600
6601         # read new mapping
6602         my $map = {};
6603         my $statfile = "$tmpdir/qmrestore.stat";
6604         if (my $fd = IO::File->new($statfile, "r")) {
6605             while (defined (my $line = <$fd>)) {
6606                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6607                     $map->{$1} = $2 if $1;
6608                 } else {
6609                     print STDERR "unable to parse line in statfile - $line\n";
6610                 }
6611             }
6612             $fd->close();
6613         }
6614
6615         my $confsrc = "$tmpdir/qemu-server.conf";
6616
6617         my $srcfd = new IO::File($confsrc, "r") ||
6618             die "unable to open file '$confsrc'\n";
6619
6620         my $outfd = new IO::File ($tmpfn, "w") ||
6621             die "unable to write config for VM $vmid\n";
6622
6623         my $cookie = { netcount => 0 };
6624         while (defined (my $line = <$srcfd>)) {
6625             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6626         }
6627
6628         $srcfd->close();
6629         $outfd->close();
6630     };
6631     my $err = $@;
6632
6633     if ($err) {
6634
6635         unlink $tmpfn;
6636
6637         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6638
6639         die $err;
6640     }
6641
6642     rmtree $tmpdir;
6643
6644     rename $tmpfn, $conffile ||
6645         die "unable to commit configuration file '$conffile'\n";
6646
6647     PVE::Cluster::cfs_update(); # make sure we read new file
6648
6649     eval { rescan($vmid, 1); };
6650     warn $@ if $@;
6651 };
6652
6653 sub foreach_storage_used_by_vm {
6654     my ($conf, $func) = @_;
6655
6656     my $sidhash = {};
6657
6658     foreach_drive($conf, sub {
6659         my ($ds, $drive) = @_;
6660         return if drive_is_cdrom($drive);
6661
6662         my $volid = $drive->{file};
6663
6664         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6665         $sidhash->{$sid} = $sid if $sid;
6666     });
6667
6668     foreach my $sid (sort keys %$sidhash) {
6669         &$func($sid);
6670     }
6671 }
6672
6673 sub do_snapshots_with_qemu {
6674     my ($storecfg, $volid) = @_;
6675
6676     my $storage_name = PVE::Storage::parse_volume_id($volid);
6677     my $scfg = $storecfg->{ids}->{$storage_name};
6678
6679     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6680         return 1;
6681     }
6682
6683     if ($volid =~ m/\.(qcow2|qed)$/){
6684         return 1;
6685     }
6686
6687     return undef;
6688 }
6689
6690 sub qga_check_running {
6691     my ($vmid, $nowarn) = @_;
6692
6693     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6694     if ($@) {
6695         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6696         return 0;
6697     }
6698     return 1;
6699 }
6700
6701 sub template_create {
6702     my ($vmid, $conf, $disk) = @_;
6703
6704     my $storecfg = PVE::Storage::config();
6705
6706     foreach_drive($conf, sub {
6707         my ($ds, $drive) = @_;
6708
6709         return if drive_is_cdrom($drive);
6710         return if $disk && $ds ne $disk;
6711
6712         my $volid = $drive->{file};
6713         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6714
6715         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6716         $drive->{file} = $voliddst;
6717         $conf->{$ds} = print_drive($vmid, $drive);
6718         PVE::QemuConfig->write_config($vmid, $conf);
6719     });
6720 }
6721
6722 sub convert_iscsi_path {
6723     my ($path) = @_;
6724
6725     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6726         my $portal = $1;
6727         my $target = $2;
6728         my $lun = $3;
6729
6730         my $initiator_name = get_initiator_name();
6731
6732         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6733                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6734     }
6735
6736     die "cannot convert iscsi path '$path', unkown format\n";
6737 }
6738
6739 sub qemu_img_convert {
6740     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6741
6742     my $storecfg = PVE::Storage::config();
6743     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6744     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6745
6746     if ($src_storeid && $dst_storeid) {
6747
6748         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6749
6750         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6751         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6752
6753         my $src_format = qemu_img_format($src_scfg, $src_volname);
6754         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6755
6756         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6757         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6758
6759         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6760         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6761
6762         my $cmd = [];
6763         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6764         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6765         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6766         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6767
6768         if ($src_is_iscsi) {
6769             push @$cmd, '--image-opts';
6770             $src_path = convert_iscsi_path($src_path);
6771         } else {
6772             push @$cmd, '-f', $src_format;
6773         }
6774
6775         if ($dst_is_iscsi) {
6776             push @$cmd, '--target-image-opts';
6777             $dst_path = convert_iscsi_path($dst_path);
6778         } else {
6779             push @$cmd, '-O', $dst_format;
6780         }
6781
6782         push @$cmd, $src_path;
6783
6784         if (!$dst_is_iscsi && $is_zero_initialized) {
6785             push @$cmd, "zeroinit:$dst_path";
6786         } else {
6787             push @$cmd, $dst_path;
6788         }
6789
6790         my $parser = sub {
6791             my $line = shift;
6792             if($line =~ m/\((\S+)\/100\%\)/){
6793                 my $percent = $1;
6794                 my $transferred = int($size * $percent / 100);
6795                 my $remaining = $size - $transferred;
6796
6797                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6798             }
6799
6800         };
6801
6802         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6803         my $err = $@;
6804         die "copy failed: $err" if $err;
6805     }
6806 }
6807
6808 sub qemu_img_format {
6809     my ($scfg, $volname) = @_;
6810
6811     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6812         return $1;
6813     } else {
6814         return "raw";
6815     }
6816 }
6817
6818 sub qemu_drive_mirror {
6819     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6820
6821     $jobs = {} if !$jobs;
6822
6823     my $qemu_target;
6824     my $format;
6825     $jobs->{"drive-$drive"} = {};
6826
6827     if ($dst_volid =~ /^nbd:/) {
6828         $qemu_target = $dst_volid;
6829         $format = "nbd";
6830     } else {
6831         my $storecfg = PVE::Storage::config();
6832         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6833
6834         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6835
6836         $format = qemu_img_format($dst_scfg, $dst_volname);
6837
6838         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6839
6840         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6841     }
6842
6843     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6844     $opts->{format} = $format if $format;
6845
6846     if (defined($bwlimit)) {
6847         $opts->{speed} = $bwlimit * 1024;
6848         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6849     } else {
6850         print "drive mirror is starting for drive-$drive\n";
6851     }
6852
6853     # if a job already runs for this device we get an error, catch it for cleanup
6854     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6855     if (my $err = $@) {
6856         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6857         warn "$@\n" if $@;
6858         die "mirroring error: $err\n";
6859     }
6860
6861     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6862 }
6863
6864 sub qemu_drive_mirror_monitor {
6865     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6866
6867     eval {
6868         my $err_complete = 0;
6869
6870         while (1) {
6871             die "storage migration timed out\n" if $err_complete > 300;
6872
6873             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6874
6875             my $running_mirror_jobs = {};
6876             foreach my $stat (@$stats) {
6877                 next if $stat->{type} ne 'mirror';
6878                 $running_mirror_jobs->{$stat->{device}} = $stat;
6879             }
6880
6881             my $readycounter = 0;
6882
6883             foreach my $job (keys %$jobs) {
6884
6885                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6886                     print "$job : finished\n";
6887                     delete $jobs->{$job};
6888                     next;
6889                 }
6890
6891                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6892
6893                 my $busy = $running_mirror_jobs->{$job}->{busy};
6894                 my $ready = $running_mirror_jobs->{$job}->{ready};
6895                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6896                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6897                     my $remaining = $total - $transferred;
6898                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6899
6900                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6901                 }
6902
6903                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6904             }
6905
6906             last if scalar(keys %$jobs) == 0;
6907
6908             if ($readycounter == scalar(keys %$jobs)) {
6909                 print "all mirroring jobs are ready \n";
6910                 last if $skipcomplete; #do the complete later
6911
6912                 if ($vmiddst && $vmiddst != $vmid) {
6913                     my $agent_running = $qga && qga_check_running($vmid);
6914                     if ($agent_running) {
6915                         print "freeze filesystem\n";
6916                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6917                     } else {
6918                         print "suspend vm\n";
6919                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6920                     }
6921
6922                     # if we clone a disk for a new target vm, we don't switch the disk
6923                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6924
6925                     if ($agent_running) {
6926                         print "unfreeze filesystem\n";
6927                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6928                     } else {
6929                         print "resume vm\n";
6930                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6931                     }
6932
6933                     last;
6934                 } else {
6935
6936                     foreach my $job (keys %$jobs) {
6937                         # try to switch the disk if source and destination are on the same guest
6938                         print "$job: Completing block job...\n";
6939
6940                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6941                         if ($@ =~ m/cannot be completed/) {
6942                             print "$job: Block job cannot be completed, try again.\n";
6943                             $err_complete++;
6944                         }else {
6945                             print "$job: Completed successfully.\n";
6946                             $jobs->{$job}->{complete} = 1;
6947                         }
6948                     }
6949                 }
6950             }
6951             sleep 1;
6952         }
6953     };
6954     my $err = $@;
6955
6956     if ($err) {
6957         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6958         die "mirroring error: $err";
6959     }
6960
6961 }
6962
6963 sub qemu_blockjobs_cancel {
6964     my ($vmid, $jobs) = @_;
6965
6966     foreach my $job (keys %$jobs) {
6967         print "$job: Cancelling block job\n";
6968         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6969         $jobs->{$job}->{cancel} = 1;
6970     }
6971
6972     while (1) {
6973         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6974
6975         my $running_jobs = {};
6976         foreach my $stat (@$stats) {
6977             $running_jobs->{$stat->{device}} = $stat;
6978         }
6979
6980         foreach my $job (keys %$jobs) {
6981
6982             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6983                 print "$job: Done.\n";
6984                 delete $jobs->{$job};
6985             }
6986         }
6987
6988         last if scalar(keys %$jobs) == 0;
6989
6990         sleep 1;
6991     }
6992 }
6993
6994 sub clone_disk {
6995     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6996         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6997
6998     my $newvolid;
6999
7000     if (!$full) {
7001         print "create linked clone of drive $drivename ($drive->{file})\n";
7002         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7003         push @$newvollist, $newvolid;
7004     } else {
7005
7006         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7007         $storeid = $storage if $storage;
7008
7009         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7010         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7011
7012         print "create full clone of drive $drivename ($drive->{file})\n";
7013         my $name = undef;
7014         if (drive_is_cloudinit($drive)) {
7015             $name = "vm-$newvmid-cloudinit";
7016             $snapname = undef;
7017             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7018             if ($dst_format ne 'raw') {
7019                 $name .= ".$dst_format";
7020             }
7021         }
7022         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7023         push @$newvollist, $newvolid;
7024
7025         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7026
7027         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7028         if (!$running || $snapname) {
7029             # TODO: handle bwlimits
7030             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7031         } else {
7032
7033             my $kvmver = get_running_qemu_version ($vmid);
7034             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7035                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7036                     if $drive->{iothread};
7037             }
7038
7039             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7040         }
7041     }
7042
7043     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7044
7045     my $disk = $drive;
7046     $disk->{format} = undef;
7047     $disk->{file} = $newvolid;
7048     $disk->{size} = $size;
7049
7050     return $disk;
7051 }
7052
7053 # this only works if VM is running
7054 sub get_current_qemu_machine {
7055     my ($vmid) = @_;
7056
7057     my $cmd = { execute => 'query-machines', arguments => {} };
7058     my $res = vm_qmp_command($vmid, $cmd);
7059
7060     my ($current, $default);
7061     foreach my $e (@$res) {
7062         $default = $e->{name} if $e->{'is-default'};
7063         $current = $e->{name} if $e->{'is-current'};
7064     }
7065
7066     # fallback to the default machine if current is not supported by qemu
7067     return $current || $default || 'pc';
7068 }
7069
7070 sub get_running_qemu_version {
7071     my ($vmid) = @_;
7072     my $cmd = { execute => 'query-version', arguments => {} };
7073     my $res = vm_qmp_command($vmid, $cmd);
7074     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7075 }
7076
7077 sub qemu_machine_feature_enabled {
7078     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7079
7080     my $current_major;
7081     my $current_minor;
7082
7083     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7084
7085         $current_major = $3;
7086         $current_minor = $4;
7087
7088     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7089
7090         $current_major = $1;
7091         $current_minor = $2;
7092     }
7093
7094     return 1 if $current_major > $version_major ||
7095                 ($current_major == $version_major &&
7096                  $current_minor >= $version_minor);
7097 }
7098
7099 sub qemu_machine_pxe {
7100     my ($vmid, $conf, $machine) = @_;
7101
7102     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7103
7104     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7105         $machine .= '.pxe';
7106     }
7107
7108     return $machine;
7109 }
7110
7111 sub qemu_use_old_bios_files {
7112     my ($machine_type) = @_;
7113
7114     return if !$machine_type;
7115
7116     my $use_old_bios_files = undef;
7117
7118     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7119         $machine_type = $1;
7120         $use_old_bios_files = 1;
7121     } else {
7122         my $kvmver = kvm_user_version();
7123         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7124         # load new efi bios files on migration. So this hack is required to allow
7125         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7126         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7127         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7128     }
7129
7130     return ($use_old_bios_files, $machine_type);
7131 }
7132
7133 sub create_efidisk($$$$$) {
7134     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7135
7136     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7137     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7138
7139     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7140     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7141     PVE::Storage::activate_volumes($storecfg, [$volid]);
7142
7143     my $path = PVE::Storage::path($storecfg, $volid);
7144     eval {
7145         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7146     };
7147     die "Copying EFI vars image failed: $@" if $@;
7148
7149     return ($volid, $vars_size);
7150 }
7151
7152 sub vm_iothreads_list {
7153     my ($vmid) = @_;
7154
7155     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7156
7157     my $iothreads = {};
7158     foreach my $iothread (@$res) {
7159         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7160     }
7161
7162     return $iothreads;
7163 }
7164
7165 sub scsihw_infos {
7166     my ($conf, $drive) = @_;
7167
7168     my $maxdev = 0;
7169
7170     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7171         $maxdev = 7;
7172     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7173         $maxdev = 1;
7174     } else {
7175         $maxdev = 256;
7176     }
7177
7178     my $controller = int($drive->{index} / $maxdev);
7179     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7180
7181     return ($maxdev, $controller, $controller_prefix);
7182 }
7183
7184 sub add_hyperv_enlightenments {
7185     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7186
7187     return if $winversion < 6;
7188     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7189
7190     if ($gpu_passthrough || defined($hv_vendor_id)) {
7191         $hv_vendor_id //= 'proxmox';
7192         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7193     }
7194
7195     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7196         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7197         push @$cpuFlags , 'hv_vapic';
7198         push @$cpuFlags , 'hv_time';
7199     } else {
7200         push @$cpuFlags , 'hv_spinlocks=0xffff';
7201     }
7202
7203     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7204         push @$cpuFlags , 'hv_reset';
7205         push @$cpuFlags , 'hv_vpindex';
7206         push @$cpuFlags , 'hv_runtime';
7207     }
7208
7209     if ($winversion >= 7) {
7210         push @$cpuFlags , 'hv_relaxed';
7211
7212         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7213             push @$cpuFlags , 'hv_synic';
7214             push @$cpuFlags , 'hv_stimer';
7215         }
7216
7217         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7218             push @$cpuFlags , 'hv_tlbflush';
7219             push @$cpuFlags , 'hv_ipi';
7220             # FIXME: AMD does not supports this currently, only add with special flag??
7221             #push @$cpuFlags , 'hv_evmcs';
7222         }
7223     }
7224 }
7225
7226 sub windows_version {
7227     my ($ostype) = @_;
7228
7229     return 0 if !$ostype;
7230
7231     my $winversion = 0;
7232
7233     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7234         $winversion = 5;
7235     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7236         $winversion = 6;
7237     } elsif ($ostype =~ m/^win(\d+)$/) {
7238         $winversion = $1;
7239     }
7240
7241     return $winversion;
7242 }
7243
7244 sub resolve_dst_disk_format {
7245         my ($storecfg, $storeid, $src_volname, $format) = @_;
7246         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7247
7248         if (!$format) {
7249             # if no target format is specified, use the source disk format as hint
7250             if ($src_volname) {
7251                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7252                 $format = qemu_img_format($scfg, $src_volname);
7253             } else {
7254                 return $defFormat;
7255             }
7256         }
7257
7258         # test if requested format is supported - else use default
7259         my $supported = grep { $_ eq $format } @$validFormats;
7260         $format = $defFormat if !$supported;
7261         return $format;
7262 }
7263
7264 sub resolve_first_disk {
7265     my $conf = shift;
7266     my @disks = PVE::QemuServer::valid_drive_names();
7267     my $firstdisk;
7268     foreach my $ds (reverse @disks) {
7269         next if !$conf->{$ds};
7270         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7271         next if PVE::QemuServer::drive_is_cdrom($disk);
7272         $firstdisk = $ds;
7273     }
7274     return $firstdisk;
7275 }
7276
7277 sub generate_uuid {
7278     my ($uuid, $uuid_str);
7279     UUID::generate($uuid);
7280     UUID::unparse($uuid, $uuid_str);
7281     return $uuid_str;
7282 }
7283
7284 sub generate_smbios1_uuid {
7285     return "uuid=".generate_uuid();
7286 }
7287
7288 sub nbd_stop {
7289     my ($vmid) = @_;
7290
7291     vm_mon_cmd($vmid, 'nbd-server-stop');
7292 }
7293
7294 # bash completion helper
7295
7296 sub complete_backup_archives {
7297     my ($cmdname, $pname, $cvalue) = @_;
7298
7299     my $cfg = PVE::Storage::config();
7300
7301     my $storeid;
7302
7303     if ($cvalue =~ m/^([^:]+):/) {
7304         $storeid = $1;
7305     }
7306
7307     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7308
7309     my $res = [];
7310     foreach my $id (keys %$data) {
7311         foreach my $item (@{$data->{$id}}) {
7312             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7313             push @$res, $item->{volid} if defined($item->{volid});
7314         }
7315     }
7316
7317     return $res;
7318 }
7319
7320 my $complete_vmid_full = sub {
7321     my ($running) = @_;
7322
7323     my $idlist = vmstatus();
7324
7325     my $res = [];
7326
7327     foreach my $id (keys %$idlist) {
7328         my $d = $idlist->{$id};
7329         if (defined($running)) {
7330             next if $d->{template};
7331             next if $running && $d->{status} ne 'running';
7332             next if !$running && $d->{status} eq 'running';
7333         }
7334         push @$res, $id;
7335
7336     }
7337     return $res;
7338 };
7339
7340 sub complete_vmid {
7341     return &$complete_vmid_full();
7342 }
7343
7344 sub complete_vmid_stopped {
7345     return &$complete_vmid_full(0);
7346 }
7347
7348 sub complete_vmid_running {
7349     return &$complete_vmid_full(1);
7350 }
7351
7352 sub complete_storage {
7353
7354     my $cfg = PVE::Storage::config();
7355     my $ids = $cfg->{ids};
7356
7357     my $res = [];
7358     foreach my $sid (keys %$ids) {
7359         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7360         next if !$ids->{$sid}->{content}->{images};
7361         push @$res, $sid;
7362     }
7363
7364     return $res;
7365 }
7366
7367 1;