PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     }
 641 };
 642
 643 my $cicustom_fmt = {
 644     meta => {
 645         type => 'string',
 646         optional => 1,
 647         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 648         format => 'pve-volume-id',
 649         format_description => 'volume',
 650     },
 651     network => {
 652         type => 'string',
 653         optional => 1,
 654         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 655         format => 'pve-volume-id',
 656         format_description => 'volume',
 657     },
 658     user => {
 659         type => 'string',
 660         optional => 1,
 661         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 662         format => 'pve-volume-id',
 663         format_description => 'volume',
 664     },
 665 };
 666 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 667
 668 my $confdesc_cloudinit = {
 669     citype => {
 670         optional => 1,
 671         type => 'string',
 672         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 673         enum => ['configdrive2', 'nocloud'],
 674     },
 675     ciuser => {
 676         optional => 1,
 677         type => 'string',
 678         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 679     },
 680     cipassword => {
 681         optional => 1,
 682         type => 'string',
 683         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 684     },
 685     cicustom => {
 686         optional => 1,
 687         type => 'string',
 688         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 689         format => 'pve-qm-cicustom',
 690     },
 691     searchdomain => {
 692         optional => 1,
 693         type => 'string',
 694         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 695     },
 696     nameserver => {
 697         optional => 1,
 698         type => 'string', format => 'address-list',
 699         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 700     },
 701     sshkeys => {
 702         optional => 1,
 703         type => 'string',
 704         format => 'urlencoded',
 705         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 706     },
 707 };
 708
 709 # what about other qemu settings ?
 710 #cpu => 'string',
 711 #machine => 'string',
 712 #fda => 'file',
 713 #fdb => 'file',
 714 #mtdblock => 'file',
 715 #sd => 'file',
 716 #pflash => 'file',
 717 #snapshot => 'bool',
 718 #bootp => 'file',
 719 ##tftp => 'dir',
 720 ##smb => 'dir',
 721 #kernel => 'file',
 722 #append => 'string',
 723 #initrd => 'file',
 724 ##soundhw => 'string',
 725
 726 while (my ($k, $v) = each %$confdesc) {
 727     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 728 }
 729
 730 my $MAX_IDE_DISKS = 4;
 731 my $MAX_SCSI_DISKS = 14;
 732 my $MAX_VIRTIO_DISKS = 16;
 733 my $MAX_SATA_DISKS = 6;
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_UNUSED_DISKS = 256;
 737 my $MAX_HOSTPCI_DEVICES = 4;
 738 my $MAX_SERIAL_PORTS = 4;
 739 my $MAX_PARALLEL_PORTS = 3;
 740 my $MAX_NUMA = 8;
 741
 742 my $numa_fmt = {
 743     cpus => {
 744         type => "string",
 745         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 746         description => "CPUs accessing this NUMA node.",
 747         format_description => "id[-id];...",
 748     },
 749     memory => {
 750         type => "number",
 751         description => "Amount of memory this NUMA node provides.",
 752         optional => 1,
 753     },
 754     hostnodes => {
 755         type => "string",
 756         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 757         description => "Host NUMA nodes to use.",
 758         format_description => "id[-id];...",
 759         optional => 1,
 760     },
 761     policy => {
 762         type => 'string',
 763         enum => [qw(preferred bind interleave)],
 764         description => "NUMA allocation policy.",
 765         optional => 1,
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 769 my $numadesc = {
 770     optional => 1,
 771     type => 'string', format => $numa_fmt,
 772     description => "NUMA topology.",
 773 };
 774 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 775
 776 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 777     $confdesc->{"numa$i"} = $numadesc;
 778 }
 779
 780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 781                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 782                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 784
 785 my $net_fmt_bridge_descr = <<__EOD__;
 786 Bridge to attach the network device to. The Proxmox VE standard bridge
 787 is called 'vmbr0'.
 788
 789 If you do not specify a bridge, we create a kvm user (NATed) network
 790 device, which provides DHCP and DNS services. The following addresses
 791 are used:
 792
 793  10.0.2.2   Gateway
 794  10.0.2.3   DNS Server
 795  10.0.2.4   SMB Server
 796
 797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 798 __EOD__
 799
 800 my $net_fmt = {
 801     macaddr  => get_standard_option('mac-addr', {
 802         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 803     }),
 804     model => {
 805         type => 'string',
 806         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 807         enum => $nic_model_list,
 808         default_key => 1,
 809     },
 810     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 811     bridge => {
 812         type => 'string',
 813         description => $net_fmt_bridge_descr,
 814         format_description => 'bridge',
 815         optional => 1,
 816     },
 817     queues => {
 818         type => 'integer',
 819         minimum => 0, maximum => 16,
 820         description => 'Number of packet queues to be used on the device.',
 821         optional => 1,
 822     },
 823     rate => {
 824         type => 'number',
 825         minimum => 0,
 826         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 827         optional => 1,
 828     },
 829     tag => {
 830         type => 'integer',
 831         minimum => 1, maximum => 4094,
 832         description => 'VLAN tag to apply to packets on this interface.',
 833         optional => 1,
 834     },
 835     trunks => {
 836         type => 'string',
 837         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 838         description => 'VLAN trunks to pass through this interface.',
 839         format_description => 'vlanid[;vlanid...]',
 840         optional => 1,
 841     },
 842     firewall => {
 843         type => 'boolean',
 844         description => 'Whether this interface should be protected by the firewall.',
 845         optional => 1,
 846     },
 847     link_down => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be disconnected (like pulling the plug).',
 850         optional => 1,
 851     },
 852 };
 853
 854 my $netdesc = {
 855     optional => 1,
 856     type => 'string', format => $net_fmt,
 857     description => "Specify network devices.",
 858 };
 859
 860 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 861
 862 my $ipconfig_fmt = {
 863     ip => {
 864         type => 'string',
 865         format => 'pve-ipv4-config',
 866         format_description => 'IPv4Format/CIDR',
 867         description => 'IPv4 address in CIDR format.',
 868         optional => 1,
 869         default => 'dhcp',
 870     },
 871     gw => {
 872         type => 'string',
 873         format => 'ipv4',
 874         format_description => 'GatewayIPv4',
 875         description => 'Default gateway for IPv4 traffic.',
 876         optional => 1,
 877         requires => 'ip',
 878     },
 879     ip6 => {
 880         type => 'string',
 881         format => 'pve-ipv6-config',
 882         format_description => 'IPv6Format/CIDR',
 883         description => 'IPv6 address in CIDR format.',
 884         optional => 1,
 885         default => 'dhcp',
 886     },
 887     gw6 => {
 888         type => 'string',
 889         format => 'ipv6',
 890         format_description => 'GatewayIPv6',
 891         description => 'Default gateway for IPv6 traffic.',
 892         optional => 1,
 893         requires => 'ip6',
 894     },
 895 };
 896 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 897 my $ipconfigdesc = {
 898     optional => 1,
 899     type => 'string', format => 'pve-qm-ipconfig',
 900     description => <<'EODESCR',
 901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 902
 903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 904
 905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 907
 908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 909 EODESCR
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 912
 913 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 914     $confdesc->{"net$i"} = $netdesc;
 915     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 916 }
 917
 918 foreach my $key (keys %$confdesc_cloudinit) {
 919     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 920 }
 921
 922 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 923 sub verify_volume_id_or_qm_path {
 924     my ($volid, $noerr) = @_;
 925
 926     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 927         return $volid;
 928     }
 929
 930     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 931     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 932     if ($@) {
 933         return undef if $noerr;
 934         die $@;
 935     }
 936     return $volid;
 937 }
 938
 939 my $drivename_hash;
 940
 941 my %drivedesc_base = (
 942     volume => { alias => 'file' },
 943     file => {
 944         type => 'string',
 945         format => 'pve-volume-id-or-qm-path',
 946         default_key => 1,
 947         format_description => 'volume',
 948         description => "The drive's backing volume.",
 949     },
 950     media => {
 951         type => 'string',
 952         enum => [qw(cdrom disk)],
 953         description => "The drive's media type.",
 954         default => 'disk',
 955         optional => 1
 956     },
 957     cyls => {
 958         type => 'integer',
 959         description => "Force the drive's physical geometry to have a specific cylinder count.",
 960         optional => 1
 961     },
 962     heads => {
 963         type => 'integer',
 964         description => "Force the drive's physical geometry to have a specific head count.",
 965         optional => 1
 966     },
 967     secs => {
 968         type => 'integer',
 969         description => "Force the drive's physical geometry to have a specific sector count.",
 970         optional => 1
 971     },
 972     trans => {
 973         type => 'string',
 974         enum => [qw(none lba auto)],
 975         description => "Force disk geometry bios translation mode.",
 976         optional => 1,
 977     },
 978     snapshot => {
 979         type => 'boolean',
 980         description => "Controls qemu's snapshot mode feature."
 981             . " If activated, changes made to the disk are temporary and will"
 982             . " be discarded when the VM is shutdown.",
 983         optional => 1,
 984     },
 985     cache => {
 986         type => 'string',
 987         enum => [qw(none writethrough writeback unsafe directsync)],
 988         description => "The drive's cache mode",
 989         optional => 1,
 990     },
 991     format => get_standard_option('pve-qm-image-format'),
 992     size => {
 993         type => 'string',
 994         format => 'disk-size',
 995         format_description => 'DiskSize',
 996         description => "Disk size. This is purely informational and has no effect.",
 997         optional => 1,
 998     },
 999     backup => {
1000         type => 'boolean',
1001         description => "Whether the drive should be included when making backups.",
1002         optional => 1,
1003     },
1004     replicate => {
1005         type => 'boolean',
1006         description => 'Whether the drive should considered for replication jobs.',
1007         optional => 1,
1008         default => 1,
1009     },
1010     rerror => {
1011         type => 'string',
1012         enum => [qw(ignore report stop)],
1013         description => 'Read error action.',
1014         optional => 1,
1015     },
1016     werror => {
1017         type => 'string',
1018         enum => [qw(enospc ignore report stop)],
1019         description => 'Write error action.',
1020         optional => 1,
1021     },
1022     aio => {
1023         type => 'string',
1024         enum => [qw(native threads)],
1025         description => 'AIO type to use.',
1026         optional => 1,
1027     },
1028     discard => {
1029         type => 'string',
1030         enum => [qw(ignore on)],
1031         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1032         optional => 1,
1033     },
1034     detect_zeroes => {
1035         type => 'boolean',
1036         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1037         optional => 1,
1038     },
1039     serial => {
1040         type => 'string',
1041         format => 'urlencoded',
1042         format_description => 'serial',
1043         maxLength => 20*3, # *3 since it's %xx url enoded
1044         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1045         optional => 1,
1046     },
1047     shared => {
1048         type => 'boolean',
1049         description => 'Mark this locally-managed volume as available on all nodes',
1050         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1051         optional => 1,
1052         default => 0,
1053     }
1054 );
1055
1056 my %iothread_fmt = ( iothread => {
1057         type => 'boolean',
1058         description => "Whether to use iothreads for this drive",
1059         optional => 1,
1060 });
1061
1062 my %model_fmt = (
1063     model => {
1064         type => 'string',
1065         format => 'urlencoded',
1066         format_description => 'model',
1067         maxLength => 40*3, # *3 since it's %xx url enoded
1068         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1069         optional => 1,
1070     },
1071 );
1072
1073 my %queues_fmt = (
1074     queues => {
1075         type => 'integer',
1076         description => "Number of queues.",
1077         minimum => 2,
1078         optional => 1
1079     }
1080 );
1081
1082 my %scsiblock_fmt = (
1083     scsiblock => {
1084         type => 'boolean',
1085         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1086         optional => 1,
1087         default => 0,
1088     },
1089 );
1090
1091 my %ssd_fmt = (
1092     ssd => {
1093         type => 'boolean',
1094         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1095         optional => 1,
1096     },
1097 );
1098
1099 my %wwn_fmt = (
1100     wwn => {
1101         type => 'string',
1102         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1103         format_description => 'wwn',
1104         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1105         optional => 1,
1106     },
1107 );
1108
1109 my $add_throttle_desc = sub {
1110     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1111     my $d = {
1112         type => $type,
1113         format_description => $unit,
1114         description => "Maximum $what in $longunit.",
1115         optional => 1,
1116     };
1117     $d->{minimum} = $minimum if defined($minimum);
1118     $drivedesc_base{$key} = $d;
1119 };
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1122 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1123 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1124 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1130
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1138
1139 # burst lengths
1140 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1146
1147 # legacy support
1148 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1152
1153 my $ide_fmt = {
1154     %drivedesc_base,
1155     %model_fmt,
1156     %ssd_fmt,
1157     %wwn_fmt,
1158 };
1159 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1160
1161 my $idedesc = {
1162     optional => 1,
1163     type => 'string', format => $ide_fmt,
1164     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1165 };
1166 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1167
1168 my $scsi_fmt = {
1169     %drivedesc_base,
1170     %iothread_fmt,
1171     %queues_fmt,
1172     %scsiblock_fmt,
1173     %ssd_fmt,
1174     %wwn_fmt,
1175 };
1176 my $scsidesc = {
1177     optional => 1,
1178     type => 'string', format => $scsi_fmt,
1179     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1182
1183 my $sata_fmt = {
1184     %drivedesc_base,
1185     %ssd_fmt,
1186     %wwn_fmt,
1187 };
1188 my $satadesc = {
1189     optional => 1,
1190     type => 'string', format => $sata_fmt,
1191     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1192 };
1193 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1194
1195 my $virtio_fmt = {
1196     %drivedesc_base,
1197     %iothread_fmt,
1198 };
1199 my $virtiodesc = {
1200     optional => 1,
1201     type => 'string', format => $virtio_fmt,
1202     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1205
1206 my $alldrive_fmt = {
1207     %drivedesc_base,
1208     %iothread_fmt,
1209     %model_fmt,
1210     %queues_fmt,
1211     %scsiblock_fmt,
1212     %ssd_fmt,
1213     %wwn_fmt,
1214 };
1215
1216 my $efidisk_fmt = {
1217     volume => { alias => 'file' },
1218     file => {
1219         type => 'string',
1220         format => 'pve-volume-id-or-qm-path',
1221         default_key => 1,
1222         format_description => 'volume',
1223         description => "The drive's backing volume.",
1224     },
1225     format => get_standard_option('pve-qm-image-format'),
1226     size => {
1227         type => 'string',
1228         format => 'disk-size',
1229         format_description => 'DiskSize',
1230         description => "Disk size. This is purely informational and has no effect.",
1231         optional => 1,
1232     },
1233 };
1234
1235 my $efidisk_desc = {
1236     optional => 1,
1237     type => 'string', format => $efidisk_fmt,
1238     description => "Configure a Disk for storing EFI vars",
1239 };
1240
1241 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1242
1243 my $usb_fmt = {
1244     host => {
1245         default_key => 1,
1246         type => 'string', format => 'pve-qm-usb-device',
1247         format_description => 'HOSTUSBDEVICE|spice',
1248         description => <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1250
1251  'bus-port(.port)*' (decimal numbers) or
1252  'vendor_id:product_id' (hexadeciaml numbers) or
1253  'spice'
1254
1255 You can use the 'lsusb -t' command to list existing usb devices.
1256
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1258
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1260 EODESCR
1261     },
1262     usb3 => {
1263         optional => 1,
1264         type => 'boolean',
1265         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1266         default => 0,
1267     },
1268 };
1269
1270 my $usbdesc = {
1271     optional => 1,
1272     type => 'string', format => $usb_fmt,
1273     description => "Configure an USB device (n is 0 to 4).",
1274 };
1275 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1276
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1278 my $hostpci_fmt = {
1279     host => {
1280         default_key => 1,
1281         type => 'string',
1282         pattern => qr/$PCIRE(;$PCIRE)*/,
1283         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1284         description => <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1287
1288 'bus:dev.func' (hexadecimal numbers)
1289
1290 You can us the 'lspci' command to list existing PCI devices.
1291 EODESCR
1292     },
1293     rombar => {
1294         type => 'boolean',
1295         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1296         optional => 1,
1297         default => 1,
1298     },
1299     romfile => {
1300         type => 'string',
1301         pattern => '[^,;]+',
1302         format_description => 'string',
1303         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1304         optional => 1,
1305     },
1306     pcie => {
1307         type => 'boolean',
1308         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1309         optional => 1,
1310         default => 0,
1311     },
1312     'x-vga' => {
1313         type => 'boolean',
1314         description =>  "Enable vfio-vga device support.",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'mdev' => {
1319         type => 'string',
1320         format_description => 'string',
1321         pattern => '[^/\.:]+',
1322         optional => 1,
1323         description => <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1327 EODESCR
1328     }
1329 };
1330 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1331
1332 my $hostpcidesc = {
1333         optional => 1,
1334         type => 'string', format => 'pve-qm-hostpci',
1335         description => "Map host PCI devices into guest.",
1336         verbose_description =>  <<EODESCR,
1337 Map host PCI devices into guest.
1338
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1346
1347 my $serialdesc = {
1348         optional => 1,
1349         type => 'string',
1350         pattern => '(/dev/.+|socket)',
1351         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1352         verbose_description =>  <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1356
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1358
1359 CAUTION: Experimental! User reported problems with this option.
1360 EODESCR
1361 };
1362
1363 my $paralleldesc= {
1364         optional => 1,
1365         type => 'string',
1366         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1367         description =>  "Map host parallel devices (n is 0 to 2).",
1368         verbose_description =>  <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1370
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1372
1373 CAUTION: Experimental! User reported problems with this option.
1374 EODESCR
1375 };
1376
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1378     $confdesc->{"parallel$i"} = $paralleldesc;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1382     $confdesc->{"serial$i"} = $serialdesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1386     $confdesc->{"hostpci$i"} = $hostpcidesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1390     $drivename_hash->{"ide$i"} = 1;
1391     $confdesc->{"ide$i"} = $idedesc;
1392 }
1393
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1395     $drivename_hash->{"sata$i"} = 1;
1396     $confdesc->{"sata$i"} = $satadesc;
1397 }
1398
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1400     $drivename_hash->{"scsi$i"} = 1;
1401     $confdesc->{"scsi$i"} = $scsidesc ;
1402 }
1403
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1405     $drivename_hash->{"virtio$i"} = 1;
1406     $confdesc->{"virtio$i"} = $virtiodesc;
1407 }
1408
1409 $drivename_hash->{efidisk0} = 1;
1410 $confdesc->{efidisk0} = $efidisk_desc;
1411
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1413     $confdesc->{"usb$i"} = $usbdesc;
1414 }
1415
1416 my $unuseddesc = {
1417     optional => 1,
1418     type => 'string', format => 'pve-volume-id',
1419     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1420 };
1421
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1423     $confdesc->{"unused$i"} = $unuseddesc;
1424 }
1425
1426 my $kvm_api_version = 0;
1427
1428 sub kvm_version {
1429     return $kvm_api_version if $kvm_api_version;
1430
1431     open my $fh, '<', '/dev/kvm'
1432         or return undef;
1433
1434     # 0xae00 => KVM_GET_API_VERSION
1435     $kvm_api_version = ioctl($fh, 0xae00, 0);
1436
1437     return $kvm_api_version;
1438 }
1439
1440 my $kvm_user_version;
1441
1442 sub kvm_user_version {
1443
1444     return $kvm_user_version if $kvm_user_version;
1445
1446     $kvm_user_version = 'unknown';
1447
1448     my $code = sub {
1449         my $line = shift;
1450         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451             $kvm_user_version = $2;
1452         }
1453     };
1454
1455     eval { run_command("kvm -version", outfunc => $code); };
1456     warn $@ if $@;
1457
1458     return $kvm_user_version;
1459
1460 }
1461
1462 sub kernel_has_vhost_net {
1463     return -c '/dev/vhost-net';
1464 }
1465
1466 sub valid_drive_names {
1467     # order is important - used to autoselect boot disk
1468     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1472             'efidisk0');
1473 }
1474
1475 sub is_valid_drivename {
1476     my $dev = shift;
1477
1478     return defined($drivename_hash->{$dev});
1479 }
1480
1481 sub option_exists {
1482     my $key = shift;
1483     return defined($confdesc->{$key});
1484 }
1485
1486 sub nic_models {
1487     return $nic_model_list;
1488 }
1489
1490 sub os_list_description {
1491
1492     return {
1493         other => 'Other',
1494         wxp => 'Windows XP',
1495         w2k => 'Windows 2000',
1496         w2k3 =>, 'Windows 2003',
1497         w2k8 => 'Windows 2008',
1498         wvista => 'Windows Vista',
1499         win7 => 'Windows 7',
1500         win8 => 'Windows 8/2012',
1501         win10 => 'Windows 10/2016',
1502         l24 => 'Linux 2.4',
1503         l26 => 'Linux 2.6',
1504     };
1505 }
1506
1507 my $cdrom_path;
1508
1509 sub get_cdrom_path {
1510
1511     return  $cdrom_path if $cdrom_path;
1512
1513     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1514     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1515     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1516 }
1517
1518 sub get_iso_path {
1519     my ($storecfg, $vmid, $cdrom) = @_;
1520
1521     if ($cdrom eq 'cdrom') {
1522         return get_cdrom_path();
1523     } elsif ($cdrom eq 'none') {
1524         return '';
1525     } elsif ($cdrom =~ m|^/|) {
1526         return $cdrom;
1527     } else {
1528         return PVE::Storage::path($storecfg, $cdrom);
1529     }
1530 }
1531
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id {
1534     my ($vmid, $file, $media) = @_;
1535
1536      if (!($file eq 'none' || $file eq 'cdrom' ||
1537           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1538
1539         return undef if $file =~ m|/|;
1540
1541         if ($media && $media eq 'cdrom') {
1542             $file = "local:iso/$file";
1543         } else {
1544             $file = "local:$vmid/$file";
1545         }
1546     }
1547
1548     return $file;
1549 }
1550
1551 sub verify_media_type {
1552     my ($opt, $vtype, $media) = @_;
1553
1554     return if !$media;
1555
1556     my $etype;
1557     if ($media eq 'disk') {
1558         $etype = 'images';
1559     } elsif ($media eq 'cdrom') {
1560         $etype = 'iso';
1561     } else {
1562         die "internal error";
1563     }
1564
1565     return if ($vtype eq $etype);
1566
1567     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1568 }
1569
1570 sub cleanup_drive_path {
1571     my ($opt, $storecfg, $drive) = @_;
1572
1573     # try to convert filesystem paths to volume IDs
1574
1575     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1576         ($drive->{file} !~ m|^/dev/.+|) &&
1577         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1578         ($drive->{file} !~ m/^\d+$/)) {
1579         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1580         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1582         verify_media_type($opt, $vtype, $drive->{media});
1583         $drive->{file} = $volid;
1584     }
1585
1586     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1587 }
1588
1589 sub parse_hotplug_features {
1590     my ($data) = @_;
1591
1592     my $res = {};
1593
1594     return $res if $data eq '0';
1595
1596     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1597
1598     foreach my $feature (PVE::Tools::split_list($data)) {
1599         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1600             $res->{$1} = 1;
1601         } else {
1602             die "invalid hotplug feature '$feature'\n";
1603         }
1604     }
1605     return $res;
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1609 sub pve_verify_hotplug_features {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_hotplug_features($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse hotplug option\n";
1617 }
1618
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 #        [,iothread=on][,serial=serial][,model=model]
1624
1625 sub parse_drive {
1626     my ($key, $data) = @_;
1627
1628     my ($interface, $index);
1629
1630     if ($key =~ m/^([^\d]+)(\d+)$/) {
1631         $interface = $1;
1632         $index = $2;
1633     } else {
1634         return undef;
1635     }
1636
1637     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1638                                      : $confdesc->{$key}->{format};
1639     if (!$desc) {
1640         warn "invalid drive key: $key\n";
1641         return undef;
1642     }
1643     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1644     return undef if !$res;
1645     $res->{interface} = $interface;
1646     $res->{index} = $index;
1647
1648     my $error = 0;
1649     foreach my $opt (qw(bps bps_rd bps_wr)) {
1650         if (my $bps = defined(delete $res->{$opt})) {
1651             if (defined($res->{"m$opt"})) {
1652                 warn "both $opt and m$opt specified\n";
1653                 ++$error;
1654                 next;
1655             }
1656             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1657         }
1658     }
1659
1660     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661     for my $requirement (
1662         [mbps_max => 'mbps'],
1663         [mbps_rd_max => 'mbps_rd'],
1664         [mbps_wr_max => 'mbps_wr'],
1665         [miops_max => 'miops'],
1666         [miops_rd_max => 'miops_rd'],
1667         [miops_wr_max => 'miops_wr'],
1668         [bps_max_length => 'mbps_max'],
1669         [bps_rd_max_length => 'mbps_rd_max'],
1670         [bps_wr_max_length => 'mbps_wr_max'],
1671         [iops_max_length => 'iops_max'],
1672         [iops_rd_max_length => 'iops_rd_max'],
1673         [iops_wr_max_length => 'iops_wr_max']) {
1674         my ($option, $requires) = @$requirement;
1675         if ($res->{$option} && !$res->{$requires}) {
1676             warn "$option requires $requires\n";
1677             ++$error;
1678         }
1679     }
1680
1681     return undef if $error;
1682
1683     return undef if $res->{mbps_rd} && $res->{mbps};
1684     return undef if $res->{mbps_wr} && $res->{mbps};
1685     return undef if $res->{iops_rd} && $res->{iops};
1686     return undef if $res->{iops_wr} && $res->{iops};
1687
1688     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1689         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1690         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1691         return undef if $res->{interface} eq 'virtio';
1692     }
1693
1694     if (my $size = $res->{size}) {
1695         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1696     }
1697
1698     return $res;
1699 }
1700
1701 sub print_drive {
1702     my ($vmid, $drive) = @_;
1703     my $data = { %$drive };
1704     delete $data->{$_} for qw(index interface);
1705     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1706 }
1707
1708 sub scsi_inquiry {
1709     my($fh, $noerr) = @_;
1710
1711     my $SG_IO = 0x2285;
1712     my $SG_GET_VERSION_NUM = 0x2282;
1713
1714     my $versionbuf = "\x00" x 8;
1715     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1716     if (!$ret) {
1717         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1718         return undef;
1719     }
1720     my $version = unpack("I", $versionbuf);
1721     if ($version < 30000) {
1722         die "scsi generic interface too old\n"  if !$noerr;
1723         return undef;
1724     }
1725
1726     my $buf = "\x00" x 36;
1727     my $sensebuf = "\x00" x 8;
1728     my $cmd = pack("C x3 C x1", 0x12, 36);
1729
1730     # see /usr/include/scsi/sg.h
1731     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1732
1733     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734                       length($sensebuf), 0, length($buf), $buf,
1735                       $cmd, $sensebuf, 6000);
1736
1737     $ret = ioctl($fh, $SG_IO, $packet);
1738     if (!$ret) {
1739         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1740         return undef;
1741     }
1742
1743     my @res = unpack($sg_io_hdr_t, $packet);
1744     if ($res[17] || $res[18]) {
1745         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my $res = {};
1750     (my $byte0, my $byte1, $res->{vendor},
1751      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1752
1753     $res->{removable} = $byte1 & 128 ? 1 : 0;
1754     $res->{type} = $byte0 & 31;
1755
1756     return $res;
1757 }
1758
1759 sub path_is_scsi {
1760     my ($path) = @_;
1761
1762     my $fh = IO::File->new("+<$path") || return undef;
1763     my $res = scsi_inquiry($fh, 1);
1764     close($fh);
1765
1766     return $res;
1767 }
1768
1769 sub machine_type_is_q35 {
1770     my ($conf) = @_;
1771
1772     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1773 }
1774
1775 sub print_tabletdevice_full {
1776     my ($conf, $arch) = @_;
1777
1778     my $q35 = machine_type_is_q35($conf);
1779
1780     # we use uhci for old VMs because tablet driver was buggy in older qemu
1781     my $usbbus;
1782     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1783         $usbbus = 'ehci';
1784     } else {
1785         $usbbus = 'uhci';
1786     }
1787
1788     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1789 }
1790
1791 sub print_keyboarddevice_full {
1792     my ($conf, $arch, $machine) = @_;
1793
1794     return undef if $arch ne 'aarch64';
1795
1796     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1797 }
1798
1799 sub print_drivedevice_full {
1800     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1801
1802     my $device = '';
1803     my $maxdev = 0;
1804
1805     if ($drive->{interface} eq 'virtio') {
1806         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1809     } elsif ($drive->{interface} eq 'scsi') {
1810
1811         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1812         my $unit = $drive->{index} % $maxdev;
1813         my $devicetype = 'hd';
1814         my $path = '';
1815         if (drive_is_cdrom($drive)) {
1816             $devicetype = 'cd';
1817         } else {
1818             if ($drive->{file} =~ m|^/|) {
1819                 $path = $drive->{file};
1820                 if (my $info = path_is_scsi($path)) {
1821                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1822                         $devicetype = 'block';
1823                     } elsif ($info->{type} == 1) { # tape
1824                         $devicetype = 'generic';
1825                     }
1826                 }
1827             } else {
1828                  $path = PVE::Storage::path($storecfg, $drive->{file});
1829             }
1830
1831             if($path =~ m/^iscsi\:\/\//){
1832                 $devicetype = 'generic';
1833             }
1834         }
1835
1836         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1837             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1838         } else {
1839             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1840         }
1841
1842         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843             $device .= ",rotation_rate=1";
1844         }
1845         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1846
1847     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1848         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1849         my $controller = int($drive->{index} / $maxdev);
1850         my $unit = $drive->{index} % $maxdev;
1851         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1852
1853         $device = "ide-$devicetype";
1854         if ($drive->{interface} eq 'ide') {
1855             $device .= ",bus=ide.$controller,unit=$unit";
1856         } else {
1857             $device .= ",bus=ahci$controller.$unit";
1858         }
1859         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1860
1861         if ($devicetype eq 'hd') {
1862             if (my $model = $drive->{model}) {
1863                 $model = URI::Escape::uri_unescape($model);
1864                 $device .= ",model=$model";
1865             }
1866             if ($drive->{ssd}) {
1867                 $device .= ",rotation_rate=1";
1868             }
1869         }
1870         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1871     } elsif ($drive->{interface} eq 'usb') {
1872         die "implement me";
1873         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1874     } else {
1875         die "unsupported interface type";
1876     }
1877
1878     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1879
1880     if (my $serial = $drive->{serial}) {
1881         $serial = URI::Escape::uri_unescape($serial);
1882         $device .= ",serial=$serial";
1883     }
1884
1885
1886     return $device;
1887 }
1888
1889 sub get_initiator_name {
1890     my $initiator;
1891
1892     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893     while (defined(my $line = <$fh>)) {
1894         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1895         $initiator = $1;
1896         last;
1897     }
1898     $fh->close();
1899
1900     return $initiator;
1901 }
1902
1903 sub print_drive_full {
1904     my ($storecfg, $vmid, $drive) = @_;
1905
1906     my $path;
1907     my $volid = $drive->{file};
1908     my $format;
1909
1910     if (drive_is_cdrom($drive)) {
1911         $path = get_iso_path($storecfg, $vmid, $volid);
1912     } else {
1913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1914         if ($storeid) {
1915             $path = PVE::Storage::path($storecfg, $volid);
1916             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1917             $format = qemu_img_format($scfg, $volname);
1918         } else {
1919             $path = $volid;
1920             $format = "raw";
1921         }
1922    }
1923
1924     my $opts = '';
1925     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926     foreach my $o (@qemu_drive_options) {
1927         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1928     }
1929
1930     # snapshot only accepts on|off
1931     if (defined($drive->{snapshot})) {
1932         my $v = $drive->{snapshot} ? 'on' : 'off';
1933         $opts .= ",snapshot=$v";
1934     }
1935
1936     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1937         my ($dir, $qmpname) = @$type;
1938         if (my $v = $drive->{"mbps$dir"}) {
1939             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1940         }
1941         if (my $v = $drive->{"mbps${dir}_max"}) {
1942             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1943         }
1944         if (my $v = $drive->{"bps${dir}_max_length"}) {
1945             $opts .= ",throttling.bps$qmpname-max-length=$v";
1946         }
1947         if (my $v = $drive->{"iops${dir}"}) {
1948             $opts .= ",throttling.iops$qmpname=$v";
1949         }
1950         if (my $v = $drive->{"iops${dir}_max"}) {
1951             $opts .= ",throttling.iops$qmpname-max=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}_max_length"}) {
1954             $opts .= ",throttling.iops$qmpname-max-length=$v";
1955         }
1956     }
1957
1958     $opts .= ",format=$format" if $format && !$drive->{format};
1959
1960     my $cache_direct = 0;
1961
1962     if (my $cache = $drive->{cache}) {
1963         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964     } elsif (!drive_is_cdrom($drive)) {
1965         $opts .= ",cache=none";
1966         $cache_direct = 1;
1967     }
1968
1969     # aio native works only with O_DIRECT
1970     if (!$drive->{aio}) {
1971         if($cache_direct) {
1972             $opts .= ",aio=native";
1973         } else {
1974             $opts .= ",aio=threads";
1975         }
1976     }
1977
1978     if (!drive_is_cdrom($drive)) {
1979         my $detectzeroes;
1980         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1981             $detectzeroes = 'off';
1982         } elsif ($drive->{discard}) {
1983             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1984         } else {
1985             # This used to be our default with discard not being specified:
1986             $detectzeroes = 'on';
1987         }
1988         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1989     }
1990
1991     my $pathinfo = $path ? "file=$path," : '';
1992
1993     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1994 }
1995
1996 sub print_netdevice_full {
1997     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1998
1999     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2000
2001     my $device = $net->{model};
2002     if ($net->{model} eq 'virtio') {
2003          $device = 'virtio-net-pci';
2004      };
2005
2006     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2007     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2009         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010         my $vectors = $net->{queues} * 2 + 2;
2011         $tmpstr .= ",vectors=$vectors,mq=on";
2012     }
2013     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2014
2015     if ($use_old_bios_files) {
2016         my $romfile;
2017         if ($device eq 'virtio-net-pci') {
2018             $romfile = 'pxe-virtio.rom';
2019         } elsif ($device eq 'e1000') {
2020             $romfile = 'pxe-e1000.rom';
2021         } elsif ($device eq 'ne2k') {
2022             $romfile = 'pxe-ne2k_pci.rom';
2023         } elsif ($device eq 'pcnet') {
2024             $romfile = 'pxe-pcnet.rom';
2025         } elsif ($device eq 'rtl8139') {
2026             $romfile = 'pxe-rtl8139.rom';
2027         }
2028         $tmpstr .= ",romfile=$romfile" if $romfile;
2029     }
2030
2031     return $tmpstr;
2032 }
2033
2034 sub print_netdev_full {
2035     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2036
2037     my $i = '';
2038     if ($netid =~ m/^net(\d+)$/) {
2039         $i = int($1);
2040     }
2041
2042     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2043
2044     my $ifname = "tap${vmid}i$i";
2045
2046     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047     die "interface name '$ifname' is too long (max 15 character)\n"
2048         if length($ifname) >= 16;
2049
2050     my $vhostparam = '';
2051     if (is_native($arch)) {
2052         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2053     }
2054
2055     my $vmname = $conf->{name} || "vm$vmid";
2056
2057     my $netdev = "";
2058     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2059
2060     if ($net->{bridge}) {
2061         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2062     } else {
2063         $netdev = "type=user,id=$netid,hostname=$vmname";
2064     }
2065
2066     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2067
2068     return $netdev;
2069 }
2070
2071
2072 sub print_cpu_device {
2073     my ($conf, $id) = @_;
2074
2075     my $kvm = $conf->{kvm} // 1;
2076     my $cpu = $kvm ? "kvm64" : "qemu64";
2077     if (my $cputype = $conf->{cpu}) {
2078         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2079             or die "Cannot parse cpu description: $cputype\n";
2080         $cpu = $cpuconf->{cputype};
2081     }
2082
2083     my $cores = $conf->{cores} || 1;
2084
2085     my $current_core = ($id - 1) % $cores;
2086     my $current_socket = int(($id - 1 - $current_core)/$cores);
2087
2088     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2089 }
2090
2091 my $vga_map = {
2092     'cirrus' => 'cirrus-vga',
2093     'std' => 'VGA',
2094     'vmware' => 'vmware-svga',
2095     'virtio' => 'virtio-vga',
2096 };
2097
2098 sub print_vga_device {
2099     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2100
2101     my $type = $vga_map->{$vga->{type}};
2102     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103         $type = 'virtio-gpu';
2104     }
2105     my $vgamem_mb = $vga->{memory};
2106     if ($qxlnum) {
2107         $type = $id ? 'qxl' : 'qxl-vga';
2108     }
2109     die "no devicetype for $vga->{type}\n" if !$type;
2110
2111     my $memory = "";
2112     if ($vgamem_mb) {
2113         if ($vga->{type} eq 'virtio') {
2114             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2115             $memory = ",max_hostmem=$bytes";
2116         } elsif ($qxlnum) {
2117             # from https://www.spice-space.org/multiple-monitors.html
2118             $memory = ",vgamem_mb=$vga->{memory}";
2119             my $ram = $vgamem_mb * 4;
2120             my $vram = $vgamem_mb * 2;
2121             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2122         } else {
2123             $memory = ",vgamem_mb=$vga->{memory}";
2124         }
2125     } elsif ($qxlnum && $id) {
2126         $memory = ",ram_size=67108864,vram_size=33554432";
2127     }
2128
2129     my $q35 = machine_type_is_q35($conf);
2130     my $vgaid = "vga" . ($id // '');
2131     my $pciaddr;
2132
2133     if ($q35 && $vgaid eq 'vga') {
2134         # the first display uses pcie.0 bus on q35 machines
2135         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2136     } else {
2137         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2138     }
2139
2140     return "$type,id=${vgaid}${memory}${pciaddr}";
2141 }
2142
2143 sub drive_is_cloudinit {
2144     my ($drive) = @_;
2145     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2146 }
2147
2148 sub drive_is_cdrom {
2149     my ($drive, $exclude_cloudinit) = @_;
2150
2151     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2152
2153     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2154
2155 }
2156
2157 sub parse_number_sets {
2158     my ($set) = @_;
2159     my $res = [];
2160     foreach my $part (split(/;/, $set)) {
2161         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163             push @$res, [ $1, $2 ];
2164         } else {
2165             die "invalid range: $part\n";
2166         }
2167     }
2168     return $res;
2169 }
2170
2171 sub parse_numa {
2172     my ($data) = @_;
2173
2174     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2175     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2176     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2177     return $res;
2178 }
2179
2180 sub parse_hostpci {
2181     my ($value) = @_;
2182
2183     return undef if !$value;
2184
2185     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2186
2187     my @idlist = split(/;/, $res->{host});
2188     delete $res->{host};
2189     foreach my $id (@idlist) {
2190         if ($id =~ m/\./) { # full id 00:00.1
2191             push @{$res->{pciid}}, {
2192                 id => $id,
2193             };
2194         } else { # partial id 00:00
2195             $res->{pciid} = PVE::SysFSTools::lspci($id);
2196         }
2197     }
2198     return $res;
2199 }
2200
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2202 sub parse_net {
2203     my ($data) = @_;
2204
2205     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2206     if ($@) {
2207         warn $@;
2208         return undef;
2209     }
2210     if (!defined($res->{macaddr})) {
2211         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2212         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2213     }
2214     return $res;
2215 }
2216
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig {
2219     my ($data) = @_;
2220
2221     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2222     if ($@) {
2223         warn $@;
2224         return undef;
2225     }
2226
2227     if ($res->{gw} && !$res->{ip}) {
2228         warn 'gateway specified without specifying an IP address';
2229         return undef;
2230     }
2231     if ($res->{gw6} && !$res->{ip6}) {
2232         warn 'IPv6 gateway specified without specifying an IPv6 address';
2233         return undef;
2234     }
2235     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2236         warn 'gateway specified together with DHCP';
2237         return undef;
2238     }
2239     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2240         # gw6 + auto/dhcp
2241         warn "IPv6 gateway specified together with $res->{ip6} address";
2242         return undef;
2243     }
2244
2245     if (!$res->{ip} && !$res->{ip6}) {
2246         return { ip => 'dhcp', ip6 => 'dhcp' };
2247     }
2248
2249     return $res;
2250 }
2251
2252 sub print_net {
2253     my $net = shift;
2254
2255     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2256 }
2257
2258 sub add_random_macs {
2259     my ($settings) = @_;
2260
2261     foreach my $opt (keys %$settings) {
2262         next if $opt !~ m/^net(\d+)$/;
2263         my $net = parse_net($settings->{$opt});
2264         next if !$net;
2265         $settings->{$opt} = print_net($net);
2266     }
2267 }
2268
2269 sub vm_is_volid_owner {
2270     my ($storecfg, $vmid, $volid) = @_;
2271
2272     if ($volid !~  m|^/|) {
2273         my ($path, $owner);
2274         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2275         if ($owner && ($owner == $vmid)) {
2276             return 1;
2277         }
2278     }
2279
2280     return undef;
2281 }
2282
2283 sub split_flagged_list {
2284     my $text = shift || '';
2285     $text =~ s/[,;]/ /g;
2286     $text =~ s/^\s+//;
2287     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2288 }
2289
2290 sub join_flagged_list {
2291     my ($how, $lst) = @_;
2292     join $how, map { $lst->{$_} . $_ } keys %$lst;
2293 }
2294
2295 sub vmconfig_delete_pending_option {
2296     my ($conf, $key, $force) = @_;
2297
2298     delete $conf->{pending}->{$key};
2299     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2300     $pending_delete_hash->{$key} = $force ? '!' : '';
2301     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2302 }
2303
2304 sub vmconfig_undelete_pending_option {
2305     my ($conf, $key) = @_;
2306
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     delete $pending_delete_hash->{$key};
2309
2310     if (%$pending_delete_hash) {
2311         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2312     } else {
2313         delete $conf->{pending}->{delete};
2314     }
2315 }
2316
2317 sub vmconfig_register_unused_drive {
2318     my ($storecfg, $vmid, $conf, $drive) = @_;
2319
2320     if (drive_is_cloudinit($drive)) {
2321         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2322         warn $@ if $@;
2323     } elsif (!drive_is_cdrom($drive)) {
2324         my $volid = $drive->{file};
2325         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2326             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2327         }
2328     }
2329 }
2330
2331 sub vmconfig_cleanup_pending {
2332     my ($conf) = @_;
2333
2334     # remove pending changes when nothing changed
2335     my $changes;
2336     foreach my $opt (keys %{$conf->{pending}}) {
2337         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2338             $changes = 1;
2339             delete $conf->{pending}->{$opt};
2340         }
2341     }
2342
2343     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2344     my $pending_delete_hash = {};
2345     while (my ($opt, $force) = each %$current_delete_hash) {
2346         if (defined($conf->{$opt})) {
2347             $pending_delete_hash->{$opt} = $force;
2348         } else {
2349             $changes = 1;
2350         }
2351     }
2352
2353     if (%$pending_delete_hash) {
2354         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2355     } else {
2356         delete $conf->{pending}->{delete};
2357     }
2358
2359     return $changes;
2360 }
2361
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2363 my $smbios1_fmt = {
2364     uuid => {
2365         type => 'string',
2366         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367         format_description => 'UUID',
2368         description => "Set SMBIOS1 UUID.",
2369         optional => 1,
2370     },
2371     version => {
2372         type => 'string',
2373         pattern => '[A-Za-z0-9+\/]+={0,2}',
2374         format_description => 'Base64 encoded string',
2375         description => "Set SMBIOS1 version.",
2376         optional => 1,
2377     },
2378     serial => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 serial number.",
2383         optional => 1,
2384     },
2385     manufacturer => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 manufacturer.",
2390         optional => 1,
2391     },
2392     product => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 product ID.",
2397         optional => 1,
2398     },
2399     sku => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 SKU string.",
2404         optional => 1,
2405     },
2406     family => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 family string.",
2411         optional => 1,
2412     },
2413     base64 => {
2414         type => 'boolean',
2415         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2416         optional => 1,
2417     },
2418 };
2419
2420 sub parse_smbios1 {
2421     my ($data) = @_;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub print_smbios1 {
2429     my ($smbios1) = @_;
2430     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2431 }
2432
2433 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2434
2435 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2436 sub verify_bootdisk {
2437     my ($value, $noerr) = @_;
2438
2439     return $value if is_valid_drivename($value);
2440
2441     return undef if $noerr;
2442
2443     die "invalid boot disk '$value'\n";
2444 }
2445
2446 sub parse_watchdog {
2447     my ($value) = @_;
2448
2449     return undef if !$value;
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2452     warn $@ if $@;
2453     return $res;
2454 }
2455
2456 sub parse_guest_agent {
2457     my ($value) = @_;
2458
2459     return {} if !defined($value->{agent});
2460
2461     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2462     warn $@ if $@;
2463
2464     # if the agent is disabled ignore the other potentially set properties
2465     return {} if !$res->{enabled};
2466     return $res;
2467 }
2468
2469 sub parse_vga {
2470     my ($value) = @_;
2471
2472     return {} if !$value;
2473     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2474     warn $@ if $@;
2475     return $res;
2476 }
2477
2478 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2479 sub verify_usb_device {
2480     my ($value, $noerr) = @_;
2481
2482     return $value if parse_usb_device($value);
2483
2484     return undef if $noerr;
2485
2486     die "unable to parse usb device\n";
2487 }
2488
2489 # add JSON properties for create and set function
2490 sub json_config_properties {
2491     my $prop = shift;
2492
2493     foreach my $opt (keys %$confdesc) {
2494         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495         $prop->{$opt} = $confdesc->{$opt};
2496     }
2497
2498     return $prop;
2499 }
2500
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties {
2503
2504     return dclone($confdesc_cloudinit);
2505 }
2506
2507 sub check_type {
2508     my ($key, $value) = @_;
2509
2510     die "unknown setting '$key'\n" if !$confdesc->{$key};
2511
2512     my $type = $confdesc->{$key}->{type};
2513
2514     if (!defined($value)) {
2515         die "got undefined value\n";
2516     }
2517
2518     if ($value =~ m/[\n\r]/) {
2519         die "property contains a line feed\n";
2520     }
2521
2522     if ($type eq 'boolean') {
2523         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525         die "type check ('boolean') failed - got '$value'\n";
2526     } elsif ($type eq 'integer') {
2527         return int($1) if $value =~ m/^(\d+)$/;
2528         die "type check ('integer') failed - got '$value'\n";
2529     } elsif ($type eq 'number') {
2530         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531         die "type check ('number') failed - got '$value'\n";
2532     } elsif ($type eq 'string') {
2533         if (my $fmt = $confdesc->{$key}->{format}) {
2534             PVE::JSONSchema::check_format($fmt, $value);
2535             return $value;
2536         }
2537         $value =~ s/^\"(.*)\"$/$1/;
2538         return $value;
2539     } else {
2540         die "internal error"
2541     }
2542 }
2543
2544 sub touch_config {
2545     my ($vmid) = @_;
2546
2547     my $conf = PVE::QemuConfig->config_file($vmid);
2548     utime undef, undef, $conf;
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conffile = PVE::QemuConfig->config_file($vmid);
2555
2556     my $conf = PVE::QemuConfig->load_config($vmid);
2557
2558     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2559
2560     if ($conf->{template}) {
2561         # check if any base image is still used by a linked clone
2562         foreach_drive($conf, sub {
2563                 my ($ds, $drive) = @_;
2564
2565                 return if drive_is_cdrom($drive);
2566
2567                 my $volid = $drive->{file};
2568
2569                 return if !$volid || $volid =~ m|^/|;
2570
2571                 die "base volume '$volid' is still in use by linked cloned\n"
2572                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2573
2574         });
2575     }
2576
2577     # only remove disks owned by this VM
2578     foreach_drive($conf, sub {
2579         my ($ds, $drive) = @_;
2580
2581         return if drive_is_cdrom($drive, 1);
2582
2583         my $volid = $drive->{file};
2584
2585         return if !$volid || $volid =~ m|^/|;
2586
2587         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2588         return if !$path || !$owner || ($owner != $vmid);
2589
2590         eval {
2591             PVE::Storage::vdisk_free($storecfg, $volid);
2592         };
2593         warn "Could not remove disk '$volid', check manually: $@" if $@;
2594
2595     });
2596
2597     if ($keep_empty_config) {
2598         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2599     } else {
2600         unlink $conffile;
2601     }
2602
2603     # also remove unused disk
2604     eval {
2605         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2606
2607         eval {
2608             PVE::Storage::foreach_volid($dl, sub {
2609                 my ($volid, $sid, $volname, $d) = @_;
2610                 PVE::Storage::vdisk_free($storecfg, $volid);
2611             });
2612         };
2613         warn $@ if $@;
2614
2615     };
2616     warn $@ if $@;
2617 }
2618
2619 sub parse_vm_config {
2620     my ($filename, $raw) = @_;
2621
2622     return undef if !defined($raw);
2623
2624     my $res = {
2625         digest => Digest::SHA::sha1_hex($raw),
2626         snapshots => {},
2627         pending => {},
2628     };
2629
2630     $filename =~ m|/qemu-server/(\d+)\.conf$|
2631         || die "got strange filename '$filename'";
2632
2633     my $vmid = $1;
2634
2635     my $conf = $res;
2636     my $descr;
2637     my $section = '';
2638
2639     my @lines = split(/\n/, $raw);
2640     foreach my $line (@lines) {
2641         next if $line =~ m/^\s*$/;
2642
2643         if ($line =~ m/^\[PENDING\]\s*$/i) {
2644             $section = 'pending';
2645             if (defined($descr)) {
2646                 $descr =~ s/\s+$//;
2647                 $conf->{description} = $descr;
2648             }
2649             $descr = undef;
2650             $conf = $res->{$section} = {};
2651             next;
2652
2653         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2654             $section = $1;
2655             if (defined($descr)) {
2656                 $descr =~ s/\s+$//;
2657                 $conf->{description} = $descr;
2658             }
2659             $descr = undef;
2660             $conf = $res->{snapshots}->{$section} = {};
2661             next;
2662         }
2663
2664         if ($line =~ m/^\#(.*)\s*$/) {
2665             $descr = '' if !defined($descr);
2666             $descr .= PVE::Tools::decode_text($1) . "\n";
2667             next;
2668         }
2669
2670         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($2);
2673         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674             $conf->{snapstate} = $1;
2675         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2676             my $key = $1;
2677             my $value = $2;
2678             $conf->{$key} = $value;
2679         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2680             my $value = $1;
2681             if ($section eq 'pending') {
2682                 $conf->{delete} = $value; # we parse this later
2683             } else {
2684                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2685             }
2686         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2687             my $key = $1;
2688             my $value = $2;
2689             eval { $value = check_type($key, $value); };
2690             if ($@) {
2691                 warn "vm $vmid - unable to parse value of '$key' - $@";
2692             } else {
2693                 $key = 'ide2' if $key eq 'cdrom';
2694                 my $fmt = $confdesc->{$key}->{format};
2695                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696                     my $v = parse_drive($key, $value);
2697                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2698                         $v->{file} = $volid;
2699                         $value = print_drive($vmid, $v);
2700                     } else {
2701                         warn "vm $vmid - unable to parse value of '$key'\n";
2702                         next;
2703                     }
2704                 }
2705
2706                 $conf->{$key} = $value;
2707             }
2708         }
2709     }
2710
2711     if (defined($descr)) {
2712         $descr =~ s/\s+$//;
2713         $conf->{description} = $descr;
2714     }
2715     delete $res->{snapstate}; # just to be sure
2716
2717     return $res;
2718 }
2719
2720 sub write_vm_config {
2721     my ($filename, $conf) = @_;
2722
2723     delete $conf->{snapstate}; # just to be sure
2724
2725     if ($conf->{cdrom}) {
2726         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2727         $conf->{ide2} = $conf->{cdrom};
2728         delete $conf->{cdrom};
2729     }
2730
2731     # we do not use 'smp' any longer
2732     if ($conf->{sockets}) {
2733         delete $conf->{smp};
2734     } elsif ($conf->{smp}) {
2735         $conf->{sockets} = $conf->{smp};
2736         delete $conf->{cores};
2737         delete $conf->{smp};
2738     }
2739
2740     my $used_volids = {};
2741
2742     my $cleanup_config = sub {
2743         my ($cref, $pending, $snapname) = @_;
2744
2745         foreach my $key (keys %$cref) {
2746             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747                 $key eq 'snapstate' || $key eq 'pending';
2748             my $value = $cref->{$key};
2749             if ($key eq 'delete') {
2750                 die "propertry 'delete' is only allowed in [PENDING]\n"
2751                     if !$pending;
2752                 # fixme: check syntax?
2753                 next;
2754             }
2755             eval { $value = check_type($key, $value); };
2756             die "unable to parse value of '$key' - $@" if $@;
2757
2758             $cref->{$key} = $value;
2759
2760             if (!$snapname && is_valid_drivename($key)) {
2761                 my $drive = parse_drive($key, $value);
2762                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2763             }
2764         }
2765     };
2766
2767     &$cleanup_config($conf);
2768
2769     &$cleanup_config($conf->{pending}, 1);
2770
2771     foreach my $snapname (keys %{$conf->{snapshots}}) {
2772         die "internal error" if $snapname eq 'pending';
2773         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2774     }
2775
2776     # remove 'unusedX' settings if we re-add a volume
2777     foreach my $key (keys %$conf) {
2778         my $value = $conf->{$key};
2779         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780             delete $conf->{$key};
2781         }
2782     }
2783
2784     my $generate_raw_config = sub {
2785         my ($conf, $pending) = @_;
2786
2787         my $raw = '';
2788
2789         # add description as comment to top of file
2790         if (defined(my $descr = $conf->{description})) {
2791             if ($descr) {
2792                 foreach my $cl (split(/\n/, $descr)) {
2793                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2794                 }
2795             } else {
2796                 $raw .= "#\n" if $pending;
2797             }
2798         }
2799
2800         foreach my $key (sort keys %$conf) {
2801             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802             $raw .= "$key: $conf->{$key}\n";
2803         }
2804         return $raw;
2805     };
2806
2807     my $raw = &$generate_raw_config($conf);
2808
2809     if (scalar(keys %{$conf->{pending}})){
2810         $raw .= "\n[PENDING]\n";
2811         $raw .= &$generate_raw_config($conf->{pending}, 1);
2812     }
2813
2814     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2815         $raw .= "\n[$snapname]\n";
2816         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2817     }
2818
2819     return $raw;
2820 }
2821
2822 sub load_defaults {
2823
2824     my $res = {};
2825
2826     # we use static defaults from our JSON schema configuration
2827     foreach my $key (keys %$confdesc) {
2828         if (defined(my $default = $confdesc->{$key}->{default})) {
2829             $res->{$key} = $default;
2830         }
2831     }
2832
2833     return $res;
2834 }
2835
2836 sub config_list {
2837     my $vmlist = PVE::Cluster::get_vmlist();
2838     my $res = {};
2839     return $res if !$vmlist || !$vmlist->{ids};
2840     my $ids = $vmlist->{ids};
2841
2842     foreach my $vmid (keys %$ids) {
2843         my $d = $ids->{$vmid};
2844         next if !$d->{node} || $d->{node} ne $nodename;
2845         next if !$d->{type} || $d->{type} ne 'qemu';
2846         $res->{$vmid}->{exists} = 1;
2847     }
2848     return $res;
2849 }
2850
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources {
2853     my ($conf, $noerr) = @_;
2854
2855     my @loc_res = ();
2856
2857     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2858     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2859
2860     push @loc_res, "ivshmem" if $conf->{ivshmem};
2861
2862     foreach my $k (keys %$conf) {
2863         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864         # sockets are safe: they will recreated be on the target side post-migrate
2865         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2867     }
2868
2869     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2870
2871     return \@loc_res;
2872 }
2873
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability {
2876     my ($storecfg, $conf, $node) = @_;
2877
2878     foreach_drive($conf, sub {
2879         my ($ds, $drive) = @_;
2880
2881         my $volid = $drive->{file};
2882         return if !$volid;
2883
2884         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2885         return if !$sid;
2886
2887         # check if storage is available on both nodes
2888         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2889         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2890    });
2891 }
2892
2893 # list nodes where all VM images are available (used by has_feature API)
2894 sub shared_nodes {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => 1 } @$nodelist };
2899     my $nodename = PVE::INotify::nodename();
2900
2901     foreach_drive($conf, sub {
2902         my ($ds, $drive) = @_;
2903
2904         my $volid = $drive->{file};
2905         return if !$volid;
2906
2907         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2908         if ($storeid) {
2909             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2910             if ($scfg->{disable}) {
2911                 $nodehash = {};
2912             } elsif (my $avail = $scfg->{nodes}) {
2913                 foreach my $node (keys %$nodehash) {
2914                     delete $nodehash->{$node} if !$avail->{$node};
2915                 }
2916             } elsif (!$scfg->{shared}) {
2917                 foreach my $node (keys %$nodehash) {
2918                     delete $nodehash->{$node} if $node ne $nodename
2919                 }
2920             }
2921         }
2922     });
2923
2924     return $nodehash
2925 }
2926
2927 sub check_local_storage_availability {
2928     my ($conf, $storecfg) = @_;
2929
2930     my $nodelist = PVE::Cluster::get_nodelist();
2931     my $nodehash = { map { $_ => {} } @$nodelist };
2932
2933     foreach_drive($conf, sub {
2934         my ($ds, $drive) = @_;
2935
2936         my $volid = $drive->{file};
2937         return if !$volid;
2938
2939         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2940         if ($storeid) {
2941             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2942
2943             if ($scfg->{disable}) {
2944                 foreach my $node (keys %$nodehash) {
2945                     push @{$nodehash->{$node}->{not_available_storages}}, $storeid;
2946                 }
2947             } elsif (my $avail = $scfg->{nodes}) {
2948                 foreach my $node (keys %$nodehash) {
2949                     if (!$avail->{$node}) {
2950                         push @{$nodehash->{$node}->{not_available_storages}}, $storeid;
2951                     }
2952                 }
2953             }
2954         }
2955     });
2956
2957     return $nodehash
2958 }
2959
2960 sub check_cmdline {
2961     my ($pidfile, $pid) = @_;
2962
2963     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2964     if (defined($fh)) {
2965         my $line = <$fh>;
2966         $fh->close;
2967         return undef if !$line;
2968         my @param = split(/\0/, $line);
2969
2970         my $cmd = $param[0];
2971         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2972
2973         for (my $i = 0; $i < scalar (@param); $i++) {
2974             my $p = $param[$i];
2975             next if !$p;
2976             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2977                 my $p = $param[$i+1];
2978                 return 1 if $p && ($p eq $pidfile);
2979                 return undef;
2980             }
2981         }
2982     }
2983     return undef;
2984 }
2985
2986 sub check_running {
2987     my ($vmid, $nocheck, $node) = @_;
2988
2989     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2990
2991     die "unable to find configuration file for VM $vmid - no such machine\n"
2992         if !$nocheck && ! -f $filename;
2993
2994     my $pidfile = pidfile_name($vmid);
2995
2996     if (my $fd = IO::File->new("<$pidfile")) {
2997         my $st = stat($fd);
2998         my $line = <$fd>;
2999         close($fd);
3000
3001         my $mtime = $st->mtime;
3002         if ($mtime > time()) {
3003             warn "file '$filename' modified in future\n";
3004         }
3005
3006         if ($line =~ m/^(\d+)$/) {
3007             my $pid = $1;
3008             if (check_cmdline($pidfile, $pid)) {
3009                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3010                     return $pid;
3011                 }
3012             }
3013         }
3014     }
3015
3016     return undef;
3017 }
3018
3019 sub vzlist {
3020
3021     my $vzlist = config_list();
3022
3023     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3024
3025     while (defined(my $de = $fd->read)) {
3026         next if $de !~ m/^(\d+)\.pid$/;
3027         my $vmid = $1;
3028         next if !defined($vzlist->{$vmid});
3029         if (my $pid = check_running($vmid)) {
3030             $vzlist->{$vmid}->{pid} = $pid;
3031         }
3032     }
3033
3034     return $vzlist;
3035 }
3036
3037 sub disksize {
3038     my ($storecfg, $conf) = @_;
3039
3040     my $bootdisk = $conf->{bootdisk};
3041     return undef if !$bootdisk;
3042     return undef if !is_valid_drivename($bootdisk);
3043
3044     return undef if !$conf->{$bootdisk};
3045
3046     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3047     return undef if !defined($drive);
3048
3049     return undef if drive_is_cdrom($drive);
3050
3051     my $volid = $drive->{file};
3052     return undef if !$volid;
3053
3054     return $drive->{size};
3055 }
3056
3057 our $vmstatus_return_properties = {
3058     vmid => get_standard_option('pve-vmid'),
3059     status => {
3060         description => "Qemu process status.",
3061         type => 'string',
3062         enum => ['stopped', 'running'],
3063     },
3064     maxmem => {
3065         description => "Maximum memory in bytes.",
3066         type => 'integer',
3067         optional => 1,
3068         renderer => 'bytes',
3069     },
3070     maxdisk => {
3071         description => "Root disk size in bytes.",
3072         type => 'integer',
3073         optional => 1,
3074         renderer => 'bytes',
3075     },
3076     name => {
3077         description => "VM name.",
3078         type => 'string',
3079         optional => 1,
3080     },
3081     qmpstatus => {
3082         description => "Qemu QMP agent status.",
3083         type => 'string',
3084         optional => 1,
3085     },
3086     pid => {
3087         description => "PID of running qemu process.",
3088         type => 'integer',
3089         optional => 1,
3090     },
3091     uptime => {
3092         description => "Uptime.",
3093         type => 'integer',
3094         optional => 1,
3095         renderer => 'duration',
3096     },
3097     cpus => {
3098         description => "Maximum usable CPUs.",
3099         type => 'number',
3100         optional => 1,
3101     },
3102     lock => {
3103         description => "The current config lock, if any.",
3104         type => 'string',
3105         optional => 1,
3106     }
3107 };
3108
3109 my $last_proc_pid_stat;
3110
3111 # get VM status information
3112 # This must be fast and should not block ($full == false)
3113 # We only query KVM using QMP if $full == true (this can be slow)
3114 sub vmstatus {
3115     my ($opt_vmid, $full) = @_;
3116
3117     my $res = {};
3118
3119     my $storecfg = PVE::Storage::config();
3120
3121     my $list = vzlist();
3122     my $defaults = load_defaults();
3123
3124     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3125
3126     my $cpucount = $cpuinfo->{cpus} || 1;
3127
3128     foreach my $vmid (keys %$list) {
3129         next if $opt_vmid && ($vmid ne $opt_vmid);
3130
3131         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3132         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3133
3134         my $d = { vmid => $vmid };
3135         $d->{pid} = $list->{$vmid}->{pid};
3136
3137         # fixme: better status?
3138         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3139
3140         my $size = disksize($storecfg, $conf);
3141         if (defined($size)) {
3142             $d->{disk} = 0; # no info available
3143             $d->{maxdisk} = $size;
3144         } else {
3145             $d->{disk} = 0;
3146             $d->{maxdisk} = 0;
3147         }
3148
3149         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3150             * ($conf->{cores} || $defaults->{cores});
3151         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3152         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3153
3154         $d->{name} = $conf->{name} || "VM $vmid";
3155         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3156             : $defaults->{memory}*(1024*1024);
3157
3158         if ($conf->{balloon}) {
3159             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3160             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3161                 : $defaults->{shares};
3162         }
3163
3164         $d->{uptime} = 0;
3165         $d->{cpu} = 0;
3166         $d->{mem} = 0;
3167
3168         $d->{netout} = 0;
3169         $d->{netin} = 0;
3170
3171         $d->{diskread} = 0;
3172         $d->{diskwrite} = 0;
3173
3174         $d->{template} = PVE::QemuConfig->is_template($conf);
3175
3176         $d->{serial} = 1 if conf_has_serial($conf);
3177         $d->{lock} = $conf->{lock} if $conf->{lock};
3178
3179         $res->{$vmid} = $d;
3180     }
3181
3182     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3183     foreach my $dev (keys %$netdev) {
3184         next if $dev !~ m/^tap([1-9]\d*)i/;
3185         my $vmid = $1;
3186         my $d = $res->{$vmid};
3187         next if !$d;
3188
3189         $d->{netout} += $netdev->{$dev}->{receive};
3190         $d->{netin} += $netdev->{$dev}->{transmit};
3191
3192         if ($full) {
3193             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3194             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3195         }
3196
3197     }
3198
3199     my $ctime = gettimeofday;
3200
3201     foreach my $vmid (keys %$list) {
3202
3203         my $d = $res->{$vmid};
3204         my $pid = $d->{pid};
3205         next if !$pid;
3206
3207         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3208         next if !$pstat; # not running
3209
3210         my $used = $pstat->{utime} + $pstat->{stime};
3211
3212         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3213
3214         if ($pstat->{vsize}) {
3215             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3216         }
3217
3218         my $old = $last_proc_pid_stat->{$pid};
3219         if (!$old) {
3220             $last_proc_pid_stat->{$pid} = {
3221                 time => $ctime,
3222                 used => $used,
3223                 cpu => 0,
3224             };
3225             next;
3226         }
3227
3228         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3229
3230         if ($dtime > 1000) {
3231             my $dutime = $used -  $old->{used};
3232
3233             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3234             $last_proc_pid_stat->{$pid} = {
3235                 time => $ctime,
3236                 used => $used,
3237                 cpu => $d->{cpu},
3238             };
3239         } else {
3240             $d->{cpu} = $old->{cpu};
3241         }
3242     }
3243
3244     return $res if !$full;
3245
3246     my $qmpclient = PVE::QMPClient->new();
3247
3248     my $ballooncb = sub {
3249         my ($vmid, $resp) = @_;
3250
3251         my $info = $resp->{'return'};
3252         return if !$info->{max_mem};
3253
3254         my $d = $res->{$vmid};
3255
3256         # use memory assigned to VM
3257         $d->{maxmem} = $info->{max_mem};
3258         $d->{balloon} = $info->{actual};
3259
3260         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3261             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3262             $d->{freemem} = $info->{free_mem};
3263         }
3264
3265         $d->{ballooninfo} = $info;
3266     };
3267
3268     my $blockstatscb = sub {
3269         my ($vmid, $resp) = @_;
3270         my $data = $resp->{'return'} || [];
3271         my $totalrdbytes = 0;
3272         my $totalwrbytes = 0;
3273
3274         for my $blockstat (@$data) {
3275             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3276             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3277
3278             $blockstat->{device} =~ s/drive-//;
3279             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3280         }
3281         $res->{$vmid}->{diskread} = $totalrdbytes;
3282         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3283     };
3284
3285     my $statuscb = sub {
3286         my ($vmid, $resp) = @_;
3287
3288         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3289         # this fails if ballon driver is not loaded, so this must be
3290         # the last commnand (following command are aborted if this fails).
3291         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3292
3293         my $status = 'unknown';
3294         if (!defined($status = $resp->{'return'}->{status})) {
3295             warn "unable to get VM status\n";
3296             return;
3297         }
3298
3299         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3300     };
3301
3302     foreach my $vmid (keys %$list) {
3303         next if $opt_vmid && ($vmid ne $opt_vmid);
3304         next if !$res->{$vmid}->{pid}; # not running
3305         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3306     }
3307
3308     $qmpclient->queue_execute(undef, 2);
3309
3310     foreach my $vmid (keys %$list) {
3311         next if $opt_vmid && ($vmid ne $opt_vmid);
3312         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3313     }
3314
3315     return $res;
3316 }
3317
3318 sub foreach_drive {
3319     my ($conf, $func, @param) = @_;
3320
3321     foreach my $ds (valid_drive_names()) {
3322         next if !defined($conf->{$ds});
3323
3324         my $drive = parse_drive($ds, $conf->{$ds});
3325         next if !$drive;
3326
3327         &$func($ds, $drive, @param);
3328     }
3329 }
3330
3331 sub foreach_volid {
3332     my ($conf, $func, @param) = @_;
3333
3334     my $volhash = {};
3335
3336     my $test_volid = sub {
3337         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3338
3339         return if !$volid;
3340
3341         $volhash->{$volid}->{cdrom} //= 1;
3342         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3343
3344         $volhash->{$volid}->{replicate} //= 0;
3345         $volhash->{$volid}->{replicate} = 1 if $replicate;
3346
3347         $volhash->{$volid}->{shared} //= 0;
3348         $volhash->{$volid}->{shared} = 1 if $shared;
3349
3350         $volhash->{$volid}->{referenced_in_config} //= 0;
3351         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3352
3353         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3354             if defined($snapname);
3355         $volhash->{$volid}->{size} = $size if $size;
3356     };
3357
3358     foreach_drive($conf, sub {
3359         my ($ds, $drive) = @_;
3360         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3361     });
3362
3363     foreach my $snapname (keys %{$conf->{snapshots}}) {
3364         my $snap = $conf->{snapshots}->{$snapname};
3365         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3366         foreach_drive($snap, sub {
3367             my ($ds, $drive) = @_;
3368             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3369         });
3370     }
3371
3372     foreach my $volid (keys %$volhash) {
3373         &$func($volid, $volhash->{$volid}, @param);
3374     }
3375 }
3376
3377 sub conf_has_serial {
3378     my ($conf) = @_;
3379
3380     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3381         if ($conf->{"serial$i"}) {
3382             return 1;
3383         }
3384     }
3385
3386     return 0;
3387 }
3388
3389 sub vga_conf_has_spice {
3390     my ($vga) = @_;
3391
3392     my $vgaconf = parse_vga($vga);
3393     my $vgatype = $vgaconf->{type};
3394     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3395
3396     return $1 || 1;
3397 }
3398
3399 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3400 sub get_host_arch() {
3401     $host_arch = (POSIX::uname())[4] if !$host_arch;
3402     return $host_arch;
3403 }
3404
3405 sub is_native($) {
3406     my ($arch) = @_;
3407     return get_host_arch() eq $arch;
3408 }
3409
3410 my $default_machines = {
3411     x86_64 => 'pc',
3412     aarch64 => 'virt',
3413 };
3414
3415 sub get_basic_machine_info {
3416     my ($conf, $forcemachine) = @_;
3417
3418     my $arch = $conf->{arch} // get_host_arch();
3419     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3420     return ($arch, $machine);
3421 }
3422
3423 sub get_ovmf_files($) {
3424     my ($arch) = @_;
3425
3426     my $ovmf = $OVMF->{$arch}
3427         or die "no OVMF images known for architecture '$arch'\n";
3428
3429     return @$ovmf;
3430 }
3431
3432 my $Arch2Qemu = {
3433     aarch64 => '/usr/bin/qemu-system-aarch64',
3434     x86_64 => '/usr/bin/qemu-system-x86_64',
3435 };
3436 sub get_command_for_arch($) {
3437     my ($arch) = @_;
3438     return '/usr/bin/kvm' if is_native($arch);
3439
3440     my $cmd = $Arch2Qemu->{$arch}
3441         or die "don't know how to emulate architecture '$arch'\n";
3442     return $cmd;
3443 }
3444
3445 sub get_cpu_options {
3446     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3447
3448     my $cpuFlags = [];
3449     my $ostype = $conf->{ostype};
3450
3451     my $cpu = $kvm ? "kvm64" : "qemu64";
3452     if ($arch eq 'aarch64') {
3453         $cpu = 'cortex-a57';
3454     }
3455     my $hv_vendor_id;
3456     if (my $cputype = $conf->{cpu}) {
3457         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3458             or die "Cannot parse cpu description: $cputype\n";
3459         $cpu = $cpuconf->{cputype};
3460         $kvm_off = 1 if $cpuconf->{hidden};
3461         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3462
3463         if (defined(my $flags = $cpuconf->{flags})) {
3464             push @$cpuFlags, split(";", $flags);
3465         }
3466     }
3467
3468     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3469
3470     push @$cpuFlags , '-x2apic'
3471         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3472
3473     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3474
3475     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3476
3477     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3478
3479         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3480         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3481     }
3482
3483     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3484
3485     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3486
3487     push @$cpuFlags, 'kvm=off' if $kvm_off;
3488
3489     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3490         push @$cpuFlags, "vendor=${cpu_vendor}"
3491             if $cpu_vendor ne 'default';
3492     } elsif ($arch ne 'aarch64') {
3493         die "internal error"; # should not happen
3494     }
3495
3496     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3497
3498     return ('-cpu', $cpu);
3499 }
3500
3501 sub config_to_command {
3502     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3503
3504     my $cmd = [];
3505     my $globalFlags = [];
3506     my $machineFlags = [];
3507     my $rtcFlags = [];
3508     my $devices = [];
3509     my $pciaddr = '';
3510     my $bridges = {};
3511     my $kvmver = kvm_user_version();
3512     my $vernum = 0; # unknown
3513     my $ostype = $conf->{ostype};
3514     my $winversion = windows_version($ostype);
3515     my $kvm = $conf->{kvm};
3516
3517     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3518     $kvm //= 1 if is_native($arch);
3519
3520     if ($kvm) {
3521         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3522             if !defined kvm_version();
3523     }
3524
3525     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3526         $vernum = $1*1000000+$2*1000;
3527     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3528         $vernum = $1*1000000+$2*1000+$3;
3529     }
3530
3531     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3532
3533     my $have_ovz = -f '/proc/vz/vestat';
3534
3535     my $q35 = machine_type_is_q35($conf);
3536     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3537     my $use_old_bios_files = undef;
3538     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3539
3540     my $cpuunits = defined($conf->{cpuunits}) ?
3541             $conf->{cpuunits} : $defaults->{cpuunits};
3542
3543     push @$cmd, get_command_for_arch($arch);
3544
3545     push @$cmd, '-id', $vmid;
3546
3547     my $vmname = $conf->{name} || "vm$vmid";
3548
3549     push @$cmd, '-name', $vmname;
3550
3551     my $use_virtio = 0;
3552
3553     my $qmpsocket = qmp_socket($vmid);
3554     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3555     push @$cmd, '-mon', "chardev=qmp,mode=control";
3556
3557     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3558         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3559         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3560     }
3561
3562     push @$cmd, '-pidfile' , pidfile_name($vmid);
3563
3564     push @$cmd, '-daemonize';
3565
3566     if ($conf->{smbios1}) {
3567         my $smbios_conf = parse_smbios1($conf->{smbios1});
3568         if ($smbios_conf->{base64}) {
3569             # Do not pass base64 flag to qemu
3570             delete $smbios_conf->{base64};
3571             my $smbios_string = "";
3572             foreach my $key (keys %$smbios_conf) {
3573                 my $value;
3574                 if ($key eq "uuid") {
3575                     $value = $smbios_conf->{uuid}
3576                 } else {
3577                     $value = decode_base64($smbios_conf->{$key});
3578                 }
3579                 # qemu accepts any binary data, only commas need escaping by double comma
3580                 $value =~ s/,/,,/g;
3581                 $smbios_string .= "," . $key . "=" . $value if $value;
3582             }
3583             push @$cmd, '-smbios', "type=1" . $smbios_string;
3584         } else {
3585             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3586         }
3587     }
3588
3589     if ($conf->{vmgenid}) {
3590         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3591     }
3592
3593     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3594     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3595         die "uefi base image not found\n" if ! -f $ovmf_code;
3596
3597         my $path;
3598         my $format;
3599         if (my $efidisk = $conf->{efidisk0}) {
3600             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3601             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3602             $format = $d->{format};
3603             if ($storeid) {
3604                 $path = PVE::Storage::path($storecfg, $d->{file});
3605                 if (!defined($format)) {
3606                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3607                     $format = qemu_img_format($scfg, $volname);
3608                 }
3609             } else {
3610                 $path = $d->{file};
3611                 die "efidisk format must be specified\n"
3612                     if !defined($format);
3613             }
3614         } else {
3615             warn "no efidisk configured! Using temporary efivars disk.\n";
3616             $path = "/tmp/$vmid-ovmf.fd";
3617             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3618             $format = 'raw';
3619         }
3620
3621         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3622         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3623     }
3624
3625
3626     # add usb controllers
3627     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3628     push @$devices, @usbcontrollers if @usbcontrollers;
3629     my $vga = parse_vga($conf->{vga});
3630
3631     my $qxlnum = vga_conf_has_spice($conf->{vga});
3632     $vga->{type} = 'qxl' if $qxlnum;
3633
3634     if (!$vga->{type}) {
3635         if ($arch eq 'aarch64') {
3636             $vga->{type} = 'virtio';
3637         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3638             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3639         } else {
3640             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3641         }
3642     }
3643
3644     # enable absolute mouse coordinates (needed by vnc)
3645     my $tablet;
3646     if (defined($conf->{tablet})) {
3647         $tablet = $conf->{tablet};
3648     } else {
3649         $tablet = $defaults->{tablet};
3650         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3651         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3652     }
3653
3654     if ($tablet) {
3655         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3656         my $kbd = print_keyboarddevice_full($conf, $arch);
3657         push @$devices, '-device', $kbd if defined($kbd);
3658     }
3659
3660     my $kvm_off = 0;
3661     my $gpu_passthrough;
3662
3663     # host pci devices
3664     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3665         my $d = parse_hostpci($conf->{"hostpci$i"});
3666         next if !$d;
3667
3668         my $pcie = $d->{pcie};
3669         if ($pcie) {
3670             die "q35 machine model is not enabled" if !$q35;
3671             # win7 wants to have the pcie devices directly on the pcie bus
3672             # instead of in the root port
3673             if ($winversion == 7) {
3674                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3675             } else {
3676                 $pciaddr = print_pcie_addr("hostpci$i");
3677             }
3678         } else {
3679             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3680         }
3681
3682         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3683         my $romfile = $d->{romfile};
3684
3685         my $xvga = '';
3686         if ($d->{'x-vga'}) {
3687             $xvga = ',x-vga=on';
3688             $kvm_off = 1;
3689             $vga->{type} = 'none' if !defined($conf->{vga});
3690             $gpu_passthrough = 1;
3691
3692             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3693                 $xvga = "";
3694             }
3695         }
3696         my $pcidevices = $d->{pciid};
3697         my $multifunction = 1 if @$pcidevices > 1;
3698         my $sysfspath;
3699         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3700             my $id = $pcidevices->[0]->{id};
3701             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3702             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3703         } elsif ($d->{mdev}) {
3704             warn "ignoring mediated device with multifunction device\n";
3705         }
3706
3707         my $j=0;
3708         foreach my $pcidevice (@$pcidevices) {
3709
3710             my $id = "hostpci$i";
3711             $id .= ".$j" if $multifunction;
3712             my $addr = $pciaddr;
3713             $addr .= ".$j" if $multifunction;
3714             my $devicestr = "vfio-pci";
3715             if ($sysfspath) {
3716                 $devicestr .= ",sysfsdev=$sysfspath";
3717             } else {
3718                 $devicestr .= ",host=$pcidevice->{id}";
3719             }
3720             $devicestr .= ",id=$id$addr";
3721
3722             if($j == 0){
3723                 $devicestr .= "$rombar$xvga";
3724                 $devicestr .= ",multifunction=on" if $multifunction;
3725                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3726             }
3727
3728             push @$devices, '-device', $devicestr;
3729             $j++;
3730         }
3731     }
3732
3733     # usb devices
3734     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3735     push @$devices, @usbdevices if @usbdevices;
3736     # serial devices
3737     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3738         if (my $path = $conf->{"serial$i"}) {
3739             if ($path eq 'socket') {
3740                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3741                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3742                 # On aarch64, serial0 is the UART device. Qemu only allows
3743                 # connecting UART devices via the '-serial' command line, as
3744                 # the device has a fixed slot on the hardware...
3745                 if ($arch eq 'aarch64' && $i == 0) {
3746                     push @$devices, '-serial', "chardev:serial$i";
3747                 } else {
3748                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3749                 }
3750             } else {
3751                 die "no such serial device\n" if ! -c $path;
3752                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3753                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3754             }
3755         }
3756     }
3757
3758     # parallel devices
3759     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3760         if (my $path = $conf->{"parallel$i"}) {
3761             die "no such parallel device\n" if ! -c $path;
3762             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3763             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3764             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3765         }
3766     }
3767
3768
3769     my $sockets = 1;
3770     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3771     $sockets = $conf->{sockets} if  $conf->{sockets};
3772
3773     my $cores = $conf->{cores} || 1;
3774
3775     my $maxcpus = $sockets * $cores;
3776
3777     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3778
3779     my $allowed_vcpus = $cpuinfo->{cpus};
3780
3781     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3782         if ($allowed_vcpus < $maxcpus);
3783
3784     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3785
3786         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3787         for (my $i = 2; $i <= $vcpus; $i++)  {
3788             my $cpustr = print_cpu_device($conf,$i);
3789             push @$cmd, '-device', $cpustr;
3790         }
3791
3792     } else {
3793
3794         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3795     }
3796     push @$cmd, '-nodefaults';
3797
3798     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3799
3800     my $bootindex_hash = {};
3801     my $i = 1;
3802     foreach my $o (split(//, $bootorder)) {
3803         $bootindex_hash->{$o} = $i*100;
3804         $i++;
3805     }
3806
3807     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3808
3809     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3810
3811     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3812
3813     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3814         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3815         my $socket = vnc_socket($vmid);
3816         push @$cmd,  '-vnc', "unix:$socket,password";
3817     } else {
3818         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3819         push @$cmd, '-nographic';
3820     }
3821
3822     # time drift fix
3823     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3824
3825     my $useLocaltime = $conf->{localtime};
3826
3827     if ($winversion >= 5) { # windows
3828         $useLocaltime = 1 if !defined($conf->{localtime});
3829
3830         # use time drift fix when acpi is enabled
3831         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3832             $tdf = 1 if !defined($conf->{tdf});
3833         }
3834     }
3835
3836     if ($winversion >= 6) {
3837         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3838         push @$cmd, '-no-hpet';
3839     }
3840
3841     push @$rtcFlags, 'driftfix=slew' if $tdf;
3842
3843     if (!$kvm) {
3844         push @$machineFlags, 'accel=tcg';
3845     }
3846
3847     if ($machine_type) {
3848         push @$machineFlags, "type=${machine_type}";
3849     }
3850
3851     if ($conf->{startdate}) {
3852         push @$rtcFlags, "base=$conf->{startdate}";
3853     } elsif ($useLocaltime) {
3854         push @$rtcFlags, 'base=localtime';
3855     }
3856
3857     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3858
3859     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3860
3861     push @$cmd, '-S' if $conf->{freeze};
3862
3863     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3864
3865     # enable sound
3866     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3867     #push @$cmd, '-soundhw', 'es1370';
3868     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3869
3870     if (parse_guest_agent($conf)->{enabled}) {
3871         my $qgasocket = qmp_socket($vmid, 1);
3872         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3873         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3874         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3875         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3876     }
3877
3878     my $spice_port;
3879
3880     if ($qxlnum) {
3881         if ($qxlnum > 1) {
3882             if ($winversion){
3883                 for(my $i = 1; $i < $qxlnum; $i++){
3884                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3885                 }
3886             } else {
3887                 # assume other OS works like Linux
3888                 my ($ram, $vram) = ("134217728", "67108864");
3889                 if ($vga->{memory}) {
3890                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3891                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3892                 }
3893                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3894                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3895             }
3896         }
3897
3898         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3899
3900         my $nodename = PVE::INotify::nodename();
3901         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3902         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3903         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3904         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3905         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3906
3907         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3908
3909         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3910         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3911         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3912     }
3913
3914     # enable balloon by default, unless explicitly disabled
3915     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3916         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3917         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3918     }
3919
3920     if ($conf->{watchdog}) {
3921         my $wdopts = parse_watchdog($conf->{watchdog});
3922         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3923         my $watchdog = $wdopts->{model} || 'i6300esb';
3924         push @$devices, '-device', "$watchdog$pciaddr";
3925         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3926     }
3927
3928     my $vollist = [];
3929     my $scsicontroller = {};
3930     my $ahcicontroller = {};
3931     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3932
3933     # Add iscsi initiator name if available
3934     if (my $initiator = get_initiator_name()) {
3935         push @$devices, '-iscsi', "initiator-name=$initiator";
3936     }
3937
3938     foreach_drive($conf, sub {
3939         my ($ds, $drive) = @_;
3940
3941         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3942             push @$vollist, $drive->{file};
3943         }
3944
3945         # ignore efidisk here, already added in bios/fw handling code above
3946         return if $drive->{interface} eq 'efidisk';
3947
3948         $use_virtio = 1 if $ds =~ m/^virtio/;
3949
3950         if (drive_is_cdrom ($drive)) {
3951             if ($bootindex_hash->{d}) {
3952                 $drive->{bootindex} = $bootindex_hash->{d};
3953                 $bootindex_hash->{d} += 1;
3954             }
3955         } else {
3956             if ($bootindex_hash->{c}) {
3957                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3958                 $bootindex_hash->{c} += 1;
3959             }
3960         }
3961
3962         if($drive->{interface} eq 'virtio'){
3963            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3964         }
3965
3966         if ($drive->{interface} eq 'scsi') {
3967
3968             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3969
3970             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3971             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3972
3973             my $iothread = '';
3974             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3975                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3976                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3977             } elsif ($drive->{iothread}) {
3978                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3979             }
3980
3981             my $queues = '';
3982             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3983                 $queues = ",num_queues=$drive->{queues}";
3984             }
3985
3986             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3987             $scsicontroller->{$controller}=1;
3988         }
3989
3990         if ($drive->{interface} eq 'sata') {
3991            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3992            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3993            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3994            $ahcicontroller->{$controller}=1;
3995         }
3996
3997         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3998         push @$devices, '-drive',$drive_cmd;
3999         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4000     });
4001
4002     for (my $i = 0; $i < $MAX_NETS; $i++) {
4003          next if !$conf->{"net$i"};
4004          my $d = parse_net($conf->{"net$i"});
4005          next if !$d;
4006
4007          $use_virtio = 1 if $d->{model} eq 'virtio';
4008
4009          if ($bootindex_hash->{n}) {
4010             $d->{bootindex} = $bootindex_hash->{n};
4011             $bootindex_hash->{n} += 1;
4012          }
4013
4014          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4015          push @$devices, '-netdev', $netdevfull;
4016
4017          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4018          push @$devices, '-device', $netdevicefull;
4019     }
4020
4021     if ($conf->{ivshmem}) {
4022         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4023
4024         my $bus;
4025         if ($q35) {
4026             $bus = print_pcie_addr("ivshmem");
4027         } else {
4028             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4029         }
4030
4031         my $ivshmem_name = $ivshmem->{name} // $vmid;
4032         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4033
4034         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4035         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4036     }
4037
4038     if (!$q35) {
4039         # add pci bridges
4040         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4041            $bridges->{1} = 1;
4042            $bridges->{2} = 1;
4043         }
4044
4045         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4046
4047         while (my ($k, $v) = each %$bridges) {
4048             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4049             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4050         }
4051     }
4052
4053     push @$cmd, @$devices;
4054     push @$cmd, '-rtc', join(',', @$rtcFlags)
4055         if scalar(@$rtcFlags);
4056     push @$cmd, '-machine', join(',', @$machineFlags)
4057         if scalar(@$machineFlags);
4058     push @$cmd, '-global', join(',', @$globalFlags)
4059         if scalar(@$globalFlags);
4060
4061     if (my $vmstate = $conf->{vmstate}) {
4062         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4063         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4064         push @$cmd, '-loadstate', $statepath;
4065     }
4066
4067     # add custom args
4068     if ($conf->{args}) {
4069         my $aa = PVE::Tools::split_args($conf->{args});
4070         push @$cmd, @$aa;
4071     }
4072
4073     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4074 }
4075
4076 sub vnc_socket {
4077     my ($vmid) = @_;
4078     return "${var_run_tmpdir}/$vmid.vnc";
4079 }
4080
4081 sub spice_port {
4082     my ($vmid) = @_;
4083
4084     my $res = vm_mon_cmd($vmid, 'query-spice');
4085
4086     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4087 }
4088
4089 sub qmp_socket {
4090     my ($vmid, $qga, $name) = @_;
4091     my $sockettype = $qga ? 'qga' : 'qmp';
4092     my $ext = $name ? '-'.$name : '';
4093     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4094 }
4095
4096 sub pidfile_name {
4097     my ($vmid) = @_;
4098     return "${var_run_tmpdir}/$vmid.pid";
4099 }
4100
4101 sub vm_devices_list {
4102     my ($vmid) = @_;
4103
4104     my $res = vm_mon_cmd($vmid, 'query-pci');
4105     my $devices_to_check = [];
4106     my $devices = {};
4107     foreach my $pcibus (@$res) {
4108         push @$devices_to_check, @{$pcibus->{devices}},
4109     }
4110
4111     while (@$devices_to_check) {
4112         my $to_check = [];
4113         for my $d (@$devices_to_check) {
4114             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4115             next if !$d->{'pci_bridge'};
4116
4117             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4118             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4119         }
4120         $devices_to_check = $to_check;
4121     }
4122
4123     my $resblock = vm_mon_cmd($vmid, 'query-block');
4124     foreach my $block (@$resblock) {
4125         if($block->{device} =~ m/^drive-(\S+)/){
4126                 $devices->{$1} = 1;
4127         }
4128     }
4129
4130     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4131     foreach my $mice (@$resmice) {
4132         if ($mice->{name} eq 'QEMU HID Tablet') {
4133             $devices->{tablet} = 1;
4134             last;
4135         }
4136     }
4137
4138     # for usb devices there is no query-usb
4139     # but we can iterate over the entries in
4140     # qom-list path=/machine/peripheral
4141     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4142     foreach my $per (@$resperipheral) {
4143         if ($per->{name} =~ m/^usb\d+$/) {
4144             $devices->{$per->{name}} = 1;
4145         }
4146     }
4147
4148     return $devices;
4149 }
4150
4151 sub vm_deviceplug {
4152     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4153
4154     my $q35 = machine_type_is_q35($conf);
4155
4156     my $devices_list = vm_devices_list($vmid);
4157     return 1 if defined($devices_list->{$deviceid});
4158
4159     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4160
4161     if ($deviceid eq 'tablet') {
4162
4163         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4164
4165     } elsif ($deviceid eq 'keyboard') {
4166
4167         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4168
4169     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4170
4171         die "usb hotplug currently not reliable\n";
4172         # since we can't reliably hot unplug all added usb devices
4173         # and usb passthrough disables live migration
4174         # we disable usb hotplugging for now
4175         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4176
4177     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4178
4179         qemu_iothread_add($vmid, $deviceid, $device);
4180
4181         qemu_driveadd($storecfg, $vmid, $device);
4182         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4183
4184         qemu_deviceadd($vmid, $devicefull);
4185         eval { qemu_deviceaddverify($vmid, $deviceid); };
4186         if (my $err = $@) {
4187             eval { qemu_drivedel($vmid, $deviceid); };
4188             warn $@ if $@;
4189             die $err;
4190         }
4191
4192     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4193
4194
4195         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4196         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4197         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4198
4199         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4200
4201         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4202             qemu_iothread_add($vmid, $deviceid, $device);
4203             $devicefull .= ",iothread=iothread-$deviceid";
4204         }
4205
4206         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4207             $devicefull .= ",num_queues=$device->{queues}";
4208         }
4209
4210         qemu_deviceadd($vmid, $devicefull);
4211         qemu_deviceaddverify($vmid, $deviceid);
4212
4213     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4214
4215         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4216         qemu_driveadd($storecfg, $vmid, $device);
4217
4218         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4219         eval { qemu_deviceadd($vmid, $devicefull); };
4220         if (my $err = $@) {
4221             eval { qemu_drivedel($vmid, $deviceid); };
4222             warn $@ if $@;
4223             die $err;
4224         }
4225
4226     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4227
4228         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4229
4230         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4231         my $use_old_bios_files = undef;
4232         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4233
4234         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4235         qemu_deviceadd($vmid, $netdevicefull);
4236         eval {
4237             qemu_deviceaddverify($vmid, $deviceid);
4238             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4239         };
4240         if (my $err = $@) {
4241             eval { qemu_netdevdel($vmid, $deviceid); };
4242             warn $@ if $@;
4243             die $err;
4244         }
4245
4246     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4247
4248         my $bridgeid = $2;
4249         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4250         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4251
4252         qemu_deviceadd($vmid, $devicefull);
4253         qemu_deviceaddverify($vmid, $deviceid);
4254
4255     } else {
4256         die "can't hotplug device '$deviceid'\n";
4257     }
4258
4259     return 1;
4260 }
4261
4262 # fixme: this should raise exceptions on error!
4263 sub vm_deviceunplug {
4264     my ($vmid, $conf, $deviceid) = @_;
4265
4266     my $devices_list = vm_devices_list($vmid);
4267     return 1 if !defined($devices_list->{$deviceid});
4268
4269     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4270
4271     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4272
4273         qemu_devicedel($vmid, $deviceid);
4274
4275     } elsif ($deviceid =~ m/^usb\d+$/) {
4276
4277         die "usb hotplug currently not reliable\n";
4278         # when unplugging usb devices this way,
4279         # there may be remaining usb controllers/hubs
4280         # so we disable it for now
4281         qemu_devicedel($vmid, $deviceid);
4282         qemu_devicedelverify($vmid, $deviceid);
4283
4284     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4285
4286         qemu_devicedel($vmid, $deviceid);
4287         qemu_devicedelverify($vmid, $deviceid);
4288         qemu_drivedel($vmid, $deviceid);
4289         qemu_iothread_del($conf, $vmid, $deviceid);
4290
4291     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4292
4293         qemu_devicedel($vmid, $deviceid);
4294         qemu_devicedelverify($vmid, $deviceid);
4295         qemu_iothread_del($conf, $vmid, $deviceid);
4296
4297     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4298
4299         qemu_devicedel($vmid, $deviceid);
4300         qemu_drivedel($vmid, $deviceid);
4301         qemu_deletescsihw($conf, $vmid, $deviceid);
4302
4303     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4304
4305         qemu_devicedel($vmid, $deviceid);
4306         qemu_devicedelverify($vmid, $deviceid);
4307         qemu_netdevdel($vmid, $deviceid);
4308
4309     } else {
4310         die "can't unplug device '$deviceid'\n";
4311     }
4312
4313     return 1;
4314 }
4315
4316 sub qemu_deviceadd {
4317     my ($vmid, $devicefull) = @_;
4318
4319     $devicefull = "driver=".$devicefull;
4320     my %options =  split(/[=,]/, $devicefull);
4321
4322     vm_mon_cmd($vmid, "device_add" , %options);
4323 }
4324
4325 sub qemu_devicedel {
4326     my ($vmid, $deviceid) = @_;
4327
4328     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4329 }
4330
4331 sub qemu_iothread_add {
4332     my($vmid, $deviceid, $device) = @_;
4333
4334     if ($device->{iothread}) {
4335         my $iothreads = vm_iothreads_list($vmid);
4336         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4337     }
4338 }
4339
4340 sub qemu_iothread_del {
4341     my($conf, $vmid, $deviceid) = @_;
4342
4343     my $confid = $deviceid;
4344     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4345         $confid = 'scsi' . $1;
4346     }
4347     my $device = parse_drive($confid, $conf->{$confid});
4348     if ($device->{iothread}) {
4349         my $iothreads = vm_iothreads_list($vmid);
4350         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4351     }
4352 }
4353
4354 sub qemu_objectadd {
4355     my($vmid, $objectid, $qomtype) = @_;
4356
4357     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4358
4359     return 1;
4360 }
4361
4362 sub qemu_objectdel {
4363     my($vmid, $objectid) = @_;
4364
4365     vm_mon_cmd($vmid, "object-del", id => $objectid);
4366
4367     return 1;
4368 }
4369
4370 sub qemu_driveadd {
4371     my ($storecfg, $vmid, $device) = @_;
4372
4373     my $drive = print_drive_full($storecfg, $vmid, $device);
4374     $drive =~ s/\\/\\\\/g;
4375     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4376
4377     # If the command succeeds qemu prints: "OK"
4378     return 1 if $ret =~ m/OK/s;
4379
4380     die "adding drive failed: $ret\n";
4381 }
4382
4383 sub qemu_drivedel {
4384     my($vmid, $deviceid) = @_;
4385
4386     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4387     $ret =~ s/^\s+//;
4388
4389     return 1 if $ret eq "";
4390
4391     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4392     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4393
4394     die "deleting drive $deviceid failed : $ret\n";
4395 }
4396
4397 sub qemu_deviceaddverify {
4398     my ($vmid, $deviceid) = @_;
4399
4400     for (my $i = 0; $i <= 5; $i++) {
4401          my $devices_list = vm_devices_list($vmid);
4402          return 1 if defined($devices_list->{$deviceid});
4403          sleep 1;
4404     }
4405
4406     die "error on hotplug device '$deviceid'\n";
4407 }
4408
4409
4410 sub qemu_devicedelverify {
4411     my ($vmid, $deviceid) = @_;
4412
4413     # need to verify that the device is correctly removed as device_del
4414     # is async and empty return is not reliable
4415
4416     for (my $i = 0; $i <= 5; $i++) {
4417          my $devices_list = vm_devices_list($vmid);
4418          return 1 if !defined($devices_list->{$deviceid});
4419          sleep 1;
4420     }
4421
4422     die "error on hot-unplugging device '$deviceid'\n";
4423 }
4424
4425 sub qemu_findorcreatescsihw {
4426     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4427
4428     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4429
4430     my $scsihwid="$controller_prefix$controller";
4431     my $devices_list = vm_devices_list($vmid);
4432
4433     if(!defined($devices_list->{$scsihwid})) {
4434         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4435     }
4436
4437     return 1;
4438 }
4439
4440 sub qemu_deletescsihw {
4441     my ($conf, $vmid, $opt) = @_;
4442
4443     my $device = parse_drive($opt, $conf->{$opt});
4444
4445     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4446         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4447         return 1;
4448     }
4449
4450     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4451
4452     my $devices_list = vm_devices_list($vmid);
4453     foreach my $opt (keys %{$devices_list}) {
4454         if (PVE::QemuServer::is_valid_drivename($opt)) {
4455             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4456             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4457                 return 1;
4458             }
4459         }
4460     }
4461
4462     my $scsihwid="scsihw$controller";
4463
4464     vm_deviceunplug($vmid, $conf, $scsihwid);
4465
4466     return 1;
4467 }
4468
4469 sub qemu_add_pci_bridge {
4470     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4471
4472     my $bridges = {};
4473
4474     my $bridgeid;
4475
4476     print_pci_addr($device, $bridges, $arch, $machine_type);
4477
4478     while (my ($k, $v) = each %$bridges) {
4479         $bridgeid = $k;
4480     }
4481     return 1 if !defined($bridgeid) || $bridgeid < 1;
4482
4483     my $bridge = "pci.$bridgeid";
4484     my $devices_list = vm_devices_list($vmid);
4485
4486     if (!defined($devices_list->{$bridge})) {
4487         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4488     }
4489
4490     return 1;
4491 }
4492
4493 sub qemu_set_link_status {
4494     my ($vmid, $device, $up) = @_;
4495
4496     vm_mon_cmd($vmid, "set_link", name => $device,
4497                up => $up ? JSON::true : JSON::false);
4498 }
4499
4500 sub qemu_netdevadd {
4501     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4502
4503     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4504     my %options =  split(/[=,]/, $netdev);
4505
4506     vm_mon_cmd($vmid, "netdev_add",  %options);
4507     return 1;
4508 }
4509
4510 sub qemu_netdevdel {
4511     my ($vmid, $deviceid) = @_;
4512
4513     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4514 }
4515
4516 sub qemu_usb_hotplug {
4517     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4518
4519     return if !$device;
4520
4521     # remove the old one first
4522     vm_deviceunplug($vmid, $conf, $deviceid);
4523
4524     # check if xhci controller is necessary and available
4525     if ($device->{usb3}) {
4526
4527         my $devicelist = vm_devices_list($vmid);
4528
4529         if (!$devicelist->{xhci}) {
4530             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4531             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4532         }
4533     }
4534     my $d = parse_usb_device($device->{host});
4535     $d->{usb3} = $device->{usb3};
4536
4537     # add the new one
4538     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4539 }
4540
4541 sub qemu_cpu_hotplug {
4542     my ($vmid, $conf, $vcpus) = @_;
4543
4544     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4545
4546     my $sockets = 1;
4547     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4548     $sockets = $conf->{sockets} if  $conf->{sockets};
4549     my $cores = $conf->{cores} || 1;
4550     my $maxcpus = $sockets * $cores;
4551
4552     $vcpus = $maxcpus if !$vcpus;
4553
4554     die "you can't add more vcpus than maxcpus\n"
4555         if $vcpus > $maxcpus;
4556
4557     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4558
4559     if ($vcpus < $currentvcpus) {
4560
4561         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4562
4563             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4564                 qemu_devicedel($vmid, "cpu$i");
4565                 my $retry = 0;
4566                 my $currentrunningvcpus = undef;
4567                 while (1) {
4568                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4569                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4570                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4571                     $retry++;
4572                     sleep 1;
4573                 }
4574                 #update conf after each succesfull cpu unplug
4575                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4576                 PVE::QemuConfig->write_config($vmid, $conf);
4577             }
4578         } else {
4579             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4580         }
4581
4582         return;
4583     }
4584
4585     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4586     die "vcpus in running vm does not match its configuration\n"
4587         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4588
4589     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4590
4591         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4592             my $cpustr = print_cpu_device($conf, $i);
4593             qemu_deviceadd($vmid, $cpustr);
4594
4595             my $retry = 0;
4596             my $currentrunningvcpus = undef;
4597             while (1) {
4598                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4599                 last if scalar(@{$currentrunningvcpus}) == $i;
4600                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4601                 sleep 1;
4602                 $retry++;
4603             }
4604             #update conf after each succesfull cpu hotplug
4605             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4606             PVE::QemuConfig->write_config($vmid, $conf);
4607         }
4608     } else {
4609
4610         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4611             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4612         }
4613     }
4614 }
4615
4616 sub qemu_block_set_io_throttle {
4617     my ($vmid, $deviceid,
4618         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4619         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4620         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4621         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4622
4623     return if !check_running($vmid) ;
4624
4625     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4626         bps => int($bps),
4627         bps_rd => int($bps_rd),
4628         bps_wr => int($bps_wr),
4629         iops => int($iops),
4630         iops_rd => int($iops_rd),
4631         iops_wr => int($iops_wr),
4632         bps_max => int($bps_max),
4633         bps_rd_max => int($bps_rd_max),
4634         bps_wr_max => int($bps_wr_max),
4635         iops_max => int($iops_max),
4636         iops_rd_max => int($iops_rd_max),
4637         iops_wr_max => int($iops_wr_max),
4638         bps_max_length => int($bps_max_length),
4639         bps_rd_max_length => int($bps_rd_max_length),
4640         bps_wr_max_length => int($bps_wr_max_length),
4641         iops_max_length => int($iops_max_length),
4642         iops_rd_max_length => int($iops_rd_max_length),
4643         iops_wr_max_length => int($iops_wr_max_length),
4644     );
4645
4646 }
4647
4648 # old code, only used to shutdown old VM after update
4649 sub __read_avail {
4650     my ($fh, $timeout) = @_;
4651
4652     my $sel = new IO::Select;
4653     $sel->add($fh);
4654
4655     my $res = '';
4656     my $buf;
4657
4658     my @ready;
4659     while (scalar (@ready = $sel->can_read($timeout))) {
4660         my $count;
4661         if ($count = $fh->sysread($buf, 8192)) {
4662             if ($buf =~ /^(.*)\(qemu\) $/s) {
4663                 $res .= $1;
4664                 last;
4665             } else {
4666                 $res .= $buf;
4667             }
4668         } else {
4669             if (!defined($count)) {
4670                 die "$!\n";
4671             }
4672             last;
4673         }
4674     }
4675
4676     die "monitor read timeout\n" if !scalar(@ready);
4677
4678     return $res;
4679 }
4680
4681 sub qemu_block_resize {
4682     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4683
4684     my $running = check_running($vmid);
4685
4686     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4687
4688     return if !$running;
4689
4690     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4691
4692 }
4693
4694 sub qemu_volume_snapshot {
4695     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4696
4697     my $running = check_running($vmid);
4698
4699     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4700         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4701     } else {
4702         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4703     }
4704 }
4705
4706 sub qemu_volume_snapshot_delete {
4707     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4708
4709     my $running = check_running($vmid);
4710
4711     if($running) {
4712
4713         $running = undef;
4714         my $conf = PVE::QemuConfig->load_config($vmid);
4715         foreach_drive($conf, sub {
4716             my ($ds, $drive) = @_;
4717             $running = 1 if $drive->{file} eq $volid;
4718         });
4719     }
4720
4721     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4722         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4723     } else {
4724         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4725     }
4726 }
4727
4728 sub set_migration_caps {
4729     my ($vmid) = @_;
4730
4731     my $cap_ref = [];
4732
4733     my $enabled_cap = {
4734         "auto-converge" => 1,
4735         "xbzrle" => 1,
4736         "x-rdma-pin-all" => 0,
4737         "zero-blocks" => 0,
4738         "compress" => 0
4739     };
4740
4741     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4742
4743     for my $supported_capability (@$supported_capabilities) {
4744         push @$cap_ref, {
4745             capability => $supported_capability->{capability},
4746             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4747         };
4748     }
4749
4750     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4751 }
4752
4753 my $fast_plug_option = {
4754     'lock' => 1,
4755     'name' => 1,
4756     'onboot' => 1,
4757     'shares' => 1,
4758     'startup' => 1,
4759     'description' => 1,
4760     'protection' => 1,
4761     'vmstatestorage' => 1,
4762     'hookscript' => 1,
4763 };
4764
4765 # hotplug changes in [PENDING]
4766 # $selection hash can be used to only apply specified options, for
4767 # example: { cores => 1 } (only apply changed 'cores')
4768 # $errors ref is used to return error messages
4769 sub vmconfig_hotplug_pending {
4770     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4771
4772     my $defaults = load_defaults();
4773     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4774
4775     # commit values which do not have any impact on running VM first
4776     # Note: those option cannot raise errors, we we do not care about
4777     # $selection and always apply them.
4778
4779     my $add_error = sub {
4780         my ($opt, $msg) = @_;
4781         $errors->{$opt} = "hotplug problem - $msg";
4782     };
4783
4784     my $changes = 0;
4785     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4786         if ($fast_plug_option->{$opt}) {
4787             $conf->{$opt} = $conf->{pending}->{$opt};
4788             delete $conf->{pending}->{$opt};
4789             $changes = 1;
4790         }
4791     }
4792
4793     if ($changes) {
4794         PVE::QemuConfig->write_config($vmid, $conf);
4795         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4796     }
4797
4798     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4799
4800     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4801     while (my ($opt, $force) = each %$pending_delete_hash) {
4802         next if $selection && !$selection->{$opt};
4803         eval {
4804             if ($opt eq 'hotplug') {
4805                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4806             } elsif ($opt eq 'tablet') {
4807                 die "skip\n" if !$hotplug_features->{usb};
4808                 if ($defaults->{tablet}) {
4809                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4810                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4811                         if $arch eq 'aarch64';
4812                 } else {
4813                     vm_deviceunplug($vmid, $conf, 'tablet');
4814                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4815                 }
4816             } elsif ($opt =~ m/^usb\d+/) {
4817                 die "skip\n";
4818                 # since we cannot reliably hot unplug usb devices
4819                 # we are disabling it
4820                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4821                 vm_deviceunplug($vmid, $conf, $opt);
4822             } elsif ($opt eq 'vcpus') {
4823                 die "skip\n" if !$hotplug_features->{cpu};
4824                 qemu_cpu_hotplug($vmid, $conf, undef);
4825             } elsif ($opt eq 'balloon') {
4826                 # enable balloon device is not hotpluggable
4827                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4828                 # here we reset the ballooning value to memory
4829                 my $balloon = $conf->{memory} || $defaults->{memory};
4830                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4831             } elsif ($fast_plug_option->{$opt}) {
4832                 # do nothing
4833             } elsif ($opt =~ m/^net(\d+)$/) {
4834                 die "skip\n" if !$hotplug_features->{network};
4835                 vm_deviceunplug($vmid, $conf, $opt);
4836             } elsif (is_valid_drivename($opt)) {
4837                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4838                 vm_deviceunplug($vmid, $conf, $opt);
4839                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4840             } elsif ($opt =~ m/^memory$/) {
4841                 die "skip\n" if !$hotplug_features->{memory};
4842                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4843             } elsif ($opt eq 'cpuunits') {
4844                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4845             } elsif ($opt eq 'cpulimit') {
4846                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4847             } else {
4848                 die "skip\n";
4849             }
4850         };
4851         if (my $err = $@) {
4852             &$add_error($opt, $err) if $err ne "skip\n";
4853         } else {
4854             # save new config if hotplug was successful
4855             delete $conf->{$opt};
4856             vmconfig_undelete_pending_option($conf, $opt);
4857             PVE::QemuConfig->write_config($vmid, $conf);
4858             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4859         }
4860     }
4861
4862     my $apply_pending_cloudinit;
4863     $apply_pending_cloudinit = sub {
4864         my ($key, $value) = @_;
4865         $apply_pending_cloudinit = sub {}; # once is enough
4866
4867         my @cloudinit_opts = keys %$confdesc_cloudinit;
4868         foreach my $opt (keys %{$conf->{pending}}) {
4869             next if !grep { $_ eq $opt } @cloudinit_opts;
4870             $conf->{$opt} = delete $conf->{pending}->{$opt};
4871         }
4872
4873         my $new_conf = { %$conf };
4874         $new_conf->{$key} = $value;
4875         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4876     };
4877
4878     foreach my $opt (keys %{$conf->{pending}}) {
4879         next if $selection && !$selection->{$opt};
4880         my $value = $conf->{pending}->{$opt};
4881         eval {
4882             if ($opt eq 'hotplug') {
4883                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4884             } elsif ($opt eq 'tablet') {
4885                 die "skip\n" if !$hotplug_features->{usb};
4886                 if ($value == 1) {
4887                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4888                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4889                         if $arch eq 'aarch64';
4890                 } elsif ($value == 0) {
4891                     vm_deviceunplug($vmid, $conf, 'tablet');
4892                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4893                 }
4894             } elsif ($opt =~ m/^usb\d+$/) {
4895                 die "skip\n";
4896                 # since we cannot reliably hot unplug usb devices
4897                 # we are disabling it
4898                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4899                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4900                 die "skip\n" if !$d;
4901                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4902             } elsif ($opt eq 'vcpus') {
4903                 die "skip\n" if !$hotplug_features->{cpu};
4904                 qemu_cpu_hotplug($vmid, $conf, $value);
4905             } elsif ($opt eq 'balloon') {
4906                 # enable/disable balloning device is not hotpluggable
4907                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4908                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4909                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4910
4911                 # allow manual ballooning if shares is set to zero
4912                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4913                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4914                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4915                 }
4916             } elsif ($opt =~ m/^net(\d+)$/) {
4917                 # some changes can be done without hotplug
4918                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4919                                     $vmid, $opt, $value, $arch, $machine_type);
4920             } elsif (is_valid_drivename($opt)) {
4921                 # some changes can be done without hotplug
4922                 my $drive = parse_drive($opt, $value);
4923                 if (drive_is_cloudinit($drive)) {
4924                     &$apply_pending_cloudinit($opt, $value);
4925                 }
4926                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4927                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4928             } elsif ($opt =~ m/^memory$/) { #dimms
4929                 die "skip\n" if !$hotplug_features->{memory};
4930                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4931             } elsif ($opt eq 'cpuunits') {
4932                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4933             } elsif ($opt eq 'cpulimit') {
4934                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4935                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4936             } else {
4937                 die "skip\n";  # skip non-hot-pluggable options
4938             }
4939         };
4940         if (my $err = $@) {
4941             &$add_error($opt, $err) if $err ne "skip\n";
4942         } else {
4943             # save new config if hotplug was successful
4944             $conf->{$opt} = $value;
4945             delete $conf->{pending}->{$opt};
4946             PVE::QemuConfig->write_config($vmid, $conf);
4947             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4948         }
4949     }
4950 }
4951
4952 sub try_deallocate_drive {
4953     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4954
4955     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4956         my $volid = $drive->{file};
4957         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4958             my $sid = PVE::Storage::parse_volume_id($volid);
4959             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4960
4961             # check if the disk is really unused
4962             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4963                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4964             PVE::Storage::vdisk_free($storecfg, $volid);
4965             return 1;
4966         } else {
4967             # If vm is not owner of this disk remove from config
4968             return 1;
4969         }
4970     }
4971
4972     return undef;
4973 }
4974
4975 sub vmconfig_delete_or_detach_drive {
4976     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4977
4978     my $drive = parse_drive($opt, $conf->{$opt});
4979
4980     my $rpcenv = PVE::RPCEnvironment::get();
4981     my $authuser = $rpcenv->get_user();
4982
4983     if ($force) {
4984         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4985         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4986     } else {
4987         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4988     }
4989 }
4990
4991 sub vmconfig_apply_pending {
4992     my ($vmid, $conf, $storecfg) = @_;
4993
4994     # cold plug
4995
4996     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4997     while (my ($opt, $force) = each %$pending_delete_hash) {
4998         die "internal error" if $opt =~ m/^unused/;
4999         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5000         if (!defined($conf->{$opt})) {
5001             vmconfig_undelete_pending_option($conf, $opt);
5002             PVE::QemuConfig->write_config($vmid, $conf);
5003         } elsif (is_valid_drivename($opt)) {
5004             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5005             vmconfig_undelete_pending_option($conf, $opt);
5006             delete $conf->{$opt};
5007             PVE::QemuConfig->write_config($vmid, $conf);
5008         } else {
5009             vmconfig_undelete_pending_option($conf, $opt);
5010             delete $conf->{$opt};
5011             PVE::QemuConfig->write_config($vmid, $conf);
5012         }
5013     }
5014
5015     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5016
5017     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5018         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5019
5020         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5021             # skip if nothing changed
5022         } elsif (is_valid_drivename($opt)) {
5023             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5024                 if defined($conf->{$opt});
5025             $conf->{$opt} = $conf->{pending}->{$opt};
5026         } else {
5027             $conf->{$opt} = $conf->{pending}->{$opt};
5028         }
5029
5030         delete $conf->{pending}->{$opt};
5031         PVE::QemuConfig->write_config($vmid, $conf);
5032     }
5033 }
5034
5035 my $safe_num_ne = sub {
5036     my ($a, $b) = @_;
5037
5038     return 0 if !defined($a) && !defined($b);
5039     return 1 if !defined($a);
5040     return 1 if !defined($b);
5041
5042     return $a != $b;
5043 };
5044
5045 my $safe_string_ne = sub {
5046     my ($a, $b) = @_;
5047
5048     return 0 if !defined($a) && !defined($b);
5049     return 1 if !defined($a);
5050     return 1 if !defined($b);
5051
5052     return $a ne $b;
5053 };
5054
5055 sub vmconfig_update_net {
5056     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5057
5058     my $newnet = parse_net($value);
5059
5060     if ($conf->{$opt}) {
5061         my $oldnet = parse_net($conf->{$opt});
5062
5063         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5064             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5065             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5066             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5067
5068             # for non online change, we try to hot-unplug
5069             die "skip\n" if !$hotplug;
5070             vm_deviceunplug($vmid, $conf, $opt);
5071         } else {
5072
5073             die "internal error" if $opt !~ m/net(\d+)/;
5074             my $iface = "tap${vmid}i$1";
5075
5076             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5077                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5078                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5079                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5080                 PVE::Network::tap_unplug($iface);
5081                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5082             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5083                 # Rate can be applied on its own but any change above needs to
5084                 # include the rate in tap_plug since OVS resets everything.
5085                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5086             }
5087
5088             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5089                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5090             }
5091
5092             return 1;
5093         }
5094     }
5095
5096     if ($hotplug) {
5097         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5098     } else {
5099         die "skip\n";
5100     }
5101 }
5102
5103 sub vmconfig_update_disk {
5104     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5105
5106     # fixme: do we need force?
5107
5108     my $drive = parse_drive($opt, $value);
5109
5110     if ($conf->{$opt}) {
5111
5112         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5113
5114             my $media = $drive->{media} || 'disk';
5115             my $oldmedia = $old_drive->{media} || 'disk';
5116             die "unable to change media type\n" if $media ne $oldmedia;
5117
5118             if (!drive_is_cdrom($old_drive)) {
5119
5120                 if ($drive->{file} ne $old_drive->{file}) {
5121
5122                     die "skip\n" if !$hotplug;
5123
5124                     # unplug and register as unused
5125                     vm_deviceunplug($vmid, $conf, $opt);
5126                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5127
5128                 } else {
5129                     # update existing disk
5130
5131                     # skip non hotpluggable value
5132                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5133                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5134                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5135                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5136                         die "skip\n";
5137                     }
5138
5139                     # apply throttle
5140                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5141                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5142                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5143                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5144                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5145                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5146                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5147                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5148                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5149                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5150                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5151                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5152                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5153                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5154                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5155                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5156                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5157                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5158
5159                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5160                                                    ($drive->{mbps} || 0)*1024*1024,
5161                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5162                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5163                                                    $drive->{iops} || 0,
5164                                                    $drive->{iops_rd} || 0,
5165                                                    $drive->{iops_wr} || 0,
5166                                                    ($drive->{mbps_max} || 0)*1024*1024,
5167                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5168                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5169                                                    $drive->{iops_max} || 0,
5170                                                    $drive->{iops_rd_max} || 0,
5171                                                    $drive->{iops_wr_max} || 0,
5172                                                    $drive->{bps_max_length} || 1,
5173                                                    $drive->{bps_rd_max_length} || 1,
5174                                                    $drive->{bps_wr_max_length} || 1,
5175                                                    $drive->{iops_max_length} || 1,
5176                                                    $drive->{iops_rd_max_length} || 1,
5177                                                    $drive->{iops_wr_max_length} || 1);
5178
5179                     }
5180
5181                     return 1;
5182                 }
5183
5184             } else { # cdrom
5185
5186                 if ($drive->{file} eq 'none') {
5187                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5188                     if (drive_is_cloudinit($old_drive)) {
5189                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5190                     }
5191                 } else {
5192                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5193                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5194                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5195                 }
5196
5197                 return 1;
5198             }
5199         }
5200     }
5201
5202     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5203     # hotplug new disks
5204     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5205     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5206 }
5207
5208 sub vm_start {
5209     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5210         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5211
5212     PVE::QemuConfig->lock_config($vmid, sub {
5213         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5214
5215         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5216
5217         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5218
5219         PVE::QemuConfig->check_lock($conf)
5220             if !($skiplock || $is_suspended);
5221
5222         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5223
5224         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5225             vmconfig_apply_pending($vmid, $conf, $storecfg);
5226             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5227         }
5228
5229         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5230
5231         my $defaults = load_defaults();
5232
5233         # set environment variable useful inside network script
5234         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5235
5236         my $local_volumes = {};
5237
5238         if ($targetstorage) {
5239             foreach_drive($conf, sub {
5240                 my ($ds, $drive) = @_;
5241
5242                 return if drive_is_cdrom($drive);
5243
5244                 my $volid = $drive->{file};
5245
5246                 return if !$volid;
5247
5248                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5249
5250                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5251                 return if $scfg->{shared};
5252                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5253             });
5254
5255             my $format = undef;
5256
5257             foreach my $opt (sort keys %$local_volumes) {
5258
5259                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5260                 my $drive = parse_drive($opt, $conf->{$opt});
5261
5262                 #if remote storage is specified, use default format
5263                 if ($targetstorage && $targetstorage ne "1") {
5264                     $storeid = $targetstorage;
5265                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5266                     $format = $defFormat;
5267                 } else {
5268                     #else we use same format than original
5269                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5270                     $format = qemu_img_format($scfg, $volid);
5271                 }
5272
5273                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5274                 my $newdrive = $drive;
5275                 $newdrive->{format} = $format;
5276                 $newdrive->{file} = $newvolid;
5277                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5278                 $local_volumes->{$opt} = $drivestr;
5279                 #pass drive to conf for command line
5280                 $conf->{$opt} = $drivestr;
5281             }
5282         }
5283
5284         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5285
5286         if ($is_suspended) {
5287             # enforce machine type on suspended vm to ensure HW compatibility
5288             $forcemachine = $conf->{runningmachine};
5289             print "Resuming suspended VM\n";
5290         }
5291
5292         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5293
5294         my $migrate_port = 0;
5295         my $migrate_uri;
5296         if ($statefile) {
5297             if ($statefile eq 'tcp') {
5298                 my $localip = "localhost";
5299                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5300                 my $nodename = PVE::INotify::nodename();
5301
5302                 if (!defined($migration_type)) {
5303                     if (defined($datacenterconf->{migration}->{type})) {
5304                         $migration_type = $datacenterconf->{migration}->{type};
5305                     } else {
5306                         $migration_type = 'secure';
5307                     }
5308                 }
5309
5310                 if ($migration_type eq 'insecure') {
5311                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5312                     if ($migrate_network_addr) {
5313                         $localip = $migrate_network_addr;
5314                     } else {
5315                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5316                     }
5317
5318                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5319                 }
5320
5321                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5322                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5323                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5324                 push @$cmd, '-incoming', $migrate_uri;
5325                 push @$cmd, '-S';
5326
5327             } elsif ($statefile eq 'unix') {
5328                 # should be default for secure migrations as a ssh TCP forward
5329                 # tunnel is not deterministic reliable ready and fails regurarly
5330                 # to set up in time, so use UNIX socket forwards
5331                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5332                 unlink $socket_addr;
5333
5334                 $migrate_uri = "unix:$socket_addr";
5335
5336                 push @$cmd, '-incoming', $migrate_uri;
5337                 push @$cmd, '-S';
5338
5339             } else {
5340                 push @$cmd, '-loadstate', $statefile;
5341             }
5342         } elsif ($paused) {
5343             push @$cmd, '-S';
5344         }
5345
5346         # host pci devices
5347         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5348           my $d = parse_hostpci($conf->{"hostpci$i"});
5349           next if !$d;
5350           my $pcidevices = $d->{pciid};
5351           foreach my $pcidevice (@$pcidevices) {
5352                 my $pciid = $pcidevice->{id};
5353
5354                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5355                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5356                 die "no pci device info for device '$pciid'\n" if !$info;
5357
5358                 if ($d->{mdev}) {
5359                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5360                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5361                 } else {
5362                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5363                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5364                     die "can't reset pci device '$pciid'\n"
5365                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5366                 }
5367           }
5368         }
5369
5370         PVE::Storage::activate_volumes($storecfg, $vollist);
5371
5372         eval {
5373             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5374                 outfunc => sub {}, errfunc => sub {});
5375         };
5376         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5377         # timeout should be more than enough here...
5378         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5379
5380         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5381                                                   : $defaults->{cpuunits};
5382
5383         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5384         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5385
5386         my %properties = (
5387             Slice => 'qemu.slice',
5388             KillMode => 'none',
5389             CPUShares => $cpuunits
5390         );
5391
5392         if (my $cpulimit = $conf->{cpulimit}) {
5393             $properties{CPUQuota} = int($cpulimit * 100);
5394         }
5395         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5396
5397         my $run_qemu = sub {
5398             PVE::Tools::run_fork sub {
5399                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5400                 run_command($cmd, %run_params);
5401             };
5402         };
5403
5404         if ($conf->{hugepages}) {
5405
5406             my $code = sub {
5407                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5408                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5409
5410                 PVE::QemuServer::Memory::hugepages_mount();
5411                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5412
5413                 eval { $run_qemu->() };
5414                 if (my $err = $@) {
5415                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5416                     die $err;
5417                 }
5418
5419                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5420             };
5421             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5422
5423         } else {
5424             eval { $run_qemu->() };
5425         }
5426
5427         if (my $err = $@) {
5428             # deactivate volumes if start fails
5429             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5430             die "start failed: $err";
5431         }
5432
5433         print "migration listens on $migrate_uri\n" if $migrate_uri;
5434
5435         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5436             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5437             warn $@ if $@;
5438         }
5439
5440         #start nbd server for storage migration
5441         if ($targetstorage) {
5442             my $nodename = PVE::INotify::nodename();
5443             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5444             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5445             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5446             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5447
5448             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5449
5450             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5451
5452             foreach my $opt (sort keys %$local_volumes) {
5453                 my $volid = $local_volumes->{$opt};
5454                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5455                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5456                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5457             }
5458         }
5459
5460         if ($migratedfrom) {
5461             eval {
5462                 set_migration_caps($vmid);
5463             };
5464             warn $@ if $@;
5465
5466             if ($spice_port) {
5467                 print "spice listens on port $spice_port\n";
5468                 if ($spice_ticket) {
5469                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5470                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5471                 }
5472             }
5473
5474         } else {
5475             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5476                 if !$statefile && $conf->{balloon};
5477
5478             foreach my $opt (keys %$conf) {
5479                 next if $opt !~  m/^net\d+$/;
5480                 my $nicconf = parse_net($conf->{$opt});
5481                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5482             }
5483         }
5484
5485         vm_mon_cmd_nocheck($vmid, 'qom-set',
5486                     path => "machine/peripheral/balloon0",
5487                     property => "guest-stats-polling-interval",
5488                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5489
5490         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5491             print "Resumed VM, removing state\n";
5492             delete $conf->@{qw(lock vmstate runningmachine)};
5493             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5494             PVE::Storage::vdisk_free($storecfg, $vmstate);
5495             PVE::QemuConfig->write_config($vmid, $conf);
5496         }
5497
5498         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5499     });
5500 }
5501
5502 sub vm_mon_cmd {
5503     my ($vmid, $execute, %params) = @_;
5504
5505     my $cmd = { execute => $execute, arguments => \%params };
5506     vm_qmp_command($vmid, $cmd);
5507 }
5508
5509 sub vm_mon_cmd_nocheck {
5510     my ($vmid, $execute, %params) = @_;
5511
5512     my $cmd = { execute => $execute, arguments => \%params };
5513     vm_qmp_command($vmid, $cmd, 1);
5514 }
5515
5516 sub vm_qmp_command {
5517     my ($vmid, $cmd, $nocheck) = @_;
5518
5519     my $res;
5520
5521     my $timeout;
5522     if ($cmd->{arguments}) {
5523         $timeout = delete $cmd->{arguments}->{timeout};
5524     }
5525
5526     eval {
5527         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5528         my $sname = qmp_socket($vmid);
5529         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5530             my $qmpclient = PVE::QMPClient->new();
5531
5532             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5533         } else {
5534             die "unable to open monitor socket\n";
5535         }
5536     };
5537     if (my $err = $@) {
5538         syslog("err", "VM $vmid qmp command failed - $err");
5539         die $err;
5540     }
5541
5542     return $res;
5543 }
5544
5545 sub vm_human_monitor_command {
5546     my ($vmid, $cmdline) = @_;
5547
5548     my $res;
5549
5550     my $cmd = {
5551         execute => 'human-monitor-command',
5552         arguments => { 'command-line' => $cmdline},
5553     };
5554
5555     return vm_qmp_command($vmid, $cmd);
5556 }
5557
5558 sub vm_commandline {
5559     my ($storecfg, $vmid, $snapname) = @_;
5560
5561     my $conf = PVE::QemuConfig->load_config($vmid);
5562
5563     if ($snapname) {
5564         my $snapshot = $conf->{snapshots}->{$snapname};
5565         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5566
5567         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5568
5569         $conf = $snapshot;
5570     }
5571
5572     my $defaults = load_defaults();
5573
5574     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5575
5576     return PVE::Tools::cmd2string($cmd);
5577 }
5578
5579 sub vm_reset {
5580     my ($vmid, $skiplock) = @_;
5581
5582     PVE::QemuConfig->lock_config($vmid, sub {
5583
5584         my $conf = PVE::QemuConfig->load_config($vmid);
5585
5586         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5587
5588         vm_mon_cmd($vmid, "system_reset");
5589     });
5590 }
5591
5592 sub get_vm_volumes {
5593     my ($conf) = @_;
5594
5595     my $vollist = [];
5596     foreach_volid($conf, sub {
5597         my ($volid, $attr) = @_;
5598
5599         return if $volid =~ m|^/|;
5600
5601         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5602         return if !$sid;
5603
5604         push @$vollist, $volid;
5605     });
5606
5607     return $vollist;
5608 }
5609
5610 sub vm_stop_cleanup {
5611     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5612
5613     eval {
5614
5615         if (!$keepActive) {
5616             my $vollist = get_vm_volumes($conf);
5617             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5618         }
5619
5620         foreach my $ext (qw(mon qmp pid vnc qga)) {
5621             unlink "/var/run/qemu-server/${vmid}.$ext";
5622         }
5623
5624         if ($conf->{ivshmem}) {
5625             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5626             # just delete it for now, VMs which have this already open do not
5627             # are affected, but new VMs will get a separated one. If this
5628             # becomes an issue we either add some sort of ref-counting or just
5629             # add a "don't delete on stop" flag to the ivshmem format.
5630             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5631         }
5632
5633         foreach my $key (keys %$conf) {
5634             next if $key !~ m/^hostpci(\d+)$/;
5635             my $hostpciindex = $1;
5636             my $d = parse_hostpci($conf->{$key});
5637             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5638
5639             foreach my $pci (@{$d->{pciid}}) {
5640                 my $pciid = $pci->{id};
5641                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5642             }
5643         }
5644
5645         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5646     };
5647     warn $@ if $@; # avoid errors - just warn
5648 }
5649
5650 # Note: use $nockeck to skip tests if VM configuration file exists.
5651 # We need that when migration VMs to other nodes (files already moved)
5652 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5653 sub vm_stop {
5654     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5655
5656     $force = 1 if !defined($force) && !$shutdown;
5657
5658     if ($migratedfrom){
5659         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5660         kill 15, $pid if $pid;
5661         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5662         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5663         return;
5664     }
5665
5666     PVE::QemuConfig->lock_config($vmid, sub {
5667
5668         my $pid = check_running($vmid, $nocheck);
5669         return if !$pid;
5670
5671         my $conf;
5672         if (!$nocheck) {
5673             $conf = PVE::QemuConfig->load_config($vmid);
5674             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5675             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5676                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5677                 $timeout = $opts->{down} if $opts->{down};
5678             }
5679             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5680         }
5681
5682         eval {
5683             if ($shutdown) {
5684                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5685                     vm_qmp_command($vmid, {
5686                         execute => "guest-shutdown",
5687                         arguments => { timeout => $timeout }
5688                     }, $nocheck);
5689                 } else {
5690                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5691                 }
5692             } else {
5693                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5694             }
5695         };
5696         my $err = $@;
5697
5698         if (!$err) {
5699             $timeout = 60 if !defined($timeout);
5700
5701             my $count = 0;
5702             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5703                 $count++;
5704                 sleep 1;
5705             }
5706
5707             if ($count >= $timeout) {
5708                 if ($force) {
5709                     warn "VM still running - terminating now with SIGTERM\n";
5710                     kill 15, $pid;
5711                 } else {
5712                     die "VM quit/powerdown failed - got timeout\n";
5713                 }
5714             } else {
5715                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5716                 return;
5717             }
5718         } else {
5719             if ($force) {
5720                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5721                 kill 15, $pid;
5722             } else {
5723                 die "VM quit/powerdown failed\n";
5724             }
5725         }
5726
5727         # wait again
5728         $timeout = 10;
5729
5730         my $count = 0;
5731         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5732             $count++;
5733             sleep 1;
5734         }
5735
5736         if ($count >= $timeout) {
5737             warn "VM still running - terminating now with SIGKILL\n";
5738             kill 9, $pid;
5739             sleep 1;
5740         }
5741
5742         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5743    });
5744 }
5745
5746 sub vm_suspend {
5747     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5748
5749     my $conf;
5750     my $path;
5751     my $storecfg;
5752     my $vmstate;
5753
5754     PVE::QemuConfig->lock_config($vmid, sub {
5755
5756         $conf = PVE::QemuConfig->load_config($vmid);
5757
5758         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5759         PVE::QemuConfig->check_lock($conf)
5760             if !($skiplock || $is_backing_up);
5761
5762         die "cannot suspend to disk during backup\n"
5763             if $is_backing_up && $includestate;
5764
5765         if ($includestate) {
5766             $conf->{lock} = 'suspending';
5767             my $date = strftime("%Y-%m-%d", localtime(time()));
5768             $storecfg = PVE::Storage::config();
5769             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5770             $path = PVE::Storage::path($storecfg, $vmstate);
5771             PVE::QemuConfig->write_config($vmid, $conf);
5772         } else {
5773             vm_mon_cmd($vmid, "stop");
5774         }
5775     });
5776
5777     if ($includestate) {
5778         # save vm state
5779         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5780
5781         eval {
5782             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5783             for(;;) {
5784                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5785                 if (!$state->{status}) {
5786                     die "savevm not active\n";
5787                 } elsif ($state->{status} eq 'active') {
5788                     sleep(1);
5789                     next;
5790                 } elsif ($state->{status} eq 'completed') {
5791                     print "State saved, quitting\n";
5792                     last;
5793                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5794                     die "query-savevm failed with error '$state->{error}'\n"
5795                 } else {
5796                     die "query-savevm returned status '$state->{status}'\n";
5797                 }
5798             }
5799         };
5800         my $err = $@;
5801
5802         PVE::QemuConfig->lock_config($vmid, sub {
5803             $conf = PVE::QemuConfig->load_config($vmid);
5804             if ($err) {
5805                 # cleanup, but leave suspending lock, to indicate something went wrong
5806                 eval {
5807                     vm_mon_cmd($vmid, "savevm-end");
5808                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5809                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5810                     delete $conf->@{qw(vmstate runningmachine)};
5811                     PVE::QemuConfig->write_config($vmid, $conf);
5812                 };
5813                 warn $@ if $@;
5814                 die $err;
5815             }
5816
5817             die "lock changed unexpectedly\n"
5818                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5819
5820             vm_qmp_command($vmid, { execute => "quit" });
5821             $conf->{lock} = 'suspended';
5822             PVE::QemuConfig->write_config($vmid, $conf);
5823         });
5824     }
5825 }
5826
5827 sub vm_resume {
5828     my ($vmid, $skiplock, $nocheck) = @_;
5829
5830     PVE::QemuConfig->lock_config($vmid, sub {
5831         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5832         my $res = $vm_mon_cmd->($vmid, 'query-status');
5833         my $resume_cmd = 'cont';
5834
5835         if ($res->{status} && $res->{status} eq 'suspended') {
5836             $resume_cmd = 'system_wakeup';
5837         }
5838
5839         if (!$nocheck) {
5840
5841             my $conf = PVE::QemuConfig->load_config($vmid);
5842
5843             PVE::QemuConfig->check_lock($conf)
5844                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5845         }
5846
5847         $vm_mon_cmd->($vmid, $resume_cmd);
5848     });
5849 }
5850
5851 sub vm_sendkey {
5852     my ($vmid, $skiplock, $key) = @_;
5853
5854     PVE::QemuConfig->lock_config($vmid, sub {
5855
5856         my $conf = PVE::QemuConfig->load_config($vmid);
5857
5858         # there is no qmp command, so we use the human monitor command
5859         vm_human_monitor_command($vmid, "sendkey $key");
5860     });
5861 }
5862
5863 sub vm_destroy {
5864     my ($storecfg, $vmid, $skiplock) = @_;
5865
5866     PVE::QemuConfig->lock_config($vmid, sub {
5867
5868         my $conf = PVE::QemuConfig->load_config($vmid);
5869
5870         if (!check_running($vmid)) {
5871             destroy_vm($storecfg, $vmid, undef, $skiplock);
5872         } else {
5873             die "VM $vmid is running - destroy failed\n";
5874         }
5875     });
5876 }
5877
5878 # vzdump restore implementaion
5879
5880 sub tar_archive_read_firstfile {
5881     my $archive = shift;
5882
5883     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5884
5885     # try to detect archive type first
5886     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5887         die "unable to open file '$archive'\n";
5888     my $firstfile = <$fh>;
5889     kill 15, $pid;
5890     close $fh;
5891
5892     die "ERROR: archive contaions no data\n" if !$firstfile;
5893     chomp $firstfile;
5894
5895     return $firstfile;
5896 }
5897
5898 sub tar_restore_cleanup {
5899     my ($storecfg, $statfile) = @_;
5900
5901     print STDERR "starting cleanup\n";
5902
5903     if (my $fd = IO::File->new($statfile, "r")) {
5904         while (defined(my $line = <$fd>)) {
5905             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5906                 my $volid = $2;
5907                 eval {
5908                     if ($volid =~ m|^/|) {
5909                         unlink $volid || die 'unlink failed\n';
5910                     } else {
5911                         PVE::Storage::vdisk_free($storecfg, $volid);
5912                     }
5913                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5914                 };
5915                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5916             } else {
5917                 print STDERR "unable to parse line in statfile - $line";
5918             }
5919         }
5920         $fd->close();
5921     }
5922 }
5923
5924 sub restore_archive {
5925     my ($archive, $vmid, $user, $opts) = @_;
5926
5927     my $format = $opts->{format};
5928     my $comp;
5929
5930     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5931         $format = 'tar' if !$format;
5932         $comp = 'gzip';
5933     } elsif ($archive =~ m/\.tar$/) {
5934         $format = 'tar' if !$format;
5935     } elsif ($archive =~ m/.tar.lzo$/) {
5936         $format = 'tar' if !$format;
5937         $comp = 'lzop';
5938     } elsif ($archive =~ m/\.vma$/) {
5939         $format = 'vma' if !$format;
5940     } elsif ($archive =~ m/\.vma\.gz$/) {
5941         $format = 'vma' if !$format;
5942         $comp = 'gzip';
5943     } elsif ($archive =~ m/\.vma\.lzo$/) {
5944         $format = 'vma' if !$format;
5945         $comp = 'lzop';
5946     } else {
5947         $format = 'vma' if !$format; # default
5948     }
5949
5950     # try to detect archive format
5951     if ($format eq 'tar') {
5952         return restore_tar_archive($archive, $vmid, $user, $opts);
5953     } else {
5954         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5955     }
5956 }
5957
5958 sub restore_update_config_line {
5959     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5960
5961     return if $line =~ m/^\#qmdump\#/;
5962     return if $line =~ m/^\#vzdump\#/;
5963     return if $line =~ m/^lock:/;
5964     return if $line =~ m/^unused\d+:/;
5965     return if $line =~ m/^parent:/;
5966
5967     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5968     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5969         # try to convert old 1.X settings
5970         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5971         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5972             my ($model, $macaddr) = split(/\=/, $devconfig);
5973             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5974             my $net = {
5975                 model => $model,
5976                 bridge => "vmbr$ind",
5977                 macaddr => $macaddr,
5978             };
5979             my $netstr = print_net($net);
5980
5981             print $outfd "net$cookie->{netcount}: $netstr\n";
5982             $cookie->{netcount}++;
5983         }
5984     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5985         my ($id, $netstr) = ($1, $2);
5986         my $net = parse_net($netstr);
5987         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5988         $netstr = print_net($net);
5989         print $outfd "$id: $netstr\n";
5990     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5991         my $virtdev = $1;
5992         my $value = $3;
5993         my $di = parse_drive($virtdev, $value);
5994         if (defined($di->{backup}) && !$di->{backup}) {
5995             print $outfd "#$line";
5996         } elsif ($map->{$virtdev}) {
5997             delete $di->{format}; # format can change on restore
5998             $di->{file} = $map->{$virtdev};
5999             $value = print_drive($vmid, $di);
6000             print $outfd "$virtdev: $value\n";
6001         } else {
6002             print $outfd $line;
6003         }
6004     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6005         my $vmgenid = $1;
6006         if ($vmgenid ne '0') {
6007             # always generate a new vmgenid if there was a valid one setup
6008             $vmgenid = generate_uuid();
6009         }
6010         print $outfd "vmgenid: $vmgenid\n";
6011     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6012         my ($uuid, $uuid_str);
6013         UUID::generate($uuid);
6014         UUID::unparse($uuid, $uuid_str);
6015         my $smbios1 = parse_smbios1($2);
6016         $smbios1->{uuid} = $uuid_str;
6017         print $outfd $1.print_smbios1($smbios1)."\n";
6018     } else {
6019         print $outfd $line;
6020     }
6021 }
6022
6023 sub scan_volids {
6024     my ($cfg, $vmid) = @_;
6025
6026     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6027
6028     my $volid_hash = {};
6029     foreach my $storeid (keys %$info) {
6030         foreach my $item (@{$info->{$storeid}}) {
6031             next if !($item->{volid} && $item->{size});
6032             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6033             $volid_hash->{$item->{volid}} = $item;
6034         }
6035     }
6036
6037     return $volid_hash;
6038 }
6039
6040 sub is_volume_in_use {
6041     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6042
6043     my $path = PVE::Storage::path($storecfg, $volid);
6044
6045     my $scan_config = sub {
6046         my ($cref, $snapname) = @_;
6047
6048         foreach my $key (keys %$cref) {
6049             my $value = $cref->{$key};
6050             if (is_valid_drivename($key)) {
6051                 next if $skip_drive && $key eq $skip_drive;
6052                 my $drive = parse_drive($key, $value);
6053                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6054                 return 1 if $volid eq $drive->{file};
6055                 if ($drive->{file} =~ m!^/!) {
6056                     return 1 if $drive->{file} eq $path;
6057                 } else {
6058                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6059                     next if !$storeid;
6060                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6061                     next if !$scfg;
6062                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6063                 }
6064             }
6065         }
6066
6067         return 0;
6068     };
6069
6070     return 1 if &$scan_config($conf);
6071
6072     undef $skip_drive;
6073
6074     foreach my $snapname (keys %{$conf->{snapshots}}) {
6075         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6076     }
6077
6078     return 0;
6079 }
6080
6081 sub update_disksize {
6082     my ($vmid, $conf, $volid_hash) = @_;
6083
6084     my $changes;
6085     my $prefix = "VM $vmid:";
6086
6087     # used and unused disks
6088     my $referenced = {};
6089
6090     # Note: it is allowed to define multiple storages with same path (alias), so
6091     # we need to check both 'volid' and real 'path' (two different volid can point
6092     # to the same path).
6093
6094     my $referencedpath = {};
6095
6096     # update size info
6097     foreach my $opt (keys %$conf) {
6098         if (is_valid_drivename($opt)) {
6099             my $drive = parse_drive($opt, $conf->{$opt});
6100             my $volid = $drive->{file};
6101             next if !$volid;
6102
6103             $referenced->{$volid} = 1;
6104             if ($volid_hash->{$volid} &&
6105                 (my $path = $volid_hash->{$volid}->{path})) {
6106                 $referencedpath->{$path} = 1;
6107             }
6108
6109             next if drive_is_cdrom($drive);
6110             next if !$volid_hash->{$volid};
6111
6112             $drive->{size} = $volid_hash->{$volid}->{size};
6113             my $new = print_drive($vmid, $drive);
6114             if ($new ne $conf->{$opt}) {
6115                 $changes = 1;
6116                 $conf->{$opt} = $new;
6117                 print "$prefix update disk '$opt' information.\n";
6118             }
6119         }
6120     }
6121
6122     # remove 'unusedX' entry if volume is used
6123     foreach my $opt (keys %$conf) {
6124         next if $opt !~ m/^unused\d+$/;
6125         my $volid = $conf->{$opt};
6126         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6127         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6128             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6129             $changes = 1;
6130             delete $conf->{$opt};
6131         }
6132
6133         $referenced->{$volid} = 1;
6134         $referencedpath->{$path} = 1 if $path;
6135     }
6136
6137     foreach my $volid (sort keys %$volid_hash) {
6138         next if $volid =~ m/vm-$vmid-state-/;
6139         next if $referenced->{$volid};
6140         my $path = $volid_hash->{$volid}->{path};
6141         next if !$path; # just to be sure
6142         next if $referencedpath->{$path};
6143         $changes = 1;
6144         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6145         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6146         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6147     }
6148
6149     return $changes;
6150 }
6151
6152 sub rescan {
6153     my ($vmid, $nolock, $dryrun) = @_;
6154
6155     my $cfg = PVE::Storage::config();
6156
6157     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6158     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6159     foreach my $stor (keys %{$cfg->{ids}}) {
6160         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6161     }
6162
6163     print "rescan volumes...\n";
6164     my $volid_hash = scan_volids($cfg, $vmid);
6165
6166     my $updatefn =  sub {
6167         my ($vmid) = @_;
6168
6169         my $conf = PVE::QemuConfig->load_config($vmid);
6170
6171         PVE::QemuConfig->check_lock($conf);
6172
6173         my $vm_volids = {};
6174         foreach my $volid (keys %$volid_hash) {
6175             my $info = $volid_hash->{$volid};
6176             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6177         }
6178
6179         my $changes = update_disksize($vmid, $conf, $vm_volids);
6180
6181         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6182     };
6183
6184     if (defined($vmid)) {
6185         if ($nolock) {
6186             &$updatefn($vmid);
6187         } else {
6188             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6189         }
6190     } else {
6191         my $vmlist = config_list();
6192         foreach my $vmid (keys %$vmlist) {
6193             if ($nolock) {
6194                 &$updatefn($vmid);
6195             } else {
6196                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6197             }
6198         }
6199     }
6200 }
6201
6202 sub restore_vma_archive {
6203     my ($archive, $vmid, $user, $opts, $comp) = @_;
6204
6205     my $readfrom = $archive;
6206
6207     my $cfg = PVE::Storage::config();
6208     my $commands = [];
6209     my $bwlimit = $opts->{bwlimit};
6210
6211     my $dbg_cmdstring = '';
6212     my $add_pipe = sub {
6213         my ($cmd) = @_;
6214         push @$commands, $cmd;
6215         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6216         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6217         $readfrom = '-';
6218     };
6219
6220     my $input = undef;
6221     if ($archive eq '-') {
6222         $input = '<&STDIN';
6223     } else {
6224         # If we use a backup from a PVE defined storage we also consider that
6225         # storage's rate limit:
6226         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6227         if (defined($volid)) {
6228             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6229             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6230             if ($readlimit) {
6231                 print STDERR "applying read rate limit: $readlimit\n";
6232                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6233                 $add_pipe->($cstream);
6234             }
6235         }
6236     }
6237
6238     if ($comp) {
6239         my $cmd;
6240         if ($comp eq 'gzip') {
6241             $cmd = ['zcat', $readfrom];
6242         } elsif ($comp eq 'lzop') {
6243             $cmd = ['lzop', '-d', '-c', $readfrom];
6244         } else {
6245             die "unknown compression method '$comp'\n";
6246         }
6247         $add_pipe->($cmd);
6248     }
6249
6250     my $tmpdir = "/var/tmp/vzdumptmp$$";
6251     rmtree $tmpdir;
6252
6253     # disable interrupts (always do cleanups)
6254     local $SIG{INT} =
6255         local $SIG{TERM} =
6256         local $SIG{QUIT} =
6257         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6258
6259     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6260     POSIX::mkfifo($mapfifo, 0600);
6261     my $fifofh;
6262
6263     my $openfifo = sub {
6264         open($fifofh, '>', $mapfifo) || die $!;
6265     };
6266
6267     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6268
6269     my $oldtimeout;
6270     my $timeout = 5;
6271
6272     my $devinfo = {};
6273
6274     my $rpcenv = PVE::RPCEnvironment::get();
6275
6276     my $conffile = PVE::QemuConfig->config_file($vmid);
6277     my $tmpfn = "$conffile.$$.tmp";
6278
6279     # Note: $oldconf is undef if VM does not exists
6280     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6281     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6282
6283     my %storage_limits;
6284
6285     my $print_devmap = sub {
6286         my $virtdev_hash = {};
6287
6288         my $cfgfn = "$tmpdir/qemu-server.conf";
6289
6290         # we can read the config - that is already extracted
6291         my $fh = IO::File->new($cfgfn, "r") ||
6292             "unable to read qemu-server.conf - $!\n";
6293
6294         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6295         if (-f $fwcfgfn) {
6296             my $pve_firewall_dir = '/etc/pve/firewall';
6297             mkdir $pve_firewall_dir; # make sure the dir exists
6298             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6299         }
6300
6301         while (defined(my $line = <$fh>)) {
6302             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6303                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6304                 die "archive does not contain data for drive '$virtdev'\n"
6305                     if !$devinfo->{$devname};
6306                 if (defined($opts->{storage})) {
6307                     $storeid = $opts->{storage} || 'local';
6308                 } elsif (!$storeid) {
6309                     $storeid = 'local';
6310                 }
6311                 $format = 'raw' if !$format;
6312                 $devinfo->{$devname}->{devname} = $devname;
6313                 $devinfo->{$devname}->{virtdev} = $virtdev;
6314                 $devinfo->{$devname}->{format} = $format;
6315                 $devinfo->{$devname}->{storeid} = $storeid;
6316
6317                 # check permission on storage
6318                 my $pool = $opts->{pool}; # todo: do we need that?
6319                 if ($user ne 'root@pam') {
6320                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6321                 }
6322
6323                 $storage_limits{$storeid} = $bwlimit;
6324
6325                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6326             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6327                 my $virtdev = $1;
6328                 my $drive = parse_drive($virtdev, $2);
6329                 if (drive_is_cloudinit($drive)) {
6330                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6331                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6332                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6333
6334                     my $d = {
6335                         format => $format,
6336                         storeid => $opts->{storage} // $storeid,
6337                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6338                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6339                         name => "vm-$vmid-cloudinit",
6340                         is_cloudinit => 1,
6341                     };
6342                     $virtdev_hash->{$virtdev} = $d;
6343                 }
6344             }
6345         }
6346
6347         foreach my $key (keys %storage_limits) {
6348             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6349             next if !$limit;
6350             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6351             $storage_limits{$key} = $limit * 1024;
6352         }
6353
6354         foreach my $devname (keys %$devinfo) {
6355             die "found no device mapping information for device '$devname'\n"
6356                 if !$devinfo->{$devname}->{virtdev};
6357         }
6358
6359         # create empty/temp config
6360         if ($oldconf) {
6361             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6362             foreach_drive($oldconf, sub {
6363                 my ($ds, $drive) = @_;
6364
6365                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6366
6367                 my $volid = $drive->{file};
6368                 return if !$volid || $volid =~ m|^/|;
6369
6370                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6371                 return if !$path || !$owner || ($owner != $vmid);
6372
6373                 # Note: only delete disk we want to restore
6374                 # other volumes will become unused
6375                 if ($virtdev_hash->{$ds}) {
6376                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6377                     if (my $err = $@) {
6378                         warn $err;
6379                     }
6380                 }
6381             });
6382
6383             # delete vmstate files, after the restore we have no snapshots anymore
6384             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6385                 my $snap = $oldconf->{snapshots}->{$snapname};
6386                 if ($snap->{vmstate}) {
6387                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6388                     if (my $err = $@) {
6389                         warn $err;
6390                     }
6391                 }
6392             }
6393         }
6394
6395         my $map = {};
6396         foreach my $virtdev (sort keys %$virtdev_hash) {
6397             my $d = $virtdev_hash->{$virtdev};
6398             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6399             my $storeid = $d->{storeid};
6400             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6401
6402             my $map_opts = '';
6403             if (my $limit = $storage_limits{$storeid}) {
6404                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6405             }
6406
6407             # test if requested format is supported
6408             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6409             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6410             $d->{format} = $defFormat if !$supported;
6411
6412             my $name;
6413             if ($d->{is_cloudinit}) {
6414                 $name = $d->{name};
6415                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6416             }
6417
6418             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6419             print STDERR "new volume ID is '$volid'\n";
6420             $d->{volid} = $volid;
6421
6422             PVE::Storage::activate_volumes($cfg, [$volid]);
6423
6424             my $write_zeros = 1;
6425             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6426                 $write_zeros = 0;
6427             }
6428
6429             if (!$d->{is_cloudinit}) {
6430                 my $path = PVE::Storage::path($cfg, $volid);
6431
6432                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6433
6434                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6435             }
6436             $map->{$virtdev} = $volid;
6437         }
6438
6439         $fh->seek(0, 0) || die "seek failed - $!\n";
6440
6441         my $outfd = new IO::File ($tmpfn, "w") ||
6442             die "unable to write config for VM $vmid\n";
6443
6444         my $cookie = { netcount => 0 };
6445         while (defined(my $line = <$fh>)) {
6446             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6447         }
6448
6449         $fh->close();
6450         $outfd->close();
6451     };
6452
6453     eval {
6454         # enable interrupts
6455         local $SIG{INT} =
6456             local $SIG{TERM} =
6457             local $SIG{QUIT} =
6458             local $SIG{HUP} =
6459             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6460         local $SIG{ALRM} = sub { die "got timeout\n"; };
6461
6462         $oldtimeout = alarm($timeout);
6463
6464         my $parser = sub {
6465             my $line = shift;
6466
6467             print "$line\n";
6468
6469             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6470                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6471                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6472             } elsif ($line =~ m/^CTIME: /) {
6473                 # we correctly received the vma config, so we can disable
6474                 # the timeout now for disk allocation (set to 10 minutes, so
6475                 # that we always timeout if something goes wrong)
6476                 alarm(600);
6477                 &$print_devmap();
6478                 print $fifofh "done\n";
6479                 my $tmp = $oldtimeout || 0;
6480                 $oldtimeout = undef;
6481                 alarm($tmp);
6482                 close($fifofh);
6483             }
6484         };
6485
6486         print "restore vma archive: $dbg_cmdstring\n";
6487         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6488     };
6489     my $err = $@;
6490
6491     alarm($oldtimeout) if $oldtimeout;
6492
6493     my $vollist = [];
6494     foreach my $devname (keys %$devinfo) {
6495         my $volid = $devinfo->{$devname}->{volid};
6496         push @$vollist, $volid if $volid;
6497     }
6498
6499     PVE::Storage::deactivate_volumes($cfg, $vollist);
6500
6501     unlink $mapfifo;
6502
6503     if ($err) {
6504         rmtree $tmpdir;
6505         unlink $tmpfn;
6506
6507         foreach my $devname (keys %$devinfo) {
6508             my $volid = $devinfo->{$devname}->{volid};
6509             next if !$volid;
6510             eval {
6511                 if ($volid =~ m|^/|) {
6512                     unlink $volid || die 'unlink failed\n';
6513                 } else {
6514                     PVE::Storage::vdisk_free($cfg, $volid);
6515                 }
6516                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6517             };
6518             print STDERR "unable to cleanup '$volid' - $@" if $@;
6519         }
6520         die $err;
6521     }
6522
6523     rmtree $tmpdir;
6524
6525     rename($tmpfn, $conffile) ||
6526         die "unable to commit configuration file '$conffile'\n";
6527
6528     PVE::Cluster::cfs_update(); # make sure we read new file
6529
6530     eval { rescan($vmid, 1); };
6531     warn $@ if $@;
6532 }
6533
6534 sub restore_tar_archive {
6535     my ($archive, $vmid, $user, $opts) = @_;
6536
6537     if ($archive ne '-') {
6538         my $firstfile = tar_archive_read_firstfile($archive);
6539         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6540             if $firstfile ne 'qemu-server.conf';
6541     }
6542
6543     my $storecfg = PVE::Storage::config();
6544
6545     # destroy existing data - keep empty config
6546     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6547     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6548
6549     my $tocmd = "/usr/lib/qemu-server/qmextract";
6550
6551     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6552     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6553     $tocmd .= ' --prealloc' if $opts->{prealloc};
6554     $tocmd .= ' --info' if $opts->{info};
6555
6556     # tar option "xf" does not autodetect compression when read from STDIN,
6557     # so we pipe to zcat
6558     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6559         PVE::Tools::shellquote("--to-command=$tocmd");
6560
6561     my $tmpdir = "/var/tmp/vzdumptmp$$";
6562     mkpath $tmpdir;
6563
6564     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6565     local $ENV{VZDUMP_VMID} = $vmid;
6566     local $ENV{VZDUMP_USER} = $user;
6567
6568     my $conffile = PVE::QemuConfig->config_file($vmid);
6569     my $tmpfn = "$conffile.$$.tmp";
6570
6571     # disable interrupts (always do cleanups)
6572     local $SIG{INT} =
6573         local $SIG{TERM} =
6574         local $SIG{QUIT} =
6575         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6576
6577     eval {
6578         # enable interrupts
6579         local $SIG{INT} =
6580             local $SIG{TERM} =
6581             local $SIG{QUIT} =
6582             local $SIG{HUP} =
6583             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6584
6585         if ($archive eq '-') {
6586             print "extracting archive from STDIN\n";
6587             run_command($cmd, input => "<&STDIN");
6588         } else {
6589             print "extracting archive '$archive'\n";
6590             run_command($cmd);
6591         }
6592
6593         return if $opts->{info};
6594
6595         # read new mapping
6596         my $map = {};
6597         my $statfile = "$tmpdir/qmrestore.stat";
6598         if (my $fd = IO::File->new($statfile, "r")) {
6599             while (defined (my $line = <$fd>)) {
6600                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6601                     $map->{$1} = $2 if $1;
6602                 } else {
6603                     print STDERR "unable to parse line in statfile - $line\n";
6604                 }
6605             }
6606             $fd->close();
6607         }
6608
6609         my $confsrc = "$tmpdir/qemu-server.conf";
6610
6611         my $srcfd = new IO::File($confsrc, "r") ||
6612             die "unable to open file '$confsrc'\n";
6613
6614         my $outfd = new IO::File ($tmpfn, "w") ||
6615             die "unable to write config for VM $vmid\n";
6616
6617         my $cookie = { netcount => 0 };
6618         while (defined (my $line = <$srcfd>)) {
6619             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6620         }
6621
6622         $srcfd->close();
6623         $outfd->close();
6624     };
6625     my $err = $@;
6626
6627     if ($err) {
6628
6629         unlink $tmpfn;
6630
6631         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6632
6633         die $err;
6634     }
6635
6636     rmtree $tmpdir;
6637
6638     rename $tmpfn, $conffile ||
6639         die "unable to commit configuration file '$conffile'\n";
6640
6641     PVE::Cluster::cfs_update(); # make sure we read new file
6642
6643     eval { rescan($vmid, 1); };
6644     warn $@ if $@;
6645 };
6646
6647 sub foreach_storage_used_by_vm {
6648     my ($conf, $func) = @_;
6649
6650     my $sidhash = {};
6651
6652     foreach_drive($conf, sub {
6653         my ($ds, $drive) = @_;
6654         return if drive_is_cdrom($drive);
6655
6656         my $volid = $drive->{file};
6657
6658         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6659         $sidhash->{$sid} = $sid if $sid;
6660     });
6661
6662     foreach my $sid (sort keys %$sidhash) {
6663         &$func($sid);
6664     }
6665 }
6666
6667 sub do_snapshots_with_qemu {
6668     my ($storecfg, $volid) = @_;
6669
6670     my $storage_name = PVE::Storage::parse_volume_id($volid);
6671     my $scfg = $storecfg->{ids}->{$storage_name};
6672
6673     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6674         return 1;
6675     }
6676
6677     if ($volid =~ m/\.(qcow2|qed)$/){
6678         return 1;
6679     }
6680
6681     return undef;
6682 }
6683
6684 sub qga_check_running {
6685     my ($vmid, $nowarn) = @_;
6686
6687     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6688     if ($@) {
6689         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6690         return 0;
6691     }
6692     return 1;
6693 }
6694
6695 sub template_create {
6696     my ($vmid, $conf, $disk) = @_;
6697
6698     my $storecfg = PVE::Storage::config();
6699
6700     foreach_drive($conf, sub {
6701         my ($ds, $drive) = @_;
6702
6703         return if drive_is_cdrom($drive);
6704         return if $disk && $ds ne $disk;
6705
6706         my $volid = $drive->{file};
6707         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6708
6709         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6710         $drive->{file} = $voliddst;
6711         $conf->{$ds} = print_drive($vmid, $drive);
6712         PVE::QemuConfig->write_config($vmid, $conf);
6713     });
6714 }
6715
6716 sub convert_iscsi_path {
6717     my ($path) = @_;
6718
6719     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6720         my $portal = $1;
6721         my $target = $2;
6722         my $lun = $3;
6723
6724         my $initiator_name = get_initiator_name();
6725
6726         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6727                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6728     }
6729
6730     die "cannot convert iscsi path '$path', unkown format\n";
6731 }
6732
6733 sub qemu_img_convert {
6734     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6735
6736     my $storecfg = PVE::Storage::config();
6737     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6738     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6739
6740     if ($src_storeid && $dst_storeid) {
6741
6742         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6743
6744         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6745         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6746
6747         my $src_format = qemu_img_format($src_scfg, $src_volname);
6748         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6749
6750         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6751         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6752
6753         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6754         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6755
6756         my $cmd = [];
6757         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6758         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6759         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6760         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6761
6762         if ($src_is_iscsi) {
6763             push @$cmd, '--image-opts';
6764             $src_path = convert_iscsi_path($src_path);
6765         } else {
6766             push @$cmd, '-f', $src_format;
6767         }
6768
6769         if ($dst_is_iscsi) {
6770             push @$cmd, '--target-image-opts';
6771             $dst_path = convert_iscsi_path($dst_path);
6772         } else {
6773             push @$cmd, '-O', $dst_format;
6774         }
6775
6776         push @$cmd, $src_path;
6777
6778         if (!$dst_is_iscsi && $is_zero_initialized) {
6779             push @$cmd, "zeroinit:$dst_path";
6780         } else {
6781             push @$cmd, $dst_path;
6782         }
6783
6784         my $parser = sub {
6785             my $line = shift;
6786             if($line =~ m/\((\S+)\/100\%\)/){
6787                 my $percent = $1;
6788                 my $transferred = int($size * $percent / 100);
6789                 my $remaining = $size - $transferred;
6790
6791                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6792             }
6793
6794         };
6795
6796         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6797         my $err = $@;
6798         die "copy failed: $err" if $err;
6799     }
6800 }
6801
6802 sub qemu_img_format {
6803     my ($scfg, $volname) = @_;
6804
6805     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6806         return $1;
6807     } else {
6808         return "raw";
6809     }
6810 }
6811
6812 sub qemu_drive_mirror {
6813     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6814
6815     $jobs = {} if !$jobs;
6816
6817     my $qemu_target;
6818     my $format;
6819     $jobs->{"drive-$drive"} = {};
6820
6821     if ($dst_volid =~ /^nbd:/) {
6822         $qemu_target = $dst_volid;
6823         $format = "nbd";
6824     } else {
6825         my $storecfg = PVE::Storage::config();
6826         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6827
6828         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6829
6830         $format = qemu_img_format($dst_scfg, $dst_volname);
6831
6832         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6833
6834         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6835     }
6836
6837     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6838     $opts->{format} = $format if $format;
6839
6840     if (defined($bwlimit)) {
6841         $opts->{speed} = $bwlimit * 1024;
6842         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6843     } else {
6844         print "drive mirror is starting for drive-$drive\n";
6845     }
6846
6847     # if a job already runs for this device we get an error, catch it for cleanup
6848     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6849     if (my $err = $@) {
6850         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6851         warn "$@\n" if $@;
6852         die "mirroring error: $err\n";
6853     }
6854
6855     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6856 }
6857
6858 sub qemu_drive_mirror_monitor {
6859     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6860
6861     eval {
6862         my $err_complete = 0;
6863
6864         while (1) {
6865             die "storage migration timed out\n" if $err_complete > 300;
6866
6867             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6868
6869             my $running_mirror_jobs = {};
6870             foreach my $stat (@$stats) {
6871                 next if $stat->{type} ne 'mirror';
6872                 $running_mirror_jobs->{$stat->{device}} = $stat;
6873             }
6874
6875             my $readycounter = 0;
6876
6877             foreach my $job (keys %$jobs) {
6878
6879                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6880                     print "$job : finished\n";
6881                     delete $jobs->{$job};
6882                     next;
6883                 }
6884
6885                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6886
6887                 my $busy = $running_mirror_jobs->{$job}->{busy};
6888                 my $ready = $running_mirror_jobs->{$job}->{ready};
6889                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6890                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6891                     my $remaining = $total - $transferred;
6892                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6893
6894                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6895                 }
6896
6897                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6898             }
6899
6900             last if scalar(keys %$jobs) == 0;
6901
6902             if ($readycounter == scalar(keys %$jobs)) {
6903                 print "all mirroring jobs are ready \n";
6904                 last if $skipcomplete; #do the complete later
6905
6906                 if ($vmiddst && $vmiddst != $vmid) {
6907                     my $agent_running = $qga && qga_check_running($vmid);
6908                     if ($agent_running) {
6909                         print "freeze filesystem\n";
6910                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6911                     } else {
6912                         print "suspend vm\n";
6913                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6914                     }
6915
6916                     # if we clone a disk for a new target vm, we don't switch the disk
6917                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6918
6919                     if ($agent_running) {
6920                         print "unfreeze filesystem\n";
6921                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6922                     } else {
6923                         print "resume vm\n";
6924                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6925                     }
6926
6927                     last;
6928                 } else {
6929
6930                     foreach my $job (keys %$jobs) {
6931                         # try to switch the disk if source and destination are on the same guest
6932                         print "$job: Completing block job...\n";
6933
6934                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6935                         if ($@ =~ m/cannot be completed/) {
6936                             print "$job: Block job cannot be completed, try again.\n";
6937                             $err_complete++;
6938                         }else {
6939                             print "$job: Completed successfully.\n";
6940                             $jobs->{$job}->{complete} = 1;
6941                         }
6942                     }
6943                 }
6944             }
6945             sleep 1;
6946         }
6947     };
6948     my $err = $@;
6949
6950     if ($err) {
6951         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6952         die "mirroring error: $err";
6953     }
6954
6955 }
6956
6957 sub qemu_blockjobs_cancel {
6958     my ($vmid, $jobs) = @_;
6959
6960     foreach my $job (keys %$jobs) {
6961         print "$job: Cancelling block job\n";
6962         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6963         $jobs->{$job}->{cancel} = 1;
6964     }
6965
6966     while (1) {
6967         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6968
6969         my $running_jobs = {};
6970         foreach my $stat (@$stats) {
6971             $running_jobs->{$stat->{device}} = $stat;
6972         }
6973
6974         foreach my $job (keys %$jobs) {
6975
6976             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6977                 print "$job: Done.\n";
6978                 delete $jobs->{$job};
6979             }
6980         }
6981
6982         last if scalar(keys %$jobs) == 0;
6983
6984         sleep 1;
6985     }
6986 }
6987
6988 sub clone_disk {
6989     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6990         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6991
6992     my $newvolid;
6993
6994     if (!$full) {
6995         print "create linked clone of drive $drivename ($drive->{file})\n";
6996         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6997         push @$newvollist, $newvolid;
6998     } else {
6999
7000         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7001         $storeid = $storage if $storage;
7002
7003         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7004         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7005
7006         print "create full clone of drive $drivename ($drive->{file})\n";
7007         my $name = undef;
7008         if (drive_is_cloudinit($drive)) {
7009             $name = "vm-$newvmid-cloudinit";
7010             $snapname = undef;
7011             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7012             if ($dst_format ne 'raw') {
7013                 $name .= ".$dst_format";
7014             }
7015         }
7016         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7017         push @$newvollist, $newvolid;
7018
7019         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7020
7021         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7022         if (!$running || $snapname) {
7023             # TODO: handle bwlimits
7024             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7025         } else {
7026
7027             my $kvmver = get_running_qemu_version ($vmid);
7028             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7029                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7030                     if $drive->{iothread};
7031             }
7032
7033             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7034         }
7035     }
7036
7037     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7038
7039     my $disk = $drive;
7040     $disk->{format} = undef;
7041     $disk->{file} = $newvolid;
7042     $disk->{size} = $size;
7043
7044     return $disk;
7045 }
7046
7047 # this only works if VM is running
7048 sub get_current_qemu_machine {
7049     my ($vmid) = @_;
7050
7051     my $cmd = { execute => 'query-machines', arguments => {} };
7052     my $res = vm_qmp_command($vmid, $cmd);
7053
7054     my ($current, $default);
7055     foreach my $e (@$res) {
7056         $default = $e->{name} if $e->{'is-default'};
7057         $current = $e->{name} if $e->{'is-current'};
7058     }
7059
7060     # fallback to the default machine if current is not supported by qemu
7061     return $current || $default || 'pc';
7062 }
7063
7064 sub get_running_qemu_version {
7065     my ($vmid) = @_;
7066     my $cmd = { execute => 'query-version', arguments => {} };
7067     my $res = vm_qmp_command($vmid, $cmd);
7068     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7069 }
7070
7071 sub qemu_machine_feature_enabled {
7072     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7073
7074     my $current_major;
7075     my $current_minor;
7076
7077     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7078
7079         $current_major = $3;
7080         $current_minor = $4;
7081
7082     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7083
7084         $current_major = $1;
7085         $current_minor = $2;
7086     }
7087
7088     return 1 if $current_major > $version_major ||
7089                 ($current_major == $version_major &&
7090                  $current_minor >= $version_minor);
7091 }
7092
7093 sub qemu_machine_pxe {
7094     my ($vmid, $conf, $machine) = @_;
7095
7096     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7097
7098     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7099         $machine .= '.pxe';
7100     }
7101
7102     return $machine;
7103 }
7104
7105 sub qemu_use_old_bios_files {
7106     my ($machine_type) = @_;
7107
7108     return if !$machine_type;
7109
7110     my $use_old_bios_files = undef;
7111
7112     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7113         $machine_type = $1;
7114         $use_old_bios_files = 1;
7115     } else {
7116         my $kvmver = kvm_user_version();
7117         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7118         # load new efi bios files on migration. So this hack is required to allow
7119         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7120         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7121         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7122     }
7123
7124     return ($use_old_bios_files, $machine_type);
7125 }
7126
7127 sub create_efidisk($$$$$) {
7128     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7129
7130     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7131     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7132
7133     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7134     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7135     PVE::Storage::activate_volumes($storecfg, [$volid]);
7136
7137     my $path = PVE::Storage::path($storecfg, $volid);
7138     eval {
7139         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7140     };
7141     die "Copying EFI vars image failed: $@" if $@;
7142
7143     return ($volid, $vars_size);
7144 }
7145
7146 sub vm_iothreads_list {
7147     my ($vmid) = @_;
7148
7149     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7150
7151     my $iothreads = {};
7152     foreach my $iothread (@$res) {
7153         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7154     }
7155
7156     return $iothreads;
7157 }
7158
7159 sub scsihw_infos {
7160     my ($conf, $drive) = @_;
7161
7162     my $maxdev = 0;
7163
7164     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7165         $maxdev = 7;
7166     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7167         $maxdev = 1;
7168     } else {
7169         $maxdev = 256;
7170     }
7171
7172     my $controller = int($drive->{index} / $maxdev);
7173     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7174
7175     return ($maxdev, $controller, $controller_prefix);
7176 }
7177
7178 sub add_hyperv_enlightenments {
7179     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7180
7181     return if $winversion < 6;
7182     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7183
7184     if ($gpu_passthrough || defined($hv_vendor_id)) {
7185         $hv_vendor_id //= 'proxmox';
7186         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7187     }
7188
7189     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7190         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7191         push @$cpuFlags , 'hv_vapic';
7192         push @$cpuFlags , 'hv_time';
7193     } else {
7194         push @$cpuFlags , 'hv_spinlocks=0xffff';
7195     }
7196
7197     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7198         push @$cpuFlags , 'hv_reset';
7199         push @$cpuFlags , 'hv_vpindex';
7200         push @$cpuFlags , 'hv_runtime';
7201     }
7202
7203     if ($winversion >= 7) {
7204         push @$cpuFlags , 'hv_relaxed';
7205
7206         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7207             push @$cpuFlags , 'hv_synic';
7208             push @$cpuFlags , 'hv_stimer';
7209         }
7210
7211         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7212             push @$cpuFlags , 'hv_tlbflush';
7213             push @$cpuFlags , 'hv_ipi';
7214             # FIXME: AMD does not supports this currently, only add with special flag??
7215             #push @$cpuFlags , 'hv_evmcs';
7216         }
7217     }
7218 }
7219
7220 sub windows_version {
7221     my ($ostype) = @_;
7222
7223     return 0 if !$ostype;
7224
7225     my $winversion = 0;
7226
7227     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7228         $winversion = 5;
7229     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7230         $winversion = 6;
7231     } elsif ($ostype =~ m/^win(\d+)$/) {
7232         $winversion = $1;
7233     }
7234
7235     return $winversion;
7236 }
7237
7238 sub resolve_dst_disk_format {
7239         my ($storecfg, $storeid, $src_volname, $format) = @_;
7240         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7241
7242         if (!$format) {
7243             # if no target format is specified, use the source disk format as hint
7244             if ($src_volname) {
7245                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7246                 $format = qemu_img_format($scfg, $src_volname);
7247             } else {
7248                 return $defFormat;
7249             }
7250         }
7251
7252         # test if requested format is supported - else use default
7253         my $supported = grep { $_ eq $format } @$validFormats;
7254         $format = $defFormat if !$supported;
7255         return $format;
7256 }
7257
7258 sub resolve_first_disk {
7259     my $conf = shift;
7260     my @disks = PVE::QemuServer::valid_drive_names();
7261     my $firstdisk;
7262     foreach my $ds (reverse @disks) {
7263         next if !$conf->{$ds};
7264         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7265         next if PVE::QemuServer::drive_is_cdrom($disk);
7266         $firstdisk = $ds;
7267     }
7268     return $firstdisk;
7269 }
7270
7271 sub generate_uuid {
7272     my ($uuid, $uuid_str);
7273     UUID::generate($uuid);
7274     UUID::unparse($uuid, $uuid_str);
7275     return $uuid_str;
7276 }
7277
7278 sub generate_smbios1_uuid {
7279     return "uuid=".generate_uuid();
7280 }
7281
7282 sub nbd_stop {
7283     my ($vmid) = @_;
7284
7285     vm_mon_cmd($vmid, 'nbd-server-stop');
7286 }
7287
7288 # bash completion helper
7289
7290 sub complete_backup_archives {
7291     my ($cmdname, $pname, $cvalue) = @_;
7292
7293     my $cfg = PVE::Storage::config();
7294
7295     my $storeid;
7296
7297     if ($cvalue =~ m/^([^:]+):/) {
7298         $storeid = $1;
7299     }
7300
7301     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7302
7303     my $res = [];
7304     foreach my $id (keys %$data) {
7305         foreach my $item (@{$data->{$id}}) {
7306             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7307             push @$res, $item->{volid} if defined($item->{volid});
7308         }
7309     }
7310
7311     return $res;
7312 }
7313
7314 my $complete_vmid_full = sub {
7315     my ($running) = @_;
7316
7317     my $idlist = vmstatus();
7318
7319     my $res = [];
7320
7321     foreach my $id (keys %$idlist) {
7322         my $d = $idlist->{$id};
7323         if (defined($running)) {
7324             next if $d->{template};
7325             next if $running && $d->{status} ne 'running';
7326             next if !$running && $d->{status} eq 'running';
7327         }
7328         push @$res, $id;
7329
7330     }
7331     return $res;
7332 };
7333
7334 sub complete_vmid {
7335     return &$complete_vmid_full();
7336 }
7337
7338 sub complete_vmid_stopped {
7339     return &$complete_vmid_full(0);
7340 }
7341
7342 sub complete_vmid_running {
7343     return &$complete_vmid_full(1);
7344 }
7345
7346 sub complete_storage {
7347
7348     my $cfg = PVE::Storage::config();
7349     my $ids = $cfg->{ids};
7350
7351     my $res = [];
7352     foreach my $sid (keys %$ids) {
7353         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7354         next if !$ids->{$sid}->{content}->{images};
7355         push @$res, $sid;
7356     }
7357
7358     return $res;
7359 }
7360
7361 1;