PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_IDE_DISKS = 4;
 735 my $MAX_SCSI_DISKS = 14;
 736 my $MAX_VIRTIO_DISKS = 16;
 737 my $MAX_SATA_DISKS = 6;
 738 my $MAX_USB_DEVICES = 5;
 739 my $MAX_NETS = 32;
 740 my $MAX_UNUSED_DISKS = 256;
 741 my $MAX_HOSTPCI_DEVICES = 4;
 742 my $MAX_SERIAL_PORTS = 4;
 743 my $MAX_PARALLEL_PORTS = 3;
 744 my $MAX_NUMA = 8;
 745
 746 my $numa_fmt = {
 747     cpus => {
 748         type => "string",
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => "CPUs accessing this NUMA node.",
 751         format_description => "id[-id];...",
 752     },
 753     memory => {
 754         type => "number",
 755         description => "Amount of memory this NUMA node provides.",
 756         optional => 1,
 757     },
 758     hostnodes => {
 759         type => "string",
 760         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 761         description => "Host NUMA nodes to use.",
 762         format_description => "id[-id];...",
 763         optional => 1,
 764     },
 765     policy => {
 766         type => 'string',
 767         enum => [qw(preferred bind interleave)],
 768         description => "NUMA allocation policy.",
 769         optional => 1,
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 773 my $numadesc = {
 774     optional => 1,
 775     type => 'string', format => $numa_fmt,
 776     description => "NUMA topology.",
 777 };
 778 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 779
 780 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 781     $confdesc->{"numa$i"} = $numadesc;
 782 }
 783
 784 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 785                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 786                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 787 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 788
 789 my $net_fmt_bridge_descr = <<__EOD__;
 790 Bridge to attach the network device to. The Proxmox VE standard bridge
 791 is called 'vmbr0'.
 792
 793 If you do not specify a bridge, we create a kvm user (NATed) network
 794 device, which provides DHCP and DNS services. The following addresses
 795 are used:
 796
 797  10.0.2.2   Gateway
 798  10.0.2.3   DNS Server
 799  10.0.2.4   SMB Server
 800
 801 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 802 __EOD__
 803
 804 my $net_fmt = {
 805     macaddr  => get_standard_option('mac-addr', {
 806         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 807     }),
 808     model => {
 809         type => 'string',
 810         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 811         enum => $nic_model_list,
 812         default_key => 1,
 813     },
 814     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 815     bridge => {
 816         type => 'string',
 817         description => $net_fmt_bridge_descr,
 818         format_description => 'bridge',
 819         optional => 1,
 820     },
 821     queues => {
 822         type => 'integer',
 823         minimum => 0, maximum => 16,
 824         description => 'Number of packet queues to be used on the device.',
 825         optional => 1,
 826     },
 827     rate => {
 828         type => 'number',
 829         minimum => 0,
 830         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 831         optional => 1,
 832     },
 833     tag => {
 834         type => 'integer',
 835         minimum => 1, maximum => 4094,
 836         description => 'VLAN tag to apply to packets on this interface.',
 837         optional => 1,
 838     },
 839     trunks => {
 840         type => 'string',
 841         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 842         description => 'VLAN trunks to pass through this interface.',
 843         format_description => 'vlanid[;vlanid...]',
 844         optional => 1,
 845     },
 846     firewall => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be protected by the firewall.',
 849         optional => 1,
 850     },
 851     link_down => {
 852         type => 'boolean',
 853         description => 'Whether this interface should be disconnected (like pulling the plug).',
 854         optional => 1,
 855     },
 856 };
 857
 858 my $netdesc = {
 859     optional => 1,
 860     type => 'string', format => $net_fmt,
 861     description => "Specify network devices.",
 862 };
 863
 864 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 865
 866 my $ipconfig_fmt = {
 867     ip => {
 868         type => 'string',
 869         format => 'pve-ipv4-config',
 870         format_description => 'IPv4Format/CIDR',
 871         description => 'IPv4 address in CIDR format.',
 872         optional => 1,
 873         default => 'dhcp',
 874     },
 875     gw => {
 876         type => 'string',
 877         format => 'ipv4',
 878         format_description => 'GatewayIPv4',
 879         description => 'Default gateway for IPv4 traffic.',
 880         optional => 1,
 881         requires => 'ip',
 882     },
 883     ip6 => {
 884         type => 'string',
 885         format => 'pve-ipv6-config',
 886         format_description => 'IPv6Format/CIDR',
 887         description => 'IPv6 address in CIDR format.',
 888         optional => 1,
 889         default => 'dhcp',
 890     },
 891     gw6 => {
 892         type => 'string',
 893         format => 'ipv6',
 894         format_description => 'GatewayIPv6',
 895         description => 'Default gateway for IPv6 traffic.',
 896         optional => 1,
 897         requires => 'ip6',
 898     },
 899 };
 900 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 901 my $ipconfigdesc = {
 902     optional => 1,
 903     type => 'string', format => 'pve-qm-ipconfig',
 904     description => <<'EODESCR',
 905 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 906
 907 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 908
 909 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 910 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 911
 912 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 913 EODESCR
 914 };
 915 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 916
 917 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 918     $confdesc->{"net$i"} = $netdesc;
 919     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 920 }
 921
 922 foreach my $key (keys %$confdesc_cloudinit) {
 923     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 924 }
 925
 926 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 927 sub verify_volume_id_or_qm_path {
 928     my ($volid, $noerr) = @_;
 929
 930     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 931         return $volid;
 932     }
 933
 934     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 935     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 936     if ($@) {
 937         return undef if $noerr;
 938         die $@;
 939     }
 940     return $volid;
 941 }
 942
 943 my $drivename_hash;
 944
 945 my %drivedesc_base = (
 946     volume => { alias => 'file' },
 947     file => {
 948         type => 'string',
 949         format => 'pve-volume-id-or-qm-path',
 950         default_key => 1,
 951         format_description => 'volume',
 952         description => "The drive's backing volume.",
 953     },
 954     media => {
 955         type => 'string',
 956         enum => [qw(cdrom disk)],
 957         description => "The drive's media type.",
 958         default => 'disk',
 959         optional => 1
 960     },
 961     cyls => {
 962         type => 'integer',
 963         description => "Force the drive's physical geometry to have a specific cylinder count.",
 964         optional => 1
 965     },
 966     heads => {
 967         type => 'integer',
 968         description => "Force the drive's physical geometry to have a specific head count.",
 969         optional => 1
 970     },
 971     secs => {
 972         type => 'integer',
 973         description => "Force the drive's physical geometry to have a specific sector count.",
 974         optional => 1
 975     },
 976     trans => {
 977         type => 'string',
 978         enum => [qw(none lba auto)],
 979         description => "Force disk geometry bios translation mode.",
 980         optional => 1,
 981     },
 982     snapshot => {
 983         type => 'boolean',
 984         description => "Controls qemu's snapshot mode feature."
 985             . " If activated, changes made to the disk are temporary and will"
 986             . " be discarded when the VM is shutdown.",
 987         optional => 1,
 988     },
 989     cache => {
 990         type => 'string',
 991         enum => [qw(none writethrough writeback unsafe directsync)],
 992         description => "The drive's cache mode",
 993         optional => 1,
 994     },
 995     format => get_standard_option('pve-qm-image-format'),
 996     size => {
 997         type => 'string',
 998         format => 'disk-size',
 999         format_description => 'DiskSize',
1000         description => "Disk size. This is purely informational and has no effect.",
1001         optional => 1,
1002     },
1003     backup => {
1004         type => 'boolean',
1005         description => "Whether the drive should be included when making backups.",
1006         optional => 1,
1007     },
1008     replicate => {
1009         type => 'boolean',
1010         description => 'Whether the drive should considered for replication jobs.',
1011         optional => 1,
1012         default => 1,
1013     },
1014     rerror => {
1015         type => 'string',
1016         enum => [qw(ignore report stop)],
1017         description => 'Read error action.',
1018         optional => 1,
1019     },
1020     werror => {
1021         type => 'string',
1022         enum => [qw(enospc ignore report stop)],
1023         description => 'Write error action.',
1024         optional => 1,
1025     },
1026     aio => {
1027         type => 'string',
1028         enum => [qw(native threads)],
1029         description => 'AIO type to use.',
1030         optional => 1,
1031     },
1032     discard => {
1033         type => 'string',
1034         enum => [qw(ignore on)],
1035         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1036         optional => 1,
1037     },
1038     detect_zeroes => {
1039         type => 'boolean',
1040         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1041         optional => 1,
1042     },
1043     serial => {
1044         type => 'string',
1045         format => 'urlencoded',
1046         format_description => 'serial',
1047         maxLength => 20*3, # *3 since it's %xx url enoded
1048         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1049         optional => 1,
1050     },
1051     shared => {
1052         type => 'boolean',
1053         description => 'Mark this locally-managed volume as available on all nodes',
1054         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1055         optional => 1,
1056         default => 0,
1057     }
1058 );
1059
1060 my %iothread_fmt = ( iothread => {
1061         type => 'boolean',
1062         description => "Whether to use iothreads for this drive",
1063         optional => 1,
1064 });
1065
1066 my %model_fmt = (
1067     model => {
1068         type => 'string',
1069         format => 'urlencoded',
1070         format_description => 'model',
1071         maxLength => 40*3, # *3 since it's %xx url enoded
1072         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1073         optional => 1,
1074     },
1075 );
1076
1077 my %queues_fmt = (
1078     queues => {
1079         type => 'integer',
1080         description => "Number of queues.",
1081         minimum => 2,
1082         optional => 1
1083     }
1084 );
1085
1086 my %scsiblock_fmt = (
1087     scsiblock => {
1088         type => 'boolean',
1089         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1090         optional => 1,
1091         default => 0,
1092     },
1093 );
1094
1095 my %ssd_fmt = (
1096     ssd => {
1097         type => 'boolean',
1098         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1099         optional => 1,
1100     },
1101 );
1102
1103 my %wwn_fmt = (
1104     wwn => {
1105         type => 'string',
1106         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1107         format_description => 'wwn',
1108         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1109         optional => 1,
1110     },
1111 );
1112
1113 my $add_throttle_desc = sub {
1114     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1115     my $d = {
1116         type => $type,
1117         format_description => $unit,
1118         description => "Maximum $what in $longunit.",
1119         optional => 1,
1120     };
1121     $d->{minimum} = $minimum if defined($minimum);
1122     $drivedesc_base{$key} = $d;
1123 };
1124 # throughput: (leaky bucket)
1125 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1126 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1127 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1128 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1129 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1130 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1131 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1132 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1133 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1134
1135 # pools: (pool of IO before throttling starts taking effect)
1136 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1137 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1138 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1139 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1140 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1141 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1142
1143 # burst lengths
1144 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1145 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1146 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1147 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1148 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1150
1151 # legacy support
1152 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1153 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1154 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1155 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1156
1157 my $ide_fmt = {
1158     %drivedesc_base,
1159     %model_fmt,
1160     %ssd_fmt,
1161     %wwn_fmt,
1162 };
1163 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1164
1165 my $idedesc = {
1166     optional => 1,
1167     type => 'string', format => $ide_fmt,
1168     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1171
1172 my $scsi_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175     %queues_fmt,
1176     %scsiblock_fmt,
1177     %ssd_fmt,
1178     %wwn_fmt,
1179 };
1180 my $scsidesc = {
1181     optional => 1,
1182     type => 'string', format => $scsi_fmt,
1183     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1184 };
1185 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1186
1187 my $sata_fmt = {
1188     %drivedesc_base,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192 my $satadesc = {
1193     optional => 1,
1194     type => 'string', format => $sata_fmt,
1195     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1196 };
1197 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1198
1199 my $virtio_fmt = {
1200     %drivedesc_base,
1201     %iothread_fmt,
1202 };
1203 my $virtiodesc = {
1204     optional => 1,
1205     type => 'string', format => $virtio_fmt,
1206     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1207 };
1208 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1209
1210 my $alldrive_fmt = {
1211     %drivedesc_base,
1212     %iothread_fmt,
1213     %model_fmt,
1214     %queues_fmt,
1215     %scsiblock_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219
1220 my $efidisk_fmt = {
1221     volume => { alias => 'file' },
1222     file => {
1223         type => 'string',
1224         format => 'pve-volume-id-or-qm-path',
1225         default_key => 1,
1226         format_description => 'volume',
1227         description => "The drive's backing volume.",
1228     },
1229     format => get_standard_option('pve-qm-image-format'),
1230     size => {
1231         type => 'string',
1232         format => 'disk-size',
1233         format_description => 'DiskSize',
1234         description => "Disk size. This is purely informational and has no effect.",
1235         optional => 1,
1236     },
1237 };
1238
1239 my $efidisk_desc = {
1240     optional => 1,
1241     type => 'string', format => $efidisk_fmt,
1242     description => "Configure a Disk for storing EFI vars",
1243 };
1244
1245 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1246
1247 my $usb_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string', format => 'pve-qm-usb-device',
1251         format_description => 'HOSTUSBDEVICE|spice',
1252         description => <<EODESCR,
1253 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1254
1255  'bus-port(.port)*' (decimal numbers) or
1256  'vendor_id:product_id' (hexadeciaml numbers) or
1257  'spice'
1258
1259 You can use the 'lsusb -t' command to list existing usb devices.
1260
1261 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1262
1263 The value 'spice' can be used to add a usb redirection devices for spice.
1264 EODESCR
1265     },
1266     usb3 => {
1267         optional => 1,
1268         type => 'boolean',
1269         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1270         default => 0,
1271     },
1272 };
1273
1274 my $usbdesc = {
1275     optional => 1,
1276     type => 'string', format => $usb_fmt,
1277     description => "Configure an USB device (n is 0 to 4).",
1278 };
1279 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1280
1281 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1282 my $hostpci_fmt = {
1283     host => {
1284         default_key => 1,
1285         type => 'string',
1286         pattern => qr/$PCIRE(;$PCIRE)*/,
1287         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1288         description => <<EODESCR,
1289 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1290 of PCI virtual functions of the host. HOSTPCIID syntax is:
1291
1292 'bus:dev.func' (hexadecimal numbers)
1293
1294 You can us the 'lspci' command to list existing PCI devices.
1295 EODESCR
1296     },
1297     rombar => {
1298         type => 'boolean',
1299         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1300         optional => 1,
1301         default => 1,
1302     },
1303     romfile => {
1304         type => 'string',
1305         pattern => '[^,;]+',
1306         format_description => 'string',
1307         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1308         optional => 1,
1309     },
1310     pcie => {
1311         type => 'boolean',
1312         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1313         optional => 1,
1314         default => 0,
1315     },
1316     'x-vga' => {
1317         type => 'boolean',
1318         description =>  "Enable vfio-vga device support.",
1319         optional => 1,
1320         default => 0,
1321     },
1322     'mdev' => {
1323         type => 'string',
1324         format_description => 'string',
1325         pattern => '[^/\.:]+',
1326         optional => 1,
1327         description => <<EODESCR
1328 The type of mediated device to use.
1329 An instance of this type will be created on startup of the VM and
1330 will be cleaned up when the VM stops.
1331 EODESCR
1332     }
1333 };
1334 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1335
1336 my $hostpcidesc = {
1337         optional => 1,
1338         type => 'string', format => 'pve-qm-hostpci',
1339         description => "Map host PCI devices into guest.",
1340         verbose_description =>  <<EODESCR,
1341 Map host PCI devices into guest.
1342
1343 NOTE: This option allows direct access to host hardware. So it is no longer
1344 possible to migrate such machines - use with special care.
1345
1346 CAUTION: Experimental! User reported problems with this option.
1347 EODESCR
1348 };
1349 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1350
1351 my $serialdesc = {
1352         optional => 1,
1353         type => 'string',
1354         pattern => '(/dev/.+|socket)',
1355         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1356         verbose_description =>  <<EODESCR,
1357 Create a serial device inside the VM (n is 0 to 3), and pass through a
1358 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1359 host side (use 'qm terminal' to open a terminal connection).
1360
1361 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1362
1363 CAUTION: Experimental! User reported problems with this option.
1364 EODESCR
1365 };
1366
1367 my $paralleldesc= {
1368         optional => 1,
1369         type => 'string',
1370         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1371         description =>  "Map host parallel devices (n is 0 to 2).",
1372         verbose_description =>  <<EODESCR,
1373 Map host parallel devices (n is 0 to 2).
1374
1375 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1376
1377 CAUTION: Experimental! User reported problems with this option.
1378 EODESCR
1379 };
1380
1381 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1382     $confdesc->{"parallel$i"} = $paralleldesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1386     $confdesc->{"serial$i"} = $serialdesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1390     $confdesc->{"hostpci$i"} = $hostpcidesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1394     $drivename_hash->{"ide$i"} = 1;
1395     $confdesc->{"ide$i"} = $idedesc;
1396 }
1397
1398 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1399     $drivename_hash->{"sata$i"} = 1;
1400     $confdesc->{"sata$i"} = $satadesc;
1401 }
1402
1403 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1404     $drivename_hash->{"scsi$i"} = 1;
1405     $confdesc->{"scsi$i"} = $scsidesc ;
1406 }
1407
1408 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1409     $drivename_hash->{"virtio$i"} = 1;
1410     $confdesc->{"virtio$i"} = $virtiodesc;
1411 }
1412
1413 $drivename_hash->{efidisk0} = 1;
1414 $confdesc->{efidisk0} = $efidisk_desc;
1415
1416 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1417     $confdesc->{"usb$i"} = $usbdesc;
1418 }
1419
1420 my $unuseddesc = {
1421     optional => 1,
1422     type => 'string', format => 'pve-volume-id',
1423     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1424 };
1425
1426 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1427     $confdesc->{"unused$i"} = $unuseddesc;
1428 }
1429
1430 my $kvm_api_version = 0;
1431
1432 sub kvm_version {
1433     return $kvm_api_version if $kvm_api_version;
1434
1435     open my $fh, '<', '/dev/kvm'
1436         or return undef;
1437
1438     # 0xae00 => KVM_GET_API_VERSION
1439     $kvm_api_version = ioctl($fh, 0xae00, 0);
1440
1441     return $kvm_api_version;
1442 }
1443
1444 my $kvm_user_version;
1445
1446 sub kvm_user_version {
1447
1448     return $kvm_user_version if $kvm_user_version;
1449
1450     $kvm_user_version = 'unknown';
1451
1452     my $code = sub {
1453         my $line = shift;
1454         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1455             $kvm_user_version = $2;
1456         }
1457     };
1458
1459     eval { run_command("kvm -version", outfunc => $code); };
1460     warn $@ if $@;
1461
1462     return $kvm_user_version;
1463
1464 }
1465
1466 sub kernel_has_vhost_net {
1467     return -c '/dev/vhost-net';
1468 }
1469
1470 sub valid_drive_names {
1471     # order is important - used to autoselect boot disk
1472     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1473             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1474             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1475             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1476             'efidisk0');
1477 }
1478
1479 sub is_valid_drivename {
1480     my $dev = shift;
1481
1482     return defined($drivename_hash->{$dev});
1483 }
1484
1485 sub option_exists {
1486     my $key = shift;
1487     return defined($confdesc->{$key});
1488 }
1489
1490 sub nic_models {
1491     return $nic_model_list;
1492 }
1493
1494 sub os_list_description {
1495
1496     return {
1497         other => 'Other',
1498         wxp => 'Windows XP',
1499         w2k => 'Windows 2000',
1500         w2k3 =>, 'Windows 2003',
1501         w2k8 => 'Windows 2008',
1502         wvista => 'Windows Vista',
1503         win7 => 'Windows 7',
1504         win8 => 'Windows 8/2012',
1505         win10 => 'Windows 10/2016',
1506         l24 => 'Linux 2.4',
1507         l26 => 'Linux 2.6',
1508     };
1509 }
1510
1511 my $cdrom_path;
1512
1513 sub get_cdrom_path {
1514
1515     return  $cdrom_path if $cdrom_path;
1516
1517     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1518     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1519     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1520 }
1521
1522 sub get_iso_path {
1523     my ($storecfg, $vmid, $cdrom) = @_;
1524
1525     if ($cdrom eq 'cdrom') {
1526         return get_cdrom_path();
1527     } elsif ($cdrom eq 'none') {
1528         return '';
1529     } elsif ($cdrom =~ m|^/|) {
1530         return $cdrom;
1531     } else {
1532         return PVE::Storage::path($storecfg, $cdrom);
1533     }
1534 }
1535
1536 # try to convert old style file names to volume IDs
1537 sub filename_to_volume_id {
1538     my ($vmid, $file, $media) = @_;
1539
1540      if (!($file eq 'none' || $file eq 'cdrom' ||
1541           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1542
1543         return undef if $file =~ m|/|;
1544
1545         if ($media && $media eq 'cdrom') {
1546             $file = "local:iso/$file";
1547         } else {
1548             $file = "local:$vmid/$file";
1549         }
1550     }
1551
1552     return $file;
1553 }
1554
1555 sub verify_media_type {
1556     my ($opt, $vtype, $media) = @_;
1557
1558     return if !$media;
1559
1560     my $etype;
1561     if ($media eq 'disk') {
1562         $etype = 'images';
1563     } elsif ($media eq 'cdrom') {
1564         $etype = 'iso';
1565     } else {
1566         die "internal error";
1567     }
1568
1569     return if ($vtype eq $etype);
1570
1571     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1572 }
1573
1574 sub cleanup_drive_path {
1575     my ($opt, $storecfg, $drive) = @_;
1576
1577     # try to convert filesystem paths to volume IDs
1578
1579     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1580         ($drive->{file} !~ m|^/dev/.+|) &&
1581         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1582         ($drive->{file} !~ m/^\d+$/)) {
1583         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1584         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1585         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1586         verify_media_type($opt, $vtype, $drive->{media});
1587         $drive->{file} = $volid;
1588     }
1589
1590     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1591 }
1592
1593 sub parse_hotplug_features {
1594     my ($data) = @_;
1595
1596     my $res = {};
1597
1598     return $res if $data eq '0';
1599
1600     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1601
1602     foreach my $feature (PVE::Tools::split_list($data)) {
1603         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1604             $res->{$1} = 1;
1605         } else {
1606             die "invalid hotplug feature '$feature'\n";
1607         }
1608     }
1609     return $res;
1610 }
1611
1612 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1613 sub pve_verify_hotplug_features {
1614     my ($value, $noerr) = @_;
1615
1616     return $value if parse_hotplug_features($value);
1617
1618     return undef if $noerr;
1619
1620     die "unable to parse hotplug option\n";
1621 }
1622
1623 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1624 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1625 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1626 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1627 #        [,iothread=on][,serial=serial][,model=model]
1628
1629 sub parse_drive {
1630     my ($key, $data) = @_;
1631
1632     my ($interface, $index);
1633
1634     if ($key =~ m/^([^\d]+)(\d+)$/) {
1635         $interface = $1;
1636         $index = $2;
1637     } else {
1638         return undef;
1639     }
1640
1641     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1642                                      : $confdesc->{$key}->{format};
1643     if (!$desc) {
1644         warn "invalid drive key: $key\n";
1645         return undef;
1646     }
1647     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1648     return undef if !$res;
1649     $res->{interface} = $interface;
1650     $res->{index} = $index;
1651
1652     my $error = 0;
1653     foreach my $opt (qw(bps bps_rd bps_wr)) {
1654         if (my $bps = defined(delete $res->{$opt})) {
1655             if (defined($res->{"m$opt"})) {
1656                 warn "both $opt and m$opt specified\n";
1657                 ++$error;
1658                 next;
1659             }
1660             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1661         }
1662     }
1663
1664     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1665     for my $requirement (
1666         [mbps_max => 'mbps'],
1667         [mbps_rd_max => 'mbps_rd'],
1668         [mbps_wr_max => 'mbps_wr'],
1669         [miops_max => 'miops'],
1670         [miops_rd_max => 'miops_rd'],
1671         [miops_wr_max => 'miops_wr'],
1672         [bps_max_length => 'mbps_max'],
1673         [bps_rd_max_length => 'mbps_rd_max'],
1674         [bps_wr_max_length => 'mbps_wr_max'],
1675         [iops_max_length => 'iops_max'],
1676         [iops_rd_max_length => 'iops_rd_max'],
1677         [iops_wr_max_length => 'iops_wr_max']) {
1678         my ($option, $requires) = @$requirement;
1679         if ($res->{$option} && !$res->{$requires}) {
1680             warn "$option requires $requires\n";
1681             ++$error;
1682         }
1683     }
1684
1685     return undef if $error;
1686
1687     return undef if $res->{mbps_rd} && $res->{mbps};
1688     return undef if $res->{mbps_wr} && $res->{mbps};
1689     return undef if $res->{iops_rd} && $res->{iops};
1690     return undef if $res->{iops_wr} && $res->{iops};
1691
1692     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1693         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1694         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1695         return undef if $res->{interface} eq 'virtio';
1696     }
1697
1698     if (my $size = $res->{size}) {
1699         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1700     }
1701
1702     return $res;
1703 }
1704
1705 sub print_drive {
1706     my ($vmid, $drive) = @_;
1707     my $data = { %$drive };
1708     delete $data->{$_} for qw(index interface);
1709     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1710 }
1711
1712 sub scsi_inquiry {
1713     my($fh, $noerr) = @_;
1714
1715     my $SG_IO = 0x2285;
1716     my $SG_GET_VERSION_NUM = 0x2282;
1717
1718     my $versionbuf = "\x00" x 8;
1719     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1720     if (!$ret) {
1721         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1722         return undef;
1723     }
1724     my $version = unpack("I", $versionbuf);
1725     if ($version < 30000) {
1726         die "scsi generic interface too old\n"  if !$noerr;
1727         return undef;
1728     }
1729
1730     my $buf = "\x00" x 36;
1731     my $sensebuf = "\x00" x 8;
1732     my $cmd = pack("C x3 C x1", 0x12, 36);
1733
1734     # see /usr/include/scsi/sg.h
1735     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1736
1737     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1738                       length($sensebuf), 0, length($buf), $buf,
1739                       $cmd, $sensebuf, 6000);
1740
1741     $ret = ioctl($fh, $SG_IO, $packet);
1742     if (!$ret) {
1743         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1744         return undef;
1745     }
1746
1747     my @res = unpack($sg_io_hdr_t, $packet);
1748     if ($res[17] || $res[18]) {
1749         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1750         return undef;
1751     }
1752
1753     my $res = {};
1754     (my $byte0, my $byte1, $res->{vendor},
1755      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1756
1757     $res->{removable} = $byte1 & 128 ? 1 : 0;
1758     $res->{type} = $byte0 & 31;
1759
1760     return $res;
1761 }
1762
1763 sub path_is_scsi {
1764     my ($path) = @_;
1765
1766     my $fh = IO::File->new("+<$path") || return undef;
1767     my $res = scsi_inquiry($fh, 1);
1768     close($fh);
1769
1770     return $res;
1771 }
1772
1773 sub machine_type_is_q35 {
1774     my ($conf) = @_;
1775
1776     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1777 }
1778
1779 sub print_tabletdevice_full {
1780     my ($conf, $arch) = @_;
1781
1782     my $q35 = machine_type_is_q35($conf);
1783
1784     # we use uhci for old VMs because tablet driver was buggy in older qemu
1785     my $usbbus;
1786     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1787         $usbbus = 'ehci';
1788     } else {
1789         $usbbus = 'uhci';
1790     }
1791
1792     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1793 }
1794
1795 sub print_keyboarddevice_full {
1796     my ($conf, $arch, $machine) = @_;
1797
1798     return undef if $arch ne 'aarch64';
1799
1800     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1801 }
1802
1803 sub print_drivedevice_full {
1804     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1805
1806     my $device = '';
1807     my $maxdev = 0;
1808
1809     if ($drive->{interface} eq 'virtio') {
1810         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1811         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1812         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1813     } elsif ($drive->{interface} eq 'scsi') {
1814
1815         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1816         my $unit = $drive->{index} % $maxdev;
1817         my $devicetype = 'hd';
1818         my $path = '';
1819         if (drive_is_cdrom($drive)) {
1820             $devicetype = 'cd';
1821         } else {
1822             if ($drive->{file} =~ m|^/|) {
1823                 $path = $drive->{file};
1824                 if (my $info = path_is_scsi($path)) {
1825                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1826                         $devicetype = 'block';
1827                     } elsif ($info->{type} == 1) { # tape
1828                         $devicetype = 'generic';
1829                     }
1830                 }
1831             } else {
1832                  $path = PVE::Storage::path($storecfg, $drive->{file});
1833             }
1834
1835             if($path =~ m/^iscsi\:\/\//){
1836                 $devicetype = 'generic';
1837             }
1838         }
1839
1840         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1841             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1842         } else {
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         }
1845
1846         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1847             $device .= ",rotation_rate=1";
1848         }
1849         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1850
1851     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1852         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1853         my $controller = int($drive->{index} / $maxdev);
1854         my $unit = $drive->{index} % $maxdev;
1855         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1856
1857         $device = "ide-$devicetype";
1858         if ($drive->{interface} eq 'ide') {
1859             $device .= ",bus=ide.$controller,unit=$unit";
1860         } else {
1861             $device .= ",bus=ahci$controller.$unit";
1862         }
1863         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1864
1865         if ($devicetype eq 'hd') {
1866             if (my $model = $drive->{model}) {
1867                 $model = URI::Escape::uri_unescape($model);
1868                 $device .= ",model=$model";
1869             }
1870             if ($drive->{ssd}) {
1871                 $device .= ",rotation_rate=1";
1872             }
1873         }
1874         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1875     } elsif ($drive->{interface} eq 'usb') {
1876         die "implement me";
1877         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1878     } else {
1879         die "unsupported interface type";
1880     }
1881
1882     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1883
1884     if (my $serial = $drive->{serial}) {
1885         $serial = URI::Escape::uri_unescape($serial);
1886         $device .= ",serial=$serial";
1887     }
1888
1889
1890     return $device;
1891 }
1892
1893 sub get_initiator_name {
1894     my $initiator;
1895
1896     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1897     while (defined(my $line = <$fh>)) {
1898         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1899         $initiator = $1;
1900         last;
1901     }
1902     $fh->close();
1903
1904     return $initiator;
1905 }
1906
1907 sub print_drive_full {
1908     my ($storecfg, $vmid, $drive) = @_;
1909
1910     my $path;
1911     my $volid = $drive->{file};
1912     my $format;
1913
1914     if (drive_is_cdrom($drive)) {
1915         $path = get_iso_path($storecfg, $vmid, $volid);
1916     } else {
1917         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1918         if ($storeid) {
1919             $path = PVE::Storage::path($storecfg, $volid);
1920             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1921             $format = qemu_img_format($scfg, $volname);
1922         } else {
1923             $path = $volid;
1924             $format = "raw";
1925         }
1926    }
1927
1928     my $opts = '';
1929     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1930     foreach my $o (@qemu_drive_options) {
1931         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1932     }
1933
1934     # snapshot only accepts on|off
1935     if (defined($drive->{snapshot})) {
1936         my $v = $drive->{snapshot} ? 'on' : 'off';
1937         $opts .= ",snapshot=$v";
1938     }
1939
1940     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1941         my ($dir, $qmpname) = @$type;
1942         if (my $v = $drive->{"mbps$dir"}) {
1943             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1944         }
1945         if (my $v = $drive->{"mbps${dir}_max"}) {
1946             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1947         }
1948         if (my $v = $drive->{"bps${dir}_max_length"}) {
1949             $opts .= ",throttling.bps$qmpname-max-length=$v";
1950         }
1951         if (my $v = $drive->{"iops${dir}"}) {
1952             $opts .= ",throttling.iops$qmpname=$v";
1953         }
1954         if (my $v = $drive->{"iops${dir}_max"}) {
1955             $opts .= ",throttling.iops$qmpname-max=$v";
1956         }
1957         if (my $v = $drive->{"iops${dir}_max_length"}) {
1958             $opts .= ",throttling.iops$qmpname-max-length=$v";
1959         }
1960     }
1961
1962     $opts .= ",format=$format" if $format && !$drive->{format};
1963
1964     my $cache_direct = 0;
1965
1966     if (my $cache = $drive->{cache}) {
1967         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         $opts .= ",cache=none";
1970         $cache_direct = 1;
1971     }
1972
1973     # aio native works only with O_DIRECT
1974     if (!$drive->{aio}) {
1975         if($cache_direct) {
1976             $opts .= ",aio=native";
1977         } else {
1978             $opts .= ",aio=threads";
1979         }
1980     }
1981
1982     if (!drive_is_cdrom($drive)) {
1983         my $detectzeroes;
1984         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1985             $detectzeroes = 'off';
1986         } elsif ($drive->{discard}) {
1987             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1988         } else {
1989             # This used to be our default with discard not being specified:
1990             $detectzeroes = 'on';
1991         }
1992         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1993     }
1994
1995     my $pathinfo = $path ? "file=$path," : '';
1996
1997     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1998 }
1999
2000 sub print_netdevice_full {
2001     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2002
2003     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2004
2005     my $device = $net->{model};
2006     if ($net->{model} eq 'virtio') {
2007          $device = 'virtio-net-pci';
2008      };
2009
2010     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2011     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2012     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2013         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2014         my $vectors = $net->{queues} * 2 + 2;
2015         $tmpstr .= ",vectors=$vectors,mq=on";
2016     }
2017     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2018
2019     if ($use_old_bios_files) {
2020         my $romfile;
2021         if ($device eq 'virtio-net-pci') {
2022             $romfile = 'pxe-virtio.rom';
2023         } elsif ($device eq 'e1000') {
2024             $romfile = 'pxe-e1000.rom';
2025         } elsif ($device eq 'ne2k') {
2026             $romfile = 'pxe-ne2k_pci.rom';
2027         } elsif ($device eq 'pcnet') {
2028             $romfile = 'pxe-pcnet.rom';
2029         } elsif ($device eq 'rtl8139') {
2030             $romfile = 'pxe-rtl8139.rom';
2031         }
2032         $tmpstr .= ",romfile=$romfile" if $romfile;
2033     }
2034
2035     return $tmpstr;
2036 }
2037
2038 sub print_netdev_full {
2039     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2040
2041     my $i = '';
2042     if ($netid =~ m/^net(\d+)$/) {
2043         $i = int($1);
2044     }
2045
2046     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2047
2048     my $ifname = "tap${vmid}i$i";
2049
2050     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2051     die "interface name '$ifname' is too long (max 15 character)\n"
2052         if length($ifname) >= 16;
2053
2054     my $vhostparam = '';
2055     if (is_native($arch)) {
2056         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2057     }
2058
2059     my $vmname = $conf->{name} || "vm$vmid";
2060
2061     my $netdev = "";
2062     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2063
2064     if ($net->{bridge}) {
2065         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2066     } else {
2067         $netdev = "type=user,id=$netid,hostname=$vmname";
2068     }
2069
2070     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2071
2072     return $netdev;
2073 }
2074
2075
2076 sub print_cpu_device {
2077     my ($conf, $id) = @_;
2078
2079     my $kvm = $conf->{kvm} // 1;
2080     my $cpu = $kvm ? "kvm64" : "qemu64";
2081     if (my $cputype = $conf->{cpu}) {
2082         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2083             or die "Cannot parse cpu description: $cputype\n";
2084         $cpu = $cpuconf->{cputype};
2085     }
2086
2087     my $cores = $conf->{cores} || 1;
2088
2089     my $current_core = ($id - 1) % $cores;
2090     my $current_socket = int(($id - 1 - $current_core)/$cores);
2091
2092     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2093 }
2094
2095 my $vga_map = {
2096     'cirrus' => 'cirrus-vga',
2097     'std' => 'VGA',
2098     'vmware' => 'vmware-svga',
2099     'virtio' => 'virtio-vga',
2100 };
2101
2102 sub print_vga_device {
2103     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2104
2105     my $type = $vga_map->{$vga->{type}};
2106     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2107         $type = 'virtio-gpu';
2108     }
2109     my $vgamem_mb = $vga->{memory};
2110     if ($qxlnum) {
2111         $type = $id ? 'qxl' : 'qxl-vga';
2112     }
2113     die "no devicetype for $vga->{type}\n" if !$type;
2114
2115     my $memory = "";
2116     if ($vgamem_mb) {
2117         if ($vga->{type} eq 'virtio') {
2118             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2119             $memory = ",max_hostmem=$bytes";
2120         } elsif ($qxlnum) {
2121             # from https://www.spice-space.org/multiple-monitors.html
2122             $memory = ",vgamem_mb=$vga->{memory}";
2123             my $ram = $vgamem_mb * 4;
2124             my $vram = $vgamem_mb * 2;
2125             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2126         } else {
2127             $memory = ",vgamem_mb=$vga->{memory}";
2128         }
2129     } elsif ($qxlnum && $id) {
2130         $memory = ",ram_size=67108864,vram_size=33554432";
2131     }
2132
2133     my $q35 = machine_type_is_q35($conf);
2134     my $vgaid = "vga" . ($id // '');
2135     my $pciaddr;
2136
2137     if ($q35 && $vgaid eq 'vga') {
2138         # the first display uses pcie.0 bus on q35 machines
2139         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2140     } else {
2141         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2142     }
2143
2144     return "$type,id=${vgaid}${memory}${pciaddr}";
2145 }
2146
2147 sub drive_is_cloudinit {
2148     my ($drive) = @_;
2149     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2150 }
2151
2152 sub drive_is_cdrom {
2153     my ($drive, $exclude_cloudinit) = @_;
2154
2155     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2156
2157     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2158
2159 }
2160
2161 sub parse_number_sets {
2162     my ($set) = @_;
2163     my $res = [];
2164     foreach my $part (split(/;/, $set)) {
2165         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2166             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2167             push @$res, [ $1, $2 ];
2168         } else {
2169             die "invalid range: $part\n";
2170         }
2171     }
2172     return $res;
2173 }
2174
2175 sub parse_numa {
2176     my ($data) = @_;
2177
2178     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2179     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2180     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2181     return $res;
2182 }
2183
2184 sub parse_hostpci {
2185     my ($value) = @_;
2186
2187     return undef if !$value;
2188
2189     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2190
2191     my @idlist = split(/;/, $res->{host});
2192     delete $res->{host};
2193     foreach my $id (@idlist) {
2194         if ($id =~ m/\./) { # full id 00:00.1
2195             push @{$res->{pciid}}, {
2196                 id => $id,
2197             };
2198         } else { # partial id 00:00
2199             $res->{pciid} = PVE::SysFSTools::lspci($id);
2200         }
2201     }
2202     return $res;
2203 }
2204
2205 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2206 sub parse_net {
2207     my ($data) = @_;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2210     if ($@) {
2211         warn $@;
2212         return undef;
2213     }
2214     if (!defined($res->{macaddr})) {
2215         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2216         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2217     }
2218     return $res;
2219 }
2220
2221 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2222 sub parse_ipconfig {
2223     my ($data) = @_;
2224
2225     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2226     if ($@) {
2227         warn $@;
2228         return undef;
2229     }
2230
2231     if ($res->{gw} && !$res->{ip}) {
2232         warn 'gateway specified without specifying an IP address';
2233         return undef;
2234     }
2235     if ($res->{gw6} && !$res->{ip6}) {
2236         warn 'IPv6 gateway specified without specifying an IPv6 address';
2237         return undef;
2238     }
2239     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2240         warn 'gateway specified together with DHCP';
2241         return undef;
2242     }
2243     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2244         # gw6 + auto/dhcp
2245         warn "IPv6 gateway specified together with $res->{ip6} address";
2246         return undef;
2247     }
2248
2249     if (!$res->{ip} && !$res->{ip6}) {
2250         return { ip => 'dhcp', ip6 => 'dhcp' };
2251     }
2252
2253     return $res;
2254 }
2255
2256 sub print_net {
2257     my $net = shift;
2258
2259     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2260 }
2261
2262 sub add_random_macs {
2263     my ($settings) = @_;
2264
2265     foreach my $opt (keys %$settings) {
2266         next if $opt !~ m/^net(\d+)$/;
2267         my $net = parse_net($settings->{$opt});
2268         next if !$net;
2269         $settings->{$opt} = print_net($net);
2270     }
2271 }
2272
2273 sub vm_is_volid_owner {
2274     my ($storecfg, $vmid, $volid) = @_;
2275
2276     if ($volid !~  m|^/|) {
2277         my ($path, $owner);
2278         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2279         if ($owner && ($owner == $vmid)) {
2280             return 1;
2281         }
2282     }
2283
2284     return undef;
2285 }
2286
2287 sub split_flagged_list {
2288     my $text = shift || '';
2289     $text =~ s/[,;]/ /g;
2290     $text =~ s/^\s+//;
2291     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2292 }
2293
2294 sub join_flagged_list {
2295     my ($how, $lst) = @_;
2296     join $how, map { $lst->{$_} . $_ } keys %$lst;
2297 }
2298
2299 sub vmconfig_delete_pending_option {
2300     my ($conf, $key, $force) = @_;
2301
2302     delete $conf->{pending}->{$key};
2303     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2304     $pending_delete_hash->{$key} = $force ? '!' : '';
2305     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2306 }
2307
2308 sub vmconfig_undelete_pending_option {
2309     my ($conf, $key) = @_;
2310
2311     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2312     delete $pending_delete_hash->{$key};
2313
2314     if (%$pending_delete_hash) {
2315         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2316     } else {
2317         delete $conf->{pending}->{delete};
2318     }
2319 }
2320
2321 sub vmconfig_register_unused_drive {
2322     my ($storecfg, $vmid, $conf, $drive) = @_;
2323
2324     if (drive_is_cloudinit($drive)) {
2325         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2326         warn $@ if $@;
2327     } elsif (!drive_is_cdrom($drive)) {
2328         my $volid = $drive->{file};
2329         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2330             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2331         }
2332     }
2333 }
2334
2335 sub vmconfig_cleanup_pending {
2336     my ($conf) = @_;
2337
2338     # remove pending changes when nothing changed
2339     my $changes;
2340     foreach my $opt (keys %{$conf->{pending}}) {
2341         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2342             $changes = 1;
2343             delete $conf->{pending}->{$opt};
2344         }
2345     }
2346
2347     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2348     my $pending_delete_hash = {};
2349     while (my ($opt, $force) = each %$current_delete_hash) {
2350         if (defined($conf->{$opt})) {
2351             $pending_delete_hash->{$opt} = $force;
2352         } else {
2353             $changes = 1;
2354         }
2355     }
2356
2357     if (%$pending_delete_hash) {
2358         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2359     } else {
2360         delete $conf->{pending}->{delete};
2361     }
2362
2363     return $changes;
2364 }
2365
2366 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2367 my $smbios1_fmt = {
2368     uuid => {
2369         type => 'string',
2370         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2371         format_description => 'UUID',
2372         description => "Set SMBIOS1 UUID.",
2373         optional => 1,
2374     },
2375     version => {
2376         type => 'string',
2377         pattern => '\S+',
2378         format_description => 'string',
2379         description => "Set SMBIOS1 version.",
2380         optional => 1,
2381     },
2382     serial => {
2383         type => 'string',
2384         pattern => '\S+',
2385         format_description => 'string',
2386         description => "Set SMBIOS1 serial number.",
2387         optional => 1,
2388     },
2389     manufacturer => {
2390         type => 'string',
2391         pattern => '\S+',
2392         format_description => 'string',
2393         description => "Set SMBIOS1 manufacturer.",
2394         optional => 1,
2395     },
2396     product => {
2397         type => 'string',
2398         pattern => '\S+',
2399         format_description => 'string',
2400         description => "Set SMBIOS1 product ID.",
2401         optional => 1,
2402     },
2403     sku => {
2404         type => 'string',
2405         pattern => '\S+',
2406         format_description => 'string',
2407         description => "Set SMBIOS1 SKU string.",
2408         optional => 1,
2409     },
2410     family => {
2411         type => 'string',
2412         pattern => '\S+',
2413         format_description => 'string',
2414         description => "Set SMBIOS1 family string.",
2415         optional => 1,
2416     },
2417 };
2418
2419 sub parse_smbios1 {
2420     my ($data) = @_;
2421
2422     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2423     warn $@ if $@;
2424     return $res;
2425 }
2426
2427 sub print_smbios1 {
2428     my ($smbios1) = @_;
2429     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2430 }
2431
2432 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2433
2434 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2435 sub verify_bootdisk {
2436     my ($value, $noerr) = @_;
2437
2438     return $value if is_valid_drivename($value);
2439
2440     return undef if $noerr;
2441
2442     die "invalid boot disk '$value'\n";
2443 }
2444
2445 sub parse_watchdog {
2446     my ($value) = @_;
2447
2448     return undef if !$value;
2449
2450     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2451     warn $@ if $@;
2452     return $res;
2453 }
2454
2455 sub parse_guest_agent {
2456     my ($value) = @_;
2457
2458     return {} if !defined($value->{agent});
2459
2460     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2461     warn $@ if $@;
2462
2463     # if the agent is disabled ignore the other potentially set properties
2464     return {} if !$res->{enabled};
2465     return $res;
2466 }
2467
2468 sub parse_vga {
2469     my ($value) = @_;
2470
2471     return {} if !$value;
2472     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2473     warn $@ if $@;
2474     return $res;
2475 }
2476
2477 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2478 sub verify_usb_device {
2479     my ($value, $noerr) = @_;
2480
2481     return $value if parse_usb_device($value);
2482
2483     return undef if $noerr;
2484
2485     die "unable to parse usb device\n";
2486 }
2487
2488 # add JSON properties for create and set function
2489 sub json_config_properties {
2490     my $prop = shift;
2491
2492     foreach my $opt (keys %$confdesc) {
2493         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2494         $prop->{$opt} = $confdesc->{$opt};
2495     }
2496
2497     return $prop;
2498 }
2499
2500 # return copy of $confdesc_cloudinit to generate documentation
2501 sub cloudinit_config_properties {
2502
2503     return dclone($confdesc_cloudinit);
2504 }
2505
2506 sub check_type {
2507     my ($key, $value) = @_;
2508
2509     die "unknown setting '$key'\n" if !$confdesc->{$key};
2510
2511     my $type = $confdesc->{$key}->{type};
2512
2513     if (!defined($value)) {
2514         die "got undefined value\n";
2515     }
2516
2517     if ($value =~ m/[\n\r]/) {
2518         die "property contains a line feed\n";
2519     }
2520
2521     if ($type eq 'boolean') {
2522         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2523         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2524         die "type check ('boolean') failed - got '$value'\n";
2525     } elsif ($type eq 'integer') {
2526         return int($1) if $value =~ m/^(\d+)$/;
2527         die "type check ('integer') failed - got '$value'\n";
2528     } elsif ($type eq 'number') {
2529         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2530         die "type check ('number') failed - got '$value'\n";
2531     } elsif ($type eq 'string') {
2532         if (my $fmt = $confdesc->{$key}->{format}) {
2533             PVE::JSONSchema::check_format($fmt, $value);
2534             return $value;
2535         }
2536         $value =~ s/^\"(.*)\"$/$1/;
2537         return $value;
2538     } else {
2539         die "internal error"
2540     }
2541 }
2542
2543 sub touch_config {
2544     my ($vmid) = @_;
2545
2546     my $conf = PVE::QemuConfig->config_file($vmid);
2547     utime undef, undef, $conf;
2548 }
2549
2550 sub destroy_vm {
2551     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2552
2553     my $conffile = PVE::QemuConfig->config_file($vmid);
2554
2555     my $conf = PVE::QemuConfig->load_config($vmid);
2556
2557     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2558
2559     if ($conf->{template}) {
2560         # check if any base image is still used by a linked clone
2561         foreach_drive($conf, sub {
2562                 my ($ds, $drive) = @_;
2563
2564                 return if drive_is_cdrom($drive);
2565
2566                 my $volid = $drive->{file};
2567
2568                 return if !$volid || $volid =~ m|^/|;
2569
2570                 die "base volume '$volid' is still in use by linked cloned\n"
2571                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2572
2573         });
2574     }
2575
2576     # only remove disks owned by this VM
2577     foreach_drive($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         return if drive_is_cdrom($drive, 1);
2581
2582         my $volid = $drive->{file};
2583
2584         return if !$volid || $volid =~ m|^/|;
2585
2586         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2587         return if !$path || !$owner || ($owner != $vmid);
2588
2589         eval {
2590             PVE::Storage::vdisk_free($storecfg, $volid);
2591         };
2592         warn "Could not remove disk '$volid', check manually: $@" if $@;
2593
2594     });
2595
2596     if ($keep_empty_config) {
2597         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2598     } else {
2599         unlink $conffile;
2600     }
2601
2602     # also remove unused disk
2603     eval {
2604         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2605
2606         eval {
2607             PVE::Storage::foreach_volid($dl, sub {
2608                 my ($volid, $sid, $volname, $d) = @_;
2609                 PVE::Storage::vdisk_free($storecfg, $volid);
2610             });
2611         };
2612         warn $@ if $@;
2613
2614     };
2615     warn $@ if $@;
2616 }
2617
2618 sub parse_vm_config {
2619     my ($filename, $raw) = @_;
2620
2621     return undef if !defined($raw);
2622
2623     my $res = {
2624         digest => Digest::SHA::sha1_hex($raw),
2625         snapshots => {},
2626         pending => {},
2627     };
2628
2629     $filename =~ m|/qemu-server/(\d+)\.conf$|
2630         || die "got strange filename '$filename'";
2631
2632     my $vmid = $1;
2633
2634     my $conf = $res;
2635     my $descr;
2636     my $section = '';
2637
2638     my @lines = split(/\n/, $raw);
2639     foreach my $line (@lines) {
2640         next if $line =~ m/^\s*$/;
2641
2642         if ($line =~ m/^\[PENDING\]\s*$/i) {
2643             $section = 'pending';
2644             if (defined($descr)) {
2645                 $descr =~ s/\s+$//;
2646                 $conf->{description} = $descr;
2647             }
2648             $descr = undef;
2649             $conf = $res->{$section} = {};
2650             next;
2651
2652         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2653             $section = $1;
2654             if (defined($descr)) {
2655                 $descr =~ s/\s+$//;
2656                 $conf->{description} = $descr;
2657             }
2658             $descr = undef;
2659             $conf = $res->{snapshots}->{$section} = {};
2660             next;
2661         }
2662
2663         if ($line =~ m/^\#(.*)\s*$/) {
2664             $descr = '' if !defined($descr);
2665             $descr .= PVE::Tools::decode_text($1) . "\n";
2666             next;
2667         }
2668
2669         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2670             $descr = '' if !defined($descr);
2671             $descr .= PVE::Tools::decode_text($2);
2672         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2673             $conf->{snapstate} = $1;
2674         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2675             my $key = $1;
2676             my $value = $2;
2677             $conf->{$key} = $value;
2678         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2679             my $value = $1;
2680             if ($section eq 'pending') {
2681                 $conf->{delete} = $value; # we parse this later
2682             } else {
2683                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2684             }
2685         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2686             my $key = $1;
2687             my $value = $2;
2688             eval { $value = check_type($key, $value); };
2689             if ($@) {
2690                 warn "vm $vmid - unable to parse value of '$key' - $@";
2691             } else {
2692                 $key = 'ide2' if $key eq 'cdrom';
2693                 my $fmt = $confdesc->{$key}->{format};
2694                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2695                     my $v = parse_drive($key, $value);
2696                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2697                         $v->{file} = $volid;
2698                         $value = print_drive($vmid, $v);
2699                     } else {
2700                         warn "vm $vmid - unable to parse value of '$key'\n";
2701                         next;
2702                     }
2703                 }
2704
2705                 $conf->{$key} = $value;
2706             }
2707         }
2708     }
2709
2710     if (defined($descr)) {
2711         $descr =~ s/\s+$//;
2712         $conf->{description} = $descr;
2713     }
2714     delete $res->{snapstate}; # just to be sure
2715
2716     return $res;
2717 }
2718
2719 sub write_vm_config {
2720     my ($filename, $conf) = @_;
2721
2722     delete $conf->{snapstate}; # just to be sure
2723
2724     if ($conf->{cdrom}) {
2725         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2726         $conf->{ide2} = $conf->{cdrom};
2727         delete $conf->{cdrom};
2728     }
2729
2730     # we do not use 'smp' any longer
2731     if ($conf->{sockets}) {
2732         delete $conf->{smp};
2733     } elsif ($conf->{smp}) {
2734         $conf->{sockets} = $conf->{smp};
2735         delete $conf->{cores};
2736         delete $conf->{smp};
2737     }
2738
2739     my $used_volids = {};
2740
2741     my $cleanup_config = sub {
2742         my ($cref, $pending, $snapname) = @_;
2743
2744         foreach my $key (keys %$cref) {
2745             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2746                 $key eq 'snapstate' || $key eq 'pending';
2747             my $value = $cref->{$key};
2748             if ($key eq 'delete') {
2749                 die "propertry 'delete' is only allowed in [PENDING]\n"
2750                     if !$pending;
2751                 # fixme: check syntax?
2752                 next;
2753             }
2754             eval { $value = check_type($key, $value); };
2755             die "unable to parse value of '$key' - $@" if $@;
2756
2757             $cref->{$key} = $value;
2758
2759             if (!$snapname && is_valid_drivename($key)) {
2760                 my $drive = parse_drive($key, $value);
2761                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2762             }
2763         }
2764     };
2765
2766     &$cleanup_config($conf);
2767
2768     &$cleanup_config($conf->{pending}, 1);
2769
2770     foreach my $snapname (keys %{$conf->{snapshots}}) {
2771         die "internal error" if $snapname eq 'pending';
2772         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2773     }
2774
2775     # remove 'unusedX' settings if we re-add a volume
2776     foreach my $key (keys %$conf) {
2777         my $value = $conf->{$key};
2778         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2779             delete $conf->{$key};
2780         }
2781     }
2782
2783     my $generate_raw_config = sub {
2784         my ($conf, $pending) = @_;
2785
2786         my $raw = '';
2787
2788         # add description as comment to top of file
2789         if (defined(my $descr = $conf->{description})) {
2790             if ($descr) {
2791                 foreach my $cl (split(/\n/, $descr)) {
2792                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2793                 }
2794             } else {
2795                 $raw .= "#\n" if $pending;
2796             }
2797         }
2798
2799         foreach my $key (sort keys %$conf) {
2800             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2801             $raw .= "$key: $conf->{$key}\n";
2802         }
2803         return $raw;
2804     };
2805
2806     my $raw = &$generate_raw_config($conf);
2807
2808     if (scalar(keys %{$conf->{pending}})){
2809         $raw .= "\n[PENDING]\n";
2810         $raw .= &$generate_raw_config($conf->{pending}, 1);
2811     }
2812
2813     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2814         $raw .= "\n[$snapname]\n";
2815         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2816     }
2817
2818     return $raw;
2819 }
2820
2821 sub load_defaults {
2822
2823     my $res = {};
2824
2825     # we use static defaults from our JSON schema configuration
2826     foreach my $key (keys %$confdesc) {
2827         if (defined(my $default = $confdesc->{$key}->{default})) {
2828             $res->{$key} = $default;
2829         }
2830     }
2831
2832     return $res;
2833 }
2834
2835 sub config_list {
2836     my $vmlist = PVE::Cluster::get_vmlist();
2837     my $res = {};
2838     return $res if !$vmlist || !$vmlist->{ids};
2839     my $ids = $vmlist->{ids};
2840
2841     foreach my $vmid (keys %$ids) {
2842         my $d = $ids->{$vmid};
2843         next if !$d->{node} || $d->{node} ne $nodename;
2844         next if !$d->{type} || $d->{type} ne 'qemu';
2845         $res->{$vmid}->{exists} = 1;
2846     }
2847     return $res;
2848 }
2849
2850 # test if VM uses local resources (to prevent migration)
2851 sub check_local_resources {
2852     my ($conf, $noerr) = @_;
2853
2854     my $loc_res = 0;
2855
2856     $loc_res = 1 if $conf->{hostusb}; # old syntax
2857     $loc_res = 1 if $conf->{hostpci}; # old syntax
2858
2859     $loc_res = 1 if $conf->{ivshmem};
2860
2861     foreach my $k (keys %$conf) {
2862         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2863         # sockets are safe: they will recreated be on the target side post-migrate
2864         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2865         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2866     }
2867
2868     die "VM uses local resources\n" if $loc_res && !$noerr;
2869
2870     return $loc_res;
2871 }
2872
2873 # check if used storages are available on all nodes (use by migrate)
2874 sub check_storage_availability {
2875     my ($storecfg, $conf, $node) = @_;
2876
2877     foreach_drive($conf, sub {
2878         my ($ds, $drive) = @_;
2879
2880         my $volid = $drive->{file};
2881         return if !$volid;
2882
2883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2884         return if !$sid;
2885
2886         # check if storage is available on both nodes
2887         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2888         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2889    });
2890 }
2891
2892 # list nodes where all VM images are available (used by has_feature API)
2893 sub shared_nodes {
2894     my ($conf, $storecfg) = @_;
2895
2896     my $nodelist = PVE::Cluster::get_nodelist();
2897     my $nodehash = { map { $_ => 1 } @$nodelist };
2898     my $nodename = PVE::INotify::nodename();
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909             if ($scfg->{disable}) {
2910                 $nodehash = {};
2911             } elsif (my $avail = $scfg->{nodes}) {
2912                 foreach my $node (keys %$nodehash) {
2913                     delete $nodehash->{$node} if !$avail->{$node};
2914                 }
2915             } elsif (!$scfg->{shared}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if $node ne $nodename
2918                 }
2919             }
2920         }
2921     });
2922
2923     return $nodehash
2924 }
2925
2926 sub check_cmdline {
2927     my ($pidfile, $pid) = @_;
2928
2929     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2930     if (defined($fh)) {
2931         my $line = <$fh>;
2932         $fh->close;
2933         return undef if !$line;
2934         my @param = split(/\0/, $line);
2935
2936         my $cmd = $param[0];
2937         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2938
2939         for (my $i = 0; $i < scalar (@param); $i++) {
2940             my $p = $param[$i];
2941             next if !$p;
2942             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2943                 my $p = $param[$i+1];
2944                 return 1 if $p && ($p eq $pidfile);
2945                 return undef;
2946             }
2947         }
2948     }
2949     return undef;
2950 }
2951
2952 sub check_running {
2953     my ($vmid, $nocheck, $node) = @_;
2954
2955     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2956
2957     die "unable to find configuration file for VM $vmid - no such machine\n"
2958         if !$nocheck && ! -f $filename;
2959
2960     my $pidfile = pidfile_name($vmid);
2961
2962     if (my $fd = IO::File->new("<$pidfile")) {
2963         my $st = stat($fd);
2964         my $line = <$fd>;
2965         close($fd);
2966
2967         my $mtime = $st->mtime;
2968         if ($mtime > time()) {
2969             warn "file '$filename' modified in future\n";
2970         }
2971
2972         if ($line =~ m/^(\d+)$/) {
2973             my $pid = $1;
2974             if (check_cmdline($pidfile, $pid)) {
2975                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2976                     return $pid;
2977                 }
2978             }
2979         }
2980     }
2981
2982     return undef;
2983 }
2984
2985 sub vzlist {
2986
2987     my $vzlist = config_list();
2988
2989     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2990
2991     while (defined(my $de = $fd->read)) {
2992         next if $de !~ m/^(\d+)\.pid$/;
2993         my $vmid = $1;
2994         next if !defined($vzlist->{$vmid});
2995         if (my $pid = check_running($vmid)) {
2996             $vzlist->{$vmid}->{pid} = $pid;
2997         }
2998     }
2999
3000     return $vzlist;
3001 }
3002
3003 sub disksize {
3004     my ($storecfg, $conf) = @_;
3005
3006     my $bootdisk = $conf->{bootdisk};
3007     return undef if !$bootdisk;
3008     return undef if !is_valid_drivename($bootdisk);
3009
3010     return undef if !$conf->{$bootdisk};
3011
3012     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3013     return undef if !defined($drive);
3014
3015     return undef if drive_is_cdrom($drive);
3016
3017     my $volid = $drive->{file};
3018     return undef if !$volid;
3019
3020     return $drive->{size};
3021 }
3022
3023 our $vmstatus_return_properties = {
3024     vmid => get_standard_option('pve-vmid'),
3025     status => {
3026         description => "Qemu process status.",
3027         type => 'string',
3028         enum => ['stopped', 'running'],
3029     },
3030     maxmem => {
3031         description => "Maximum memory in bytes.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'bytes',
3035     },
3036     maxdisk => {
3037         description => "Root disk size in bytes.",
3038         type => 'integer',
3039         optional => 1,
3040         renderer => 'bytes',
3041     },
3042     name => {
3043         description => "VM name.",
3044         type => 'string',
3045         optional => 1,
3046     },
3047     qmpstatus => {
3048         description => "Qemu QMP agent status.",
3049         type => 'string',
3050         optional => 1,
3051     },
3052     pid => {
3053         description => "PID of running qemu process.",
3054         type => 'integer',
3055         optional => 1,
3056     },
3057     uptime => {
3058         description => "Uptime.",
3059         type => 'integer',
3060         optional => 1,
3061         renderer => 'duration',
3062     },
3063     cpus => {
3064         description => "Maximum usable CPUs.",
3065         type => 'number',
3066         optional => 1,
3067     },
3068 };
3069
3070 my $last_proc_pid_stat;
3071
3072 # get VM status information
3073 # This must be fast and should not block ($full == false)
3074 # We only query KVM using QMP if $full == true (this can be slow)
3075 sub vmstatus {
3076     my ($opt_vmid, $full) = @_;
3077
3078     my $res = {};
3079
3080     my $storecfg = PVE::Storage::config();
3081
3082     my $list = vzlist();
3083     my $defaults = load_defaults();
3084
3085     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3086
3087     my $cpucount = $cpuinfo->{cpus} || 1;
3088
3089     foreach my $vmid (keys %$list) {
3090         next if $opt_vmid && ($vmid ne $opt_vmid);
3091
3092         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3093         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3094
3095         my $d = { vmid => $vmid };
3096         $d->{pid} = $list->{$vmid}->{pid};
3097
3098         # fixme: better status?
3099         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3100
3101         my $size = disksize($storecfg, $conf);
3102         if (defined($size)) {
3103             $d->{disk} = 0; # no info available
3104             $d->{maxdisk} = $size;
3105         } else {
3106             $d->{disk} = 0;
3107             $d->{maxdisk} = 0;
3108         }
3109
3110         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3111             * ($conf->{cores} || $defaults->{cores});
3112         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3113         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3114
3115         $d->{name} = $conf->{name} || "VM $vmid";
3116         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3117             : $defaults->{memory}*(1024*1024);
3118
3119         if ($conf->{balloon}) {
3120             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3121             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3122                 : $defaults->{shares};
3123         }
3124
3125         $d->{uptime} = 0;
3126         $d->{cpu} = 0;
3127         $d->{mem} = 0;
3128
3129         $d->{netout} = 0;
3130         $d->{netin} = 0;
3131
3132         $d->{diskread} = 0;
3133         $d->{diskwrite} = 0;
3134
3135         $d->{template} = PVE::QemuConfig->is_template($conf);
3136
3137         $d->{serial} = 1 if conf_has_serial($conf);
3138
3139         $res->{$vmid} = $d;
3140     }
3141
3142     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3143     foreach my $dev (keys %$netdev) {
3144         next if $dev !~ m/^tap([1-9]\d*)i/;
3145         my $vmid = $1;
3146         my $d = $res->{$vmid};
3147         next if !$d;
3148
3149         $d->{netout} += $netdev->{$dev}->{receive};
3150         $d->{netin} += $netdev->{$dev}->{transmit};
3151
3152         if ($full) {
3153             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3154             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3155         }
3156
3157     }
3158
3159     my $ctime = gettimeofday;
3160
3161     foreach my $vmid (keys %$list) {
3162
3163         my $d = $res->{$vmid};
3164         my $pid = $d->{pid};
3165         next if !$pid;
3166
3167         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3168         next if !$pstat; # not running
3169
3170         my $used = $pstat->{utime} + $pstat->{stime};
3171
3172         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3173
3174         if ($pstat->{vsize}) {
3175             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3176         }
3177
3178         my $old = $last_proc_pid_stat->{$pid};
3179         if (!$old) {
3180             $last_proc_pid_stat->{$pid} = {
3181                 time => $ctime,
3182                 used => $used,
3183                 cpu => 0,
3184             };
3185             next;
3186         }
3187
3188         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3189
3190         if ($dtime > 1000) {
3191             my $dutime = $used -  $old->{used};
3192
3193             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3194             $last_proc_pid_stat->{$pid} = {
3195                 time => $ctime,
3196                 used => $used,
3197                 cpu => $d->{cpu},
3198             };
3199         } else {
3200             $d->{cpu} = $old->{cpu};
3201         }
3202     }
3203
3204     return $res if !$full;
3205
3206     my $qmpclient = PVE::QMPClient->new();
3207
3208     my $ballooncb = sub {
3209         my ($vmid, $resp) = @_;
3210
3211         my $info = $resp->{'return'};
3212         return if !$info->{max_mem};
3213
3214         my $d = $res->{$vmid};
3215
3216         # use memory assigned to VM
3217         $d->{maxmem} = $info->{max_mem};
3218         $d->{balloon} = $info->{actual};
3219
3220         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3221             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3222             $d->{freemem} = $info->{free_mem};
3223         }
3224
3225         $d->{ballooninfo} = $info;
3226     };
3227
3228     my $blockstatscb = sub {
3229         my ($vmid, $resp) = @_;
3230         my $data = $resp->{'return'} || [];
3231         my $totalrdbytes = 0;
3232         my $totalwrbytes = 0;
3233
3234         for my $blockstat (@$data) {
3235             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3236             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3237
3238             $blockstat->{device} =~ s/drive-//;
3239             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3240         }
3241         $res->{$vmid}->{diskread} = $totalrdbytes;
3242         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3243     };
3244
3245     my $statuscb = sub {
3246         my ($vmid, $resp) = @_;
3247
3248         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3249         # this fails if ballon driver is not loaded, so this must be
3250         # the last commnand (following command are aborted if this fails).
3251         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3252
3253         my $status = 'unknown';
3254         if (!defined($status = $resp->{'return'}->{status})) {
3255             warn "unable to get VM status\n";
3256             return;
3257         }
3258
3259         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3260     };
3261
3262     foreach my $vmid (keys %$list) {
3263         next if $opt_vmid && ($vmid ne $opt_vmid);
3264         next if !$res->{$vmid}->{pid}; # not running
3265         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3266     }
3267
3268     $qmpclient->queue_execute(undef, 2);
3269
3270     foreach my $vmid (keys %$list) {
3271         next if $opt_vmid && ($vmid ne $opt_vmid);
3272         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3273     }
3274
3275     return $res;
3276 }
3277
3278 sub foreach_drive {
3279     my ($conf, $func, @param) = @_;
3280
3281     foreach my $ds (valid_drive_names()) {
3282         next if !defined($conf->{$ds});
3283
3284         my $drive = parse_drive($ds, $conf->{$ds});
3285         next if !$drive;
3286
3287         &$func($ds, $drive, @param);
3288     }
3289 }
3290
3291 sub foreach_volid {
3292     my ($conf, $func, @param) = @_;
3293
3294     my $volhash = {};
3295
3296     my $test_volid = sub {
3297         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3298
3299         return if !$volid;
3300
3301         $volhash->{$volid}->{cdrom} //= 1;
3302         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3303
3304         $volhash->{$volid}->{replicate} //= 0;
3305         $volhash->{$volid}->{replicate} = 1 if $replicate;
3306
3307         $volhash->{$volid}->{shared} //= 0;
3308         $volhash->{$volid}->{shared} = 1 if $shared;
3309
3310         $volhash->{$volid}->{referenced_in_config} //= 0;
3311         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3312
3313         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3314             if defined($snapname);
3315     };
3316
3317     foreach_drive($conf, sub {
3318         my ($ds, $drive) = @_;
3319         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3320     });
3321
3322     foreach my $snapname (keys %{$conf->{snapshots}}) {
3323         my $snap = $conf->{snapshots}->{$snapname};
3324         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3325         foreach_drive($snap, sub {
3326             my ($ds, $drive) = @_;
3327             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3328         });
3329     }
3330
3331     foreach my $volid (keys %$volhash) {
3332         &$func($volid, $volhash->{$volid}, @param);
3333     }
3334 }
3335
3336 sub conf_has_serial {
3337     my ($conf) = @_;
3338
3339     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3340         if ($conf->{"serial$i"}) {
3341             return 1;
3342         }
3343     }
3344
3345     return 0;
3346 }
3347
3348 sub vga_conf_has_spice {
3349     my ($vga) = @_;
3350
3351     my $vgaconf = parse_vga($vga);
3352     my $vgatype = $vgaconf->{type};
3353     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3354
3355     return $1 || 1;
3356 }
3357
3358 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3359 sub get_host_arch() {
3360     $host_arch = (POSIX::uname())[4] if !$host_arch;
3361     return $host_arch;
3362 }
3363
3364 sub is_native($) {
3365     my ($arch) = @_;
3366     return get_host_arch() eq $arch;
3367 }
3368
3369 my $default_machines = {
3370     x86_64 => 'pc',
3371     aarch64 => 'virt',
3372 };
3373
3374 sub get_basic_machine_info {
3375     my ($conf, $forcemachine) = @_;
3376
3377     my $arch = $conf->{arch} // get_host_arch();
3378     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3379     return ($arch, $machine);
3380 }
3381
3382 sub get_ovmf_files($) {
3383     my ($arch) = @_;
3384
3385     my $ovmf = $OVMF->{$arch}
3386         or die "no OVMF images known for architecture '$arch'\n";
3387
3388     return @$ovmf;
3389 }
3390
3391 my $Arch2Qemu = {
3392     aarch64 => '/usr/bin/qemu-system-aarch64',
3393     x86_64 => '/usr/bin/qemu-system-x86_64',
3394 };
3395 sub get_command_for_arch($) {
3396     my ($arch) = @_;
3397     return '/usr/bin/kvm' if is_native($arch);
3398
3399     my $cmd = $Arch2Qemu->{$arch}
3400         or die "don't know how to emulate architecture '$arch'\n";
3401     return $cmd;
3402 }
3403
3404 sub get_cpu_options {
3405     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3406
3407     my $cpuFlags = [];
3408     my $ostype = $conf->{ostype};
3409
3410     my $cpu = $kvm ? "kvm64" : "qemu64";
3411     if ($arch eq 'aarch64') {
3412         $cpu = 'cortex-a57';
3413     }
3414     my $hv_vendor_id;
3415     if (my $cputype = $conf->{cpu}) {
3416         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3417             or die "Cannot parse cpu description: $cputype\n";
3418         $cpu = $cpuconf->{cputype};
3419         $kvm_off = 1 if $cpuconf->{hidden};
3420         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3421
3422         if (defined(my $flags = $cpuconf->{flags})) {
3423             push @$cpuFlags, split(";", $flags);
3424         }
3425     }
3426
3427     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3428
3429     push @$cpuFlags , '-x2apic'
3430         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3431
3432     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3433
3434     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3435
3436     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3437
3438         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3439         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3440     }
3441
3442     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3443
3444     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3445
3446     push @$cpuFlags, 'kvm=off' if $kvm_off;
3447
3448     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3449         push @$cpuFlags, "vendor=${cpu_vendor}"
3450             if $cpu_vendor ne 'default';
3451     } elsif ($arch ne 'aarch64') {
3452         die "internal error"; # should not happen
3453     }
3454
3455     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3456
3457     return ('-cpu', $cpu);
3458 }
3459
3460 sub config_to_command {
3461     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3462
3463     my $cmd = [];
3464     my $globalFlags = [];
3465     my $machineFlags = [];
3466     my $rtcFlags = [];
3467     my $devices = [];
3468     my $pciaddr = '';
3469     my $bridges = {};
3470     my $kvmver = kvm_user_version();
3471     my $vernum = 0; # unknown
3472     my $ostype = $conf->{ostype};
3473     my $winversion = windows_version($ostype);
3474     my $kvm = $conf->{kvm};
3475
3476     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3477     $kvm //= 1 if is_native($arch);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $have_ovz = -f '/proc/vz/vestat';
3493
3494     my $q35 = machine_type_is_q35($conf);
3495     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3496     my $use_old_bios_files = undef;
3497     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3498
3499     my $cpuunits = defined($conf->{cpuunits}) ?
3500             $conf->{cpuunits} : $defaults->{cpuunits};
3501
3502     push @$cmd, get_command_for_arch($arch);
3503
3504     push @$cmd, '-id', $vmid;
3505
3506     my $vmname = $conf->{name} || "vm$vmid";
3507
3508     push @$cmd, '-name', $vmname;
3509
3510     my $use_virtio = 0;
3511
3512     my $qmpsocket = qmp_socket($vmid);
3513     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3514     push @$cmd, '-mon', "chardev=qmp,mode=control";
3515
3516     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3517         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3518         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3519     }
3520
3521     push @$cmd, '-pidfile' , pidfile_name($vmid);
3522
3523     push @$cmd, '-daemonize';
3524
3525     if ($conf->{smbios1}) {
3526         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3527     }
3528
3529     if ($conf->{vmgenid}) {
3530         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3531     }
3532
3533     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3534     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3535         die "uefi base image not found\n" if ! -f $ovmf_code;
3536
3537         my $path;
3538         my $format;
3539         if (my $efidisk = $conf->{efidisk0}) {
3540             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3541             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3542             $format = $d->{format};
3543             if ($storeid) {
3544                 $path = PVE::Storage::path($storecfg, $d->{file});
3545                 if (!defined($format)) {
3546                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3547                     $format = qemu_img_format($scfg, $volname);
3548                 }
3549             } else {
3550                 $path = $d->{file};
3551                 die "efidisk format must be specified\n"
3552                     if !defined($format);
3553             }
3554         } else {
3555             warn "no efidisk configured! Using temporary efivars disk.\n";
3556             $path = "/tmp/$vmid-ovmf.fd";
3557             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3558             $format = 'raw';
3559         }
3560
3561         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3562         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3563     }
3564
3565
3566     # add usb controllers
3567     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3568     push @$devices, @usbcontrollers if @usbcontrollers;
3569     my $vga = parse_vga($conf->{vga});
3570
3571     my $qxlnum = vga_conf_has_spice($conf->{vga});
3572     $vga->{type} = 'qxl' if $qxlnum;
3573
3574     if (!$vga->{type}) {
3575         if ($arch eq 'aarch64') {
3576             $vga->{type} = 'virtio';
3577         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3578             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3579         } else {
3580             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3581         }
3582     }
3583
3584     # enable absolute mouse coordinates (needed by vnc)
3585     my $tablet;
3586     if (defined($conf->{tablet})) {
3587         $tablet = $conf->{tablet};
3588     } else {
3589         $tablet = $defaults->{tablet};
3590         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3591         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3592     }
3593
3594     if ($tablet) {
3595         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3596         my $kbd = print_keyboarddevice_full($conf, $arch);
3597         push @$devices, '-device', $kbd if defined($kbd);
3598     }
3599
3600     my $kvm_off = 0;
3601     my $gpu_passthrough;
3602
3603     # host pci devices
3604     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3605         my $d = parse_hostpci($conf->{"hostpci$i"});
3606         next if !$d;
3607
3608         my $pcie = $d->{pcie};
3609         if ($pcie) {
3610             die "q35 machine model is not enabled" if !$q35;
3611             # win7 wants to have the pcie devices directly on the pcie bus
3612             # instead of in the root port
3613             if ($winversion == 7) {
3614                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3615             } else {
3616                 $pciaddr = print_pcie_addr("hostpci$i");
3617             }
3618         } else {
3619             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3620         }
3621
3622         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3623         my $romfile = $d->{romfile};
3624
3625         my $xvga = '';
3626         if ($d->{'x-vga'}) {
3627             $xvga = ',x-vga=on';
3628             $kvm_off = 1;
3629             $vga->{type} = 'none' if !defined($conf->{vga});
3630             $gpu_passthrough = 1;
3631
3632             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3633                 $xvga = "";
3634             }
3635         }
3636         my $pcidevices = $d->{pciid};
3637         my $multifunction = 1 if @$pcidevices > 1;
3638         my $sysfspath;
3639         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3640             my $id = $pcidevices->[0]->{id};
3641             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3642             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3643         } elsif ($d->{mdev}) {
3644             warn "ignoring mediated device with multifunction device\n";
3645         }
3646
3647         my $j=0;
3648         foreach my $pcidevice (@$pcidevices) {
3649
3650             my $id = "hostpci$i";
3651             $id .= ".$j" if $multifunction;
3652             my $addr = $pciaddr;
3653             $addr .= ".$j" if $multifunction;
3654             my $devicestr = "vfio-pci";
3655             if ($sysfspath) {
3656                 $devicestr .= ",sysfsdev=$sysfspath";
3657             } else {
3658                 $devicestr .= ",host=$pcidevice->{id}";
3659             }
3660             $devicestr .= ",id=$id$addr";
3661
3662             if($j == 0){
3663                 $devicestr .= "$rombar$xvga";
3664                 $devicestr .= ",multifunction=on" if $multifunction;
3665                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3666             }
3667
3668             push @$devices, '-device', $devicestr;
3669             $j++;
3670         }
3671     }
3672
3673     # usb devices
3674     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3675     push @$devices, @usbdevices if @usbdevices;
3676     # serial devices
3677     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3678         if (my $path = $conf->{"serial$i"}) {
3679             if ($path eq 'socket') {
3680                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3681                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3682                 # On aarch64, serial0 is the UART device. Qemu only allows
3683                 # connecting UART devices via the '-serial' command line, as
3684                 # the device has a fixed slot on the hardware...
3685                 if ($arch eq 'aarch64' && $i == 0) {
3686                     push @$devices, '-serial', "chardev:serial$i";
3687                 } else {
3688                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3689                 }
3690             } else {
3691                 die "no such serial device\n" if ! -c $path;
3692                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3693                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3694             }
3695         }
3696     }
3697
3698     # parallel devices
3699     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3700         if (my $path = $conf->{"parallel$i"}) {
3701             die "no such parallel device\n" if ! -c $path;
3702             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3703             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3704             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3705         }
3706     }
3707
3708
3709     my $sockets = 1;
3710     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3711     $sockets = $conf->{sockets} if  $conf->{sockets};
3712
3713     my $cores = $conf->{cores} || 1;
3714
3715     my $maxcpus = $sockets * $cores;
3716
3717     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3718
3719     my $allowed_vcpus = $cpuinfo->{cpus};
3720
3721     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3722         if ($allowed_vcpus < $maxcpus);
3723
3724     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3725
3726         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3727         for (my $i = 2; $i <= $vcpus; $i++)  {
3728             my $cpustr = print_cpu_device($conf,$i);
3729             push @$cmd, '-device', $cpustr;
3730         }
3731
3732     } else {
3733
3734         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3735     }
3736     push @$cmd, '-nodefaults';
3737
3738     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3739
3740     my $bootindex_hash = {};
3741     my $i = 1;
3742     foreach my $o (split(//, $bootorder)) {
3743         $bootindex_hash->{$o} = $i*100;
3744         $i++;
3745     }
3746
3747     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3748
3749     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3750
3751     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3752
3753     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3754         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3755         my $socket = vnc_socket($vmid);
3756         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3757     } else {
3758         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3759         push @$cmd, '-nographic';
3760     }
3761
3762     # time drift fix
3763     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3764
3765     my $useLocaltime = $conf->{localtime};
3766
3767     if ($winversion >= 5) { # windows
3768         $useLocaltime = 1 if !defined($conf->{localtime});
3769
3770         # use time drift fix when acpi is enabled
3771         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3772             $tdf = 1 if !defined($conf->{tdf});
3773         }
3774     }
3775
3776     if ($winversion >= 6) {
3777         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3778         push @$cmd, '-no-hpet';
3779     }
3780
3781     push @$rtcFlags, 'driftfix=slew' if $tdf;
3782
3783     if (!$kvm) {
3784         push @$machineFlags, 'accel=tcg';
3785     }
3786
3787     if ($machine_type) {
3788         push @$machineFlags, "type=${machine_type}";
3789     }
3790
3791     if ($conf->{startdate}) {
3792         push @$rtcFlags, "base=$conf->{startdate}";
3793     } elsif ($useLocaltime) {
3794         push @$rtcFlags, 'base=localtime';
3795     }
3796
3797     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3798
3799     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3800
3801     push @$cmd, '-S' if $conf->{freeze};
3802
3803     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3804
3805     # enable sound
3806     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3807     #push @$cmd, '-soundhw', 'es1370';
3808     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3809
3810     if (parse_guest_agent($conf)->{enabled}) {
3811         my $qgasocket = qmp_socket($vmid, 1);
3812         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3813         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3814         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3815         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3816     }
3817
3818     my $spice_port;
3819
3820     if ($qxlnum) {
3821         if ($qxlnum > 1) {
3822             if ($winversion){
3823                 for(my $i = 1; $i < $qxlnum; $i++){
3824                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3825                 }
3826             } else {
3827                 # assume other OS works like Linux
3828                 my ($ram, $vram) = ("134217728", "67108864");
3829                 if ($vga->{memory}) {
3830                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3831                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3832                 }
3833                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3834                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3835             }
3836         }
3837
3838         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3839
3840         my $nodename = PVE::INotify::nodename();
3841         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3842         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3843         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3844         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3845         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3846
3847         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3848
3849         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3850         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3851         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3852     }
3853
3854     # enable balloon by default, unless explicitly disabled
3855     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3856         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3857         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3858     }
3859
3860     if ($conf->{watchdog}) {
3861         my $wdopts = parse_watchdog($conf->{watchdog});
3862         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3863         my $watchdog = $wdopts->{model} || 'i6300esb';
3864         push @$devices, '-device', "$watchdog$pciaddr";
3865         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3866     }
3867
3868     my $vollist = [];
3869     my $scsicontroller = {};
3870     my $ahcicontroller = {};
3871     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3872
3873     # Add iscsi initiator name if available
3874     if (my $initiator = get_initiator_name()) {
3875         push @$devices, '-iscsi', "initiator-name=$initiator";
3876     }
3877
3878     foreach_drive($conf, sub {
3879         my ($ds, $drive) = @_;
3880
3881         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3882             push @$vollist, $drive->{file};
3883         }
3884
3885         # ignore efidisk here, already added in bios/fw handling code above
3886         return if $drive->{interface} eq 'efidisk';
3887
3888         $use_virtio = 1 if $ds =~ m/^virtio/;
3889
3890         if (drive_is_cdrom ($drive)) {
3891             if ($bootindex_hash->{d}) {
3892                 $drive->{bootindex} = $bootindex_hash->{d};
3893                 $bootindex_hash->{d} += 1;
3894             }
3895         } else {
3896             if ($bootindex_hash->{c}) {
3897                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3898                 $bootindex_hash->{c} += 1;
3899             }
3900         }
3901
3902         if($drive->{interface} eq 'virtio'){
3903            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3904         }
3905
3906         if ($drive->{interface} eq 'scsi') {
3907
3908             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3909
3910             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3911             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3912
3913             my $iothread = '';
3914             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3915                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3916                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3917             } elsif ($drive->{iothread}) {
3918                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3919             }
3920
3921             my $queues = '';
3922             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3923                 $queues = ",num_queues=$drive->{queues}";
3924             }
3925
3926             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3927             $scsicontroller->{$controller}=1;
3928         }
3929
3930         if ($drive->{interface} eq 'sata') {
3931            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3932            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3933            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3934            $ahcicontroller->{$controller}=1;
3935         }
3936
3937         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3938         push @$devices, '-drive',$drive_cmd;
3939         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3940     });
3941
3942     for (my $i = 0; $i < $MAX_NETS; $i++) {
3943          next if !$conf->{"net$i"};
3944          my $d = parse_net($conf->{"net$i"});
3945          next if !$d;
3946
3947          $use_virtio = 1 if $d->{model} eq 'virtio';
3948
3949          if ($bootindex_hash->{n}) {
3950             $d->{bootindex} = $bootindex_hash->{n};
3951             $bootindex_hash->{n} += 1;
3952          }
3953
3954          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3955          push @$devices, '-netdev', $netdevfull;
3956
3957          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3958          push @$devices, '-device', $netdevicefull;
3959     }
3960
3961     if ($conf->{ivshmem}) {
3962         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3963
3964         my $bus;
3965         if ($q35) {
3966             $bus = print_pcie_addr("ivshmem");
3967         } else {
3968             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3969         }
3970
3971         my $ivshmem_name = $ivshmem->{name} // $vmid;
3972         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3973
3974         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3975         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3976     }
3977
3978     if (!$q35) {
3979         # add pci bridges
3980         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3981            $bridges->{1} = 1;
3982            $bridges->{2} = 1;
3983         }
3984
3985         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3986
3987         while (my ($k, $v) = each %$bridges) {
3988             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3989             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3990         }
3991     }
3992
3993     push @$cmd, @$devices;
3994     push @$cmd, '-rtc', join(',', @$rtcFlags)
3995         if scalar(@$rtcFlags);
3996     push @$cmd, '-machine', join(',', @$machineFlags)
3997         if scalar(@$machineFlags);
3998     push @$cmd, '-global', join(',', @$globalFlags)
3999         if scalar(@$globalFlags);
4000
4001     if (my $vmstate = $conf->{vmstate}) {
4002         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4003         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4004         push @$cmd, '-loadstate', $statepath;
4005     }
4006
4007     # add custom args
4008     if ($conf->{args}) {
4009         my $aa = PVE::Tools::split_args($conf->{args});
4010         push @$cmd, @$aa;
4011     }
4012
4013     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4014 }
4015
4016 sub vnc_socket {
4017     my ($vmid) = @_;
4018     return "${var_run_tmpdir}/$vmid.vnc";
4019 }
4020
4021 sub spice_port {
4022     my ($vmid) = @_;
4023
4024     my $res = vm_mon_cmd($vmid, 'query-spice');
4025
4026     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4027 }
4028
4029 sub qmp_socket {
4030     my ($vmid, $qga, $name) = @_;
4031     my $sockettype = $qga ? 'qga' : 'qmp';
4032     my $ext = $name ? '-'.$name : '';
4033     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4034 }
4035
4036 sub pidfile_name {
4037     my ($vmid) = @_;
4038     return "${var_run_tmpdir}/$vmid.pid";
4039 }
4040
4041 sub vm_devices_list {
4042     my ($vmid) = @_;
4043
4044     my $res = vm_mon_cmd($vmid, 'query-pci');
4045     my $devices_to_check = [];
4046     my $devices = {};
4047     foreach my $pcibus (@$res) {
4048         push @$devices_to_check, @{$pcibus->{devices}},
4049     }
4050
4051     while (@$devices_to_check) {
4052         my $to_check = [];
4053         for my $d (@$devices_to_check) {
4054             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4055             next if !$d->{'pci_bridge'};
4056
4057             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4058             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4059         }
4060         $devices_to_check = $to_check;
4061     }
4062
4063     my $resblock = vm_mon_cmd($vmid, 'query-block');
4064     foreach my $block (@$resblock) {
4065         if($block->{device} =~ m/^drive-(\S+)/){
4066                 $devices->{$1} = 1;
4067         }
4068     }
4069
4070     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4071     foreach my $mice (@$resmice) {
4072         if ($mice->{name} eq 'QEMU HID Tablet') {
4073             $devices->{tablet} = 1;
4074             last;
4075         }
4076     }
4077
4078     # for usb devices there is no query-usb
4079     # but we can iterate over the entries in
4080     # qom-list path=/machine/peripheral
4081     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4082     foreach my $per (@$resperipheral) {
4083         if ($per->{name} =~ m/^usb\d+$/) {
4084             $devices->{$per->{name}} = 1;
4085         }
4086     }
4087
4088     return $devices;
4089 }
4090
4091 sub vm_deviceplug {
4092     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4093
4094     my $q35 = machine_type_is_q35($conf);
4095
4096     my $devices_list = vm_devices_list($vmid);
4097     return 1 if defined($devices_list->{$deviceid});
4098
4099     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4100
4101     if ($deviceid eq 'tablet') {
4102
4103         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4104
4105     } elsif ($deviceid eq 'keyboard') {
4106
4107         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4108
4109     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4110
4111         die "usb hotplug currently not reliable\n";
4112         # since we can't reliably hot unplug all added usb devices
4113         # and usb passthrough disables live migration
4114         # we disable usb hotplugging for now
4115         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4116
4117     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4118
4119         qemu_iothread_add($vmid, $deviceid, $device);
4120
4121         qemu_driveadd($storecfg, $vmid, $device);
4122         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4123
4124         qemu_deviceadd($vmid, $devicefull);
4125         eval { qemu_deviceaddverify($vmid, $deviceid); };
4126         if (my $err = $@) {
4127             eval { qemu_drivedel($vmid, $deviceid); };
4128             warn $@ if $@;
4129             die $err;
4130         }
4131
4132     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4133
4134
4135         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4136         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4137         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4138
4139         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4140
4141         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4142             qemu_iothread_add($vmid, $deviceid, $device);
4143             $devicefull .= ",iothread=iothread-$deviceid";
4144         }
4145
4146         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4147             $devicefull .= ",num_queues=$device->{queues}";
4148         }
4149
4150         qemu_deviceadd($vmid, $devicefull);
4151         qemu_deviceaddverify($vmid, $deviceid);
4152
4153     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4154
4155         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4156         qemu_driveadd($storecfg, $vmid, $device);
4157
4158         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4159         eval { qemu_deviceadd($vmid, $devicefull); };
4160         if (my $err = $@) {
4161             eval { qemu_drivedel($vmid, $deviceid); };
4162             warn $@ if $@;
4163             die $err;
4164         }
4165
4166     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4167
4168         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4169
4170         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4171         my $use_old_bios_files = undef;
4172         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4173
4174         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4175         qemu_deviceadd($vmid, $netdevicefull);
4176         eval {
4177             qemu_deviceaddverify($vmid, $deviceid);
4178             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4179         };
4180         if (my $err = $@) {
4181             eval { qemu_netdevdel($vmid, $deviceid); };
4182             warn $@ if $@;
4183             die $err;
4184         }
4185
4186     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4187
4188         my $bridgeid = $2;
4189         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4190         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4191
4192         qemu_deviceadd($vmid, $devicefull);
4193         qemu_deviceaddverify($vmid, $deviceid);
4194
4195     } else {
4196         die "can't hotplug device '$deviceid'\n";
4197     }
4198
4199     return 1;
4200 }
4201
4202 # fixme: this should raise exceptions on error!
4203 sub vm_deviceunplug {
4204     my ($vmid, $conf, $deviceid) = @_;
4205
4206     my $devices_list = vm_devices_list($vmid);
4207     return 1 if !defined($devices_list->{$deviceid});
4208
4209     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4210
4211     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4212
4213         qemu_devicedel($vmid, $deviceid);
4214
4215     } elsif ($deviceid =~ m/^usb\d+$/) {
4216
4217         die "usb hotplug currently not reliable\n";
4218         # when unplugging usb devices this way,
4219         # there may be remaining usb controllers/hubs
4220         # so we disable it for now
4221         qemu_devicedel($vmid, $deviceid);
4222         qemu_devicedelverify($vmid, $deviceid);
4223
4224     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4225
4226         qemu_devicedel($vmid, $deviceid);
4227         qemu_devicedelverify($vmid, $deviceid);
4228         qemu_drivedel($vmid, $deviceid);
4229         qemu_iothread_del($conf, $vmid, $deviceid);
4230
4231     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4232
4233         qemu_devicedel($vmid, $deviceid);
4234         qemu_devicedelverify($vmid, $deviceid);
4235         qemu_iothread_del($conf, $vmid, $deviceid);
4236
4237     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240         qemu_drivedel($vmid, $deviceid);
4241         qemu_deletescsihw($conf, $vmid, $deviceid);
4242
4243     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4244
4245         qemu_devicedel($vmid, $deviceid);
4246         qemu_devicedelverify($vmid, $deviceid);
4247         qemu_netdevdel($vmid, $deviceid);
4248
4249     } else {
4250         die "can't unplug device '$deviceid'\n";
4251     }
4252
4253     return 1;
4254 }
4255
4256 sub qemu_deviceadd {
4257     my ($vmid, $devicefull) = @_;
4258
4259     $devicefull = "driver=".$devicefull;
4260     my %options =  split(/[=,]/, $devicefull);
4261
4262     vm_mon_cmd($vmid, "device_add" , %options);
4263 }
4264
4265 sub qemu_devicedel {
4266     my ($vmid, $deviceid) = @_;
4267
4268     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4269 }
4270
4271 sub qemu_iothread_add {
4272     my($vmid, $deviceid, $device) = @_;
4273
4274     if ($device->{iothread}) {
4275         my $iothreads = vm_iothreads_list($vmid);
4276         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4277     }
4278 }
4279
4280 sub qemu_iothread_del {
4281     my($conf, $vmid, $deviceid) = @_;
4282
4283     my $confid = $deviceid;
4284     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4285         $confid = 'scsi' . $1;
4286     }
4287     my $device = parse_drive($confid, $conf->{$confid});
4288     if ($device->{iothread}) {
4289         my $iothreads = vm_iothreads_list($vmid);
4290         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4291     }
4292 }
4293
4294 sub qemu_objectadd {
4295     my($vmid, $objectid, $qomtype) = @_;
4296
4297     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4298
4299     return 1;
4300 }
4301
4302 sub qemu_objectdel {
4303     my($vmid, $objectid) = @_;
4304
4305     vm_mon_cmd($vmid, "object-del", id => $objectid);
4306
4307     return 1;
4308 }
4309
4310 sub qemu_driveadd {
4311     my ($storecfg, $vmid, $device) = @_;
4312
4313     my $drive = print_drive_full($storecfg, $vmid, $device);
4314     $drive =~ s/\\/\\\\/g;
4315     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4316
4317     # If the command succeeds qemu prints: "OK"
4318     return 1 if $ret =~ m/OK/s;
4319
4320     die "adding drive failed: $ret\n";
4321 }
4322
4323 sub qemu_drivedel {
4324     my($vmid, $deviceid) = @_;
4325
4326     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4327     $ret =~ s/^\s+//;
4328
4329     return 1 if $ret eq "";
4330
4331     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4332     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4333
4334     die "deleting drive $deviceid failed : $ret\n";
4335 }
4336
4337 sub qemu_deviceaddverify {
4338     my ($vmid, $deviceid) = @_;
4339
4340     for (my $i = 0; $i <= 5; $i++) {
4341          my $devices_list = vm_devices_list($vmid);
4342          return 1 if defined($devices_list->{$deviceid});
4343          sleep 1;
4344     }
4345
4346     die "error on hotplug device '$deviceid'\n";
4347 }
4348
4349
4350 sub qemu_devicedelverify {
4351     my ($vmid, $deviceid) = @_;
4352
4353     # need to verify that the device is correctly removed as device_del
4354     # is async and empty return is not reliable
4355
4356     for (my $i = 0; $i <= 5; $i++) {
4357          my $devices_list = vm_devices_list($vmid);
4358          return 1 if !defined($devices_list->{$deviceid});
4359          sleep 1;
4360     }
4361
4362     die "error on hot-unplugging device '$deviceid'\n";
4363 }
4364
4365 sub qemu_findorcreatescsihw {
4366     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4367
4368     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4369
4370     my $scsihwid="$controller_prefix$controller";
4371     my $devices_list = vm_devices_list($vmid);
4372
4373     if(!defined($devices_list->{$scsihwid})) {
4374         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4375     }
4376
4377     return 1;
4378 }
4379
4380 sub qemu_deletescsihw {
4381     my ($conf, $vmid, $opt) = @_;
4382
4383     my $device = parse_drive($opt, $conf->{$opt});
4384
4385     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4386         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4387         return 1;
4388     }
4389
4390     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4391
4392     my $devices_list = vm_devices_list($vmid);
4393     foreach my $opt (keys %{$devices_list}) {
4394         if (PVE::QemuServer::is_valid_drivename($opt)) {
4395             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4396             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4397                 return 1;
4398             }
4399         }
4400     }
4401
4402     my $scsihwid="scsihw$controller";
4403
4404     vm_deviceunplug($vmid, $conf, $scsihwid);
4405
4406     return 1;
4407 }
4408
4409 sub qemu_add_pci_bridge {
4410     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4411
4412     my $bridges = {};
4413
4414     my $bridgeid;
4415
4416     print_pci_addr($device, $bridges, $arch, $machine_type);
4417
4418     while (my ($k, $v) = each %$bridges) {
4419         $bridgeid = $k;
4420     }
4421     return 1 if !defined($bridgeid) || $bridgeid < 1;
4422
4423     my $bridge = "pci.$bridgeid";
4424     my $devices_list = vm_devices_list($vmid);
4425
4426     if (!defined($devices_list->{$bridge})) {
4427         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4428     }
4429
4430     return 1;
4431 }
4432
4433 sub qemu_set_link_status {
4434     my ($vmid, $device, $up) = @_;
4435
4436     vm_mon_cmd($vmid, "set_link", name => $device,
4437                up => $up ? JSON::true : JSON::false);
4438 }
4439
4440 sub qemu_netdevadd {
4441     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4442
4443     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4444     my %options =  split(/[=,]/, $netdev);
4445
4446     vm_mon_cmd($vmid, "netdev_add",  %options);
4447     return 1;
4448 }
4449
4450 sub qemu_netdevdel {
4451     my ($vmid, $deviceid) = @_;
4452
4453     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4454 }
4455
4456 sub qemu_usb_hotplug {
4457     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4458
4459     return if !$device;
4460
4461     # remove the old one first
4462     vm_deviceunplug($vmid, $conf, $deviceid);
4463
4464     # check if xhci controller is necessary and available
4465     if ($device->{usb3}) {
4466
4467         my $devicelist = vm_devices_list($vmid);
4468
4469         if (!$devicelist->{xhci}) {
4470             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4471             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4472         }
4473     }
4474     my $d = parse_usb_device($device->{host});
4475     $d->{usb3} = $device->{usb3};
4476
4477     # add the new one
4478     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4479 }
4480
4481 sub qemu_cpu_hotplug {
4482     my ($vmid, $conf, $vcpus) = @_;
4483
4484     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4485
4486     my $sockets = 1;
4487     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4488     $sockets = $conf->{sockets} if  $conf->{sockets};
4489     my $cores = $conf->{cores} || 1;
4490     my $maxcpus = $sockets * $cores;
4491
4492     $vcpus = $maxcpus if !$vcpus;
4493
4494     die "you can't add more vcpus than maxcpus\n"
4495         if $vcpus > $maxcpus;
4496
4497     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4498
4499     if ($vcpus < $currentvcpus) {
4500
4501         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4502
4503             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4504                 qemu_devicedel($vmid, "cpu$i");
4505                 my $retry = 0;
4506                 my $currentrunningvcpus = undef;
4507                 while (1) {
4508                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4509                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4510                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4511                     $retry++;
4512                     sleep 1;
4513                 }
4514                 #update conf after each succesfull cpu unplug
4515                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4516                 PVE::QemuConfig->write_config($vmid, $conf);
4517             }
4518         } else {
4519             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4520         }
4521
4522         return;
4523     }
4524
4525     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4526     die "vcpus in running vm does not match its configuration\n"
4527         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4528
4529     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4530
4531         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4532             my $cpustr = print_cpu_device($conf, $i);
4533             qemu_deviceadd($vmid, $cpustr);
4534
4535             my $retry = 0;
4536             my $currentrunningvcpus = undef;
4537             while (1) {
4538                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4539                 last if scalar(@{$currentrunningvcpus}) == $i;
4540                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4541                 sleep 1;
4542                 $retry++;
4543             }
4544             #update conf after each succesfull cpu hotplug
4545             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4546             PVE::QemuConfig->write_config($vmid, $conf);
4547         }
4548     } else {
4549
4550         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4551             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4552         }
4553     }
4554 }
4555
4556 sub qemu_block_set_io_throttle {
4557     my ($vmid, $deviceid,
4558         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4559         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4560         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4561         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4562
4563     return if !check_running($vmid) ;
4564
4565     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4566         bps => int($bps),
4567         bps_rd => int($bps_rd),
4568         bps_wr => int($bps_wr),
4569         iops => int($iops),
4570         iops_rd => int($iops_rd),
4571         iops_wr => int($iops_wr),
4572         bps_max => int($bps_max),
4573         bps_rd_max => int($bps_rd_max),
4574         bps_wr_max => int($bps_wr_max),
4575         iops_max => int($iops_max),
4576         iops_rd_max => int($iops_rd_max),
4577         iops_wr_max => int($iops_wr_max),
4578         bps_max_length => int($bps_max_length),
4579         bps_rd_max_length => int($bps_rd_max_length),
4580         bps_wr_max_length => int($bps_wr_max_length),
4581         iops_max_length => int($iops_max_length),
4582         iops_rd_max_length => int($iops_rd_max_length),
4583         iops_wr_max_length => int($iops_wr_max_length),
4584     );
4585
4586 }
4587
4588 # old code, only used to shutdown old VM after update
4589 sub __read_avail {
4590     my ($fh, $timeout) = @_;
4591
4592     my $sel = new IO::Select;
4593     $sel->add($fh);
4594
4595     my $res = '';
4596     my $buf;
4597
4598     my @ready;
4599     while (scalar (@ready = $sel->can_read($timeout))) {
4600         my $count;
4601         if ($count = $fh->sysread($buf, 8192)) {
4602             if ($buf =~ /^(.*)\(qemu\) $/s) {
4603                 $res .= $1;
4604                 last;
4605             } else {
4606                 $res .= $buf;
4607             }
4608         } else {
4609             if (!defined($count)) {
4610                 die "$!\n";
4611             }
4612             last;
4613         }
4614     }
4615
4616     die "monitor read timeout\n" if !scalar(@ready);
4617
4618     return $res;
4619 }
4620
4621 sub qemu_block_resize {
4622     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4623
4624     my $running = check_running($vmid);
4625
4626     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4627
4628     return if !$running;
4629
4630     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4631
4632 }
4633
4634 sub qemu_volume_snapshot {
4635     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4636
4637     my $running = check_running($vmid);
4638
4639     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4640         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4641     } else {
4642         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4643     }
4644 }
4645
4646 sub qemu_volume_snapshot_delete {
4647     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4648
4649     my $running = check_running($vmid);
4650
4651     if($running) {
4652
4653         $running = undef;
4654         my $conf = PVE::QemuConfig->load_config($vmid);
4655         foreach_drive($conf, sub {
4656             my ($ds, $drive) = @_;
4657             $running = 1 if $drive->{file} eq $volid;
4658         });
4659     }
4660
4661     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4662         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4663     } else {
4664         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4665     }
4666 }
4667
4668 sub set_migration_caps {
4669     my ($vmid) = @_;
4670
4671     my $cap_ref = [];
4672
4673     my $enabled_cap = {
4674         "auto-converge" => 1,
4675         "xbzrle" => 1,
4676         "x-rdma-pin-all" => 0,
4677         "zero-blocks" => 0,
4678         "compress" => 0
4679     };
4680
4681     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4682
4683     for my $supported_capability (@$supported_capabilities) {
4684         push @$cap_ref, {
4685             capability => $supported_capability->{capability},
4686             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4687         };
4688     }
4689
4690     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4691 }
4692
4693 my $fast_plug_option = {
4694     'lock' => 1,
4695     'name' => 1,
4696     'onboot' => 1,
4697     'shares' => 1,
4698     'startup' => 1,
4699     'description' => 1,
4700     'protection' => 1,
4701     'vmstatestorage' => 1,
4702     'hookscript' => 1,
4703 };
4704
4705 # hotplug changes in [PENDING]
4706 # $selection hash can be used to only apply specified options, for
4707 # example: { cores => 1 } (only apply changed 'cores')
4708 # $errors ref is used to return error messages
4709 sub vmconfig_hotplug_pending {
4710     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4711
4712     my $defaults = load_defaults();
4713     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4714
4715     # commit values which do not have any impact on running VM first
4716     # Note: those option cannot raise errors, we we do not care about
4717     # $selection and always apply them.
4718
4719     my $add_error = sub {
4720         my ($opt, $msg) = @_;
4721         $errors->{$opt} = "hotplug problem - $msg";
4722     };
4723
4724     my $changes = 0;
4725     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4726         if ($fast_plug_option->{$opt}) {
4727             $conf->{$opt} = $conf->{pending}->{$opt};
4728             delete $conf->{pending}->{$opt};
4729             $changes = 1;
4730         }
4731     }
4732
4733     if ($changes) {
4734         PVE::QemuConfig->write_config($vmid, $conf);
4735         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4736     }
4737
4738     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4739
4740     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4741     while (my ($opt, $force) = each %$pending_delete_hash) {
4742         next if $selection && !$selection->{$opt};
4743         eval {
4744             if ($opt eq 'hotplug') {
4745                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4746             } elsif ($opt eq 'tablet') {
4747                 die "skip\n" if !$hotplug_features->{usb};
4748                 if ($defaults->{tablet}) {
4749                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4750                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4751                         if $arch eq 'aarch64';
4752                 } else {
4753                     vm_deviceunplug($vmid, $conf, 'tablet');
4754                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4755                 }
4756             } elsif ($opt =~ m/^usb\d+/) {
4757                 die "skip\n";
4758                 # since we cannot reliably hot unplug usb devices
4759                 # we are disabling it
4760                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4761                 vm_deviceunplug($vmid, $conf, $opt);
4762             } elsif ($opt eq 'vcpus') {
4763                 die "skip\n" if !$hotplug_features->{cpu};
4764                 qemu_cpu_hotplug($vmid, $conf, undef);
4765             } elsif ($opt eq 'balloon') {
4766                 # enable balloon device is not hotpluggable
4767                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4768                 # here we reset the ballooning value to memory
4769                 my $balloon = $conf->{memory} || $defaults->{memory};
4770                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4771             } elsif ($fast_plug_option->{$opt}) {
4772                 # do nothing
4773             } elsif ($opt =~ m/^net(\d+)$/) {
4774                 die "skip\n" if !$hotplug_features->{network};
4775                 vm_deviceunplug($vmid, $conf, $opt);
4776             } elsif (is_valid_drivename($opt)) {
4777                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4778                 vm_deviceunplug($vmid, $conf, $opt);
4779                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4780             } elsif ($opt =~ m/^memory$/) {
4781                 die "skip\n" if !$hotplug_features->{memory};
4782                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4783             } elsif ($opt eq 'cpuunits') {
4784                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4785             } elsif ($opt eq 'cpulimit') {
4786                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4787             } else {
4788                 die "skip\n";
4789             }
4790         };
4791         if (my $err = $@) {
4792             &$add_error($opt, $err) if $err ne "skip\n";
4793         } else {
4794             # save new config if hotplug was successful
4795             delete $conf->{$opt};
4796             vmconfig_undelete_pending_option($conf, $opt);
4797             PVE::QemuConfig->write_config($vmid, $conf);
4798             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4799         }
4800     }
4801
4802     my $apply_pending_cloudinit;
4803     $apply_pending_cloudinit = sub {
4804         my ($key, $value) = @_;
4805         $apply_pending_cloudinit = sub {}; # once is enough
4806
4807         my @cloudinit_opts = keys %$confdesc_cloudinit;
4808         foreach my $opt (keys %{$conf->{pending}}) {
4809             next if !grep { $_ eq $opt } @cloudinit_opts;
4810             $conf->{$opt} = delete $conf->{pending}->{$opt};
4811         }
4812
4813         my $new_conf = { %$conf };
4814         $new_conf->{$key} = $value;
4815         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4816     };
4817
4818     foreach my $opt (keys %{$conf->{pending}}) {
4819         next if $selection && !$selection->{$opt};
4820         my $value = $conf->{pending}->{$opt};
4821         eval {
4822             if ($opt eq 'hotplug') {
4823                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4824             } elsif ($opt eq 'tablet') {
4825                 die "skip\n" if !$hotplug_features->{usb};
4826                 if ($value == 1) {
4827                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4828                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4829                         if $arch eq 'aarch64';
4830                 } elsif ($value == 0) {
4831                     vm_deviceunplug($vmid, $conf, 'tablet');
4832                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4833                 }
4834             } elsif ($opt =~ m/^usb\d+$/) {
4835                 die "skip\n";
4836                 # since we cannot reliably hot unplug usb devices
4837                 # we are disabling it
4838                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4839                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4840                 die "skip\n" if !$d;
4841                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4842             } elsif ($opt eq 'vcpus') {
4843                 die "skip\n" if !$hotplug_features->{cpu};
4844                 qemu_cpu_hotplug($vmid, $conf, $value);
4845             } elsif ($opt eq 'balloon') {
4846                 # enable/disable balloning device is not hotpluggable
4847                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4848                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4849                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4850
4851                 # allow manual ballooning if shares is set to zero
4852                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4853                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4854                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4855                 }
4856             } elsif ($opt =~ m/^net(\d+)$/) {
4857                 # some changes can be done without hotplug
4858                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4859                                     $vmid, $opt, $value, $arch, $machine_type);
4860             } elsif (is_valid_drivename($opt)) {
4861                 # some changes can be done without hotplug
4862                 my $drive = parse_drive($opt, $value);
4863                 if (drive_is_cloudinit($drive)) {
4864                     &$apply_pending_cloudinit($opt, $value);
4865                 }
4866                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4867                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4868             } elsif ($opt =~ m/^memory$/) { #dimms
4869                 die "skip\n" if !$hotplug_features->{memory};
4870                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4871             } elsif ($opt eq 'cpuunits') {
4872                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4873             } elsif ($opt eq 'cpulimit') {
4874                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4875                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4876             } else {
4877                 die "skip\n";  # skip non-hot-pluggable options
4878             }
4879         };
4880         if (my $err = $@) {
4881             &$add_error($opt, $err) if $err ne "skip\n";
4882         } else {
4883             # save new config if hotplug was successful
4884             $conf->{$opt} = $value;
4885             delete $conf->{pending}->{$opt};
4886             PVE::QemuConfig->write_config($vmid, $conf);
4887             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4888         }
4889     }
4890 }
4891
4892 sub try_deallocate_drive {
4893     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4894
4895     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4896         my $volid = $drive->{file};
4897         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4898             my $sid = PVE::Storage::parse_volume_id($volid);
4899             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4900
4901             # check if the disk is really unused
4902             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4903                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4904             PVE::Storage::vdisk_free($storecfg, $volid);
4905             return 1;
4906         } else {
4907             # If vm is not owner of this disk remove from config
4908             return 1;
4909         }
4910     }
4911
4912     return undef;
4913 }
4914
4915 sub vmconfig_delete_or_detach_drive {
4916     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4917
4918     my $drive = parse_drive($opt, $conf->{$opt});
4919
4920     my $rpcenv = PVE::RPCEnvironment::get();
4921     my $authuser = $rpcenv->get_user();
4922
4923     if ($force) {
4924         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4925         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4926     } else {
4927         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4928     }
4929 }
4930
4931 sub vmconfig_apply_pending {
4932     my ($vmid, $conf, $storecfg) = @_;
4933
4934     # cold plug
4935
4936     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4937     while (my ($opt, $force) = each %$pending_delete_hash) {
4938         die "internal error" if $opt =~ m/^unused/;
4939         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4940         if (!defined($conf->{$opt})) {
4941             vmconfig_undelete_pending_option($conf, $opt);
4942             PVE::QemuConfig->write_config($vmid, $conf);
4943         } elsif (is_valid_drivename($opt)) {
4944             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4945             vmconfig_undelete_pending_option($conf, $opt);
4946             delete $conf->{$opt};
4947             PVE::QemuConfig->write_config($vmid, $conf);
4948         } else {
4949             vmconfig_undelete_pending_option($conf, $opt);
4950             delete $conf->{$opt};
4951             PVE::QemuConfig->write_config($vmid, $conf);
4952         }
4953     }
4954
4955     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4956
4957     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4958         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4959
4960         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4961             # skip if nothing changed
4962         } elsif (is_valid_drivename($opt)) {
4963             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4964                 if defined($conf->{$opt});
4965             $conf->{$opt} = $conf->{pending}->{$opt};
4966         } else {
4967             $conf->{$opt} = $conf->{pending}->{$opt};
4968         }
4969
4970         delete $conf->{pending}->{$opt};
4971         PVE::QemuConfig->write_config($vmid, $conf);
4972     }
4973 }
4974
4975 my $safe_num_ne = sub {
4976     my ($a, $b) = @_;
4977
4978     return 0 if !defined($a) && !defined($b);
4979     return 1 if !defined($a);
4980     return 1 if !defined($b);
4981
4982     return $a != $b;
4983 };
4984
4985 my $safe_string_ne = sub {
4986     my ($a, $b) = @_;
4987
4988     return 0 if !defined($a) && !defined($b);
4989     return 1 if !defined($a);
4990     return 1 if !defined($b);
4991
4992     return $a ne $b;
4993 };
4994
4995 sub vmconfig_update_net {
4996     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4997
4998     my $newnet = parse_net($value);
4999
5000     if ($conf->{$opt}) {
5001         my $oldnet = parse_net($conf->{$opt});
5002
5003         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5004             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5005             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5006             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5007
5008             # for non online change, we try to hot-unplug
5009             die "skip\n" if !$hotplug;
5010             vm_deviceunplug($vmid, $conf, $opt);
5011         } else {
5012
5013             die "internal error" if $opt !~ m/net(\d+)/;
5014             my $iface = "tap${vmid}i$1";
5015
5016             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5017                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5018                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5019                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5020                 PVE::Network::tap_unplug($iface);
5021                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5022             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5023                 # Rate can be applied on its own but any change above needs to
5024                 # include the rate in tap_plug since OVS resets everything.
5025                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5026             }
5027
5028             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5029                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5030             }
5031
5032             return 1;
5033         }
5034     }
5035
5036     if ($hotplug) {
5037         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5038     } else {
5039         die "skip\n";
5040     }
5041 }
5042
5043 sub vmconfig_update_disk {
5044     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5045
5046     # fixme: do we need force?
5047
5048     my $drive = parse_drive($opt, $value);
5049
5050     if ($conf->{$opt}) {
5051
5052         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5053
5054             my $media = $drive->{media} || 'disk';
5055             my $oldmedia = $old_drive->{media} || 'disk';
5056             die "unable to change media type\n" if $media ne $oldmedia;
5057
5058             if (!drive_is_cdrom($old_drive)) {
5059
5060                 if ($drive->{file} ne $old_drive->{file}) {
5061
5062                     die "skip\n" if !$hotplug;
5063
5064                     # unplug and register as unused
5065                     vm_deviceunplug($vmid, $conf, $opt);
5066                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5067
5068                 } else {
5069                     # update existing disk
5070
5071                     # skip non hotpluggable value
5072                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5073                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5074                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5075                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5076                         die "skip\n";
5077                     }
5078
5079                     # apply throttle
5080                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5081                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5082                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5083                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5084                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5085                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5086                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5087                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5088                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5089                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5090                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5091                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5092                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5093                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5094                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5095                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5096                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5097                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5098
5099                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5100                                                    ($drive->{mbps} || 0)*1024*1024,
5101                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5102                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5103                                                    $drive->{iops} || 0,
5104                                                    $drive->{iops_rd} || 0,
5105                                                    $drive->{iops_wr} || 0,
5106                                                    ($drive->{mbps_max} || 0)*1024*1024,
5107                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5108                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5109                                                    $drive->{iops_max} || 0,
5110                                                    $drive->{iops_rd_max} || 0,
5111                                                    $drive->{iops_wr_max} || 0,
5112                                                    $drive->{bps_max_length} || 1,
5113                                                    $drive->{bps_rd_max_length} || 1,
5114                                                    $drive->{bps_wr_max_length} || 1,
5115                                                    $drive->{iops_max_length} || 1,
5116                                                    $drive->{iops_rd_max_length} || 1,
5117                                                    $drive->{iops_wr_max_length} || 1);
5118
5119                     }
5120
5121                     return 1;
5122                 }
5123
5124             } else { # cdrom
5125
5126                 if ($drive->{file} eq 'none') {
5127                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5128                     if (drive_is_cloudinit($old_drive)) {
5129                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5130                     }
5131                 } else {
5132                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5133                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5134                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5135                 }
5136
5137                 return 1;
5138             }
5139         }
5140     }
5141
5142     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5143     # hotplug new disks
5144     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5145     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5146 }
5147
5148 sub vm_start {
5149     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5150         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5151
5152     PVE::QemuConfig->lock_config($vmid, sub {
5153         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5154
5155         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5156
5157         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5158
5159         PVE::QemuConfig->check_lock($conf)
5160             if !($skiplock || $is_suspended);
5161
5162         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5163
5164         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5165             vmconfig_apply_pending($vmid, $conf, $storecfg);
5166             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5167         }
5168
5169         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5170
5171         my $defaults = load_defaults();
5172
5173         # set environment variable useful inside network script
5174         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5175
5176         my $local_volumes = {};
5177
5178         if ($targetstorage) {
5179             foreach_drive($conf, sub {
5180                 my ($ds, $drive) = @_;
5181
5182                 return if drive_is_cdrom($drive);
5183
5184                 my $volid = $drive->{file};
5185
5186                 return if !$volid;
5187
5188                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5189
5190                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5191                 return if $scfg->{shared};
5192                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5193             });
5194
5195             my $format = undef;
5196
5197             foreach my $opt (sort keys %$local_volumes) {
5198
5199                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5200                 my $drive = parse_drive($opt, $conf->{$opt});
5201
5202                 #if remote storage is specified, use default format
5203                 if ($targetstorage && $targetstorage ne "1") {
5204                     $storeid = $targetstorage;
5205                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5206                     $format = $defFormat;
5207                 } else {
5208                     #else we use same format than original
5209                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5210                     $format = qemu_img_format($scfg, $volid);
5211                 }
5212
5213                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5214                 my $newdrive = $drive;
5215                 $newdrive->{format} = $format;
5216                 $newdrive->{file} = $newvolid;
5217                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5218                 $local_volumes->{$opt} = $drivestr;
5219                 #pass drive to conf for command line
5220                 $conf->{$opt} = $drivestr;
5221             }
5222         }
5223
5224         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5225
5226         if ($is_suspended) {
5227             # enforce machine type on suspended vm to ensure HW compatibility
5228             $forcemachine = $conf->{runningmachine};
5229         }
5230
5231         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5232
5233         my $migrate_port = 0;
5234         my $migrate_uri;
5235         if ($statefile) {
5236             if ($statefile eq 'tcp') {
5237                 my $localip = "localhost";
5238                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5239                 my $nodename = PVE::INotify::nodename();
5240
5241                 if (!defined($migration_type)) {
5242                     if (defined($datacenterconf->{migration}->{type})) {
5243                         $migration_type = $datacenterconf->{migration}->{type};
5244                     } else {
5245                         $migration_type = 'secure';
5246                     }
5247                 }
5248
5249                 if ($migration_type eq 'insecure') {
5250                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5251                     if ($migrate_network_addr) {
5252                         $localip = $migrate_network_addr;
5253                     } else {
5254                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5255                     }
5256
5257                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5258                 }
5259
5260                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5261                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5262                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5263                 push @$cmd, '-incoming', $migrate_uri;
5264                 push @$cmd, '-S';
5265
5266             } elsif ($statefile eq 'unix') {
5267                 # should be default for secure migrations as a ssh TCP forward
5268                 # tunnel is not deterministic reliable ready and fails regurarly
5269                 # to set up in time, so use UNIX socket forwards
5270                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5271                 unlink $socket_addr;
5272
5273                 $migrate_uri = "unix:$socket_addr";
5274
5275                 push @$cmd, '-incoming', $migrate_uri;
5276                 push @$cmd, '-S';
5277
5278             } else {
5279                 push @$cmd, '-loadstate', $statefile;
5280             }
5281         } elsif ($paused) {
5282             push @$cmd, '-S';
5283         }
5284
5285         # host pci devices
5286         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5287           my $d = parse_hostpci($conf->{"hostpci$i"});
5288           next if !$d;
5289           my $pcidevices = $d->{pciid};
5290           foreach my $pcidevice (@$pcidevices) {
5291                 my $pciid = $pcidevice->{id};
5292
5293                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5294                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5295                 die "no pci device info for device '$pciid'\n" if !$info;
5296
5297                 if ($d->{mdev}) {
5298                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5299                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5300                 } else {
5301                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5302                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5303                     die "can't reset pci device '$pciid'\n"
5304                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5305                 }
5306           }
5307         }
5308
5309         PVE::Storage::activate_volumes($storecfg, $vollist);
5310
5311         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5312             eval {
5313                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5314                     outfunc => sub {}, errfunc => sub {});
5315             };
5316         }
5317
5318         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5319                                                   : $defaults->{cpuunits};
5320
5321         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5322         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5323
5324         my %properties = (
5325             Slice => 'qemu.slice',
5326             KillMode => 'none',
5327             CPUShares => $cpuunits
5328         );
5329
5330         if (my $cpulimit = $conf->{cpulimit}) {
5331             $properties{CPUQuota} = int($cpulimit * 100);
5332         }
5333         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5334
5335         my $run_qemu = sub {
5336             PVE::Tools::run_fork sub {
5337                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5338                 run_command($cmd, %run_params);
5339             };
5340         };
5341
5342         if ($conf->{hugepages}) {
5343
5344             my $code = sub {
5345                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5346                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5347
5348                 PVE::QemuServer::Memory::hugepages_mount();
5349                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5350
5351                 eval { $run_qemu->() };
5352                 if (my $err = $@) {
5353                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5354                     die $err;
5355                 }
5356
5357                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5358             };
5359             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5360
5361         } else {
5362             eval { $run_qemu->() };
5363         }
5364
5365         if (my $err = $@) {
5366             # deactivate volumes if start fails
5367             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5368             die "start failed: $err";
5369         }
5370
5371         print "migration listens on $migrate_uri\n" if $migrate_uri;
5372
5373         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5374             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5375             warn $@ if $@;
5376         }
5377
5378         #start nbd server for storage migration
5379         if ($targetstorage) {
5380             my $nodename = PVE::INotify::nodename();
5381             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5382             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5383             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5384             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5385
5386             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5387
5388             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5389
5390             foreach my $opt (sort keys %$local_volumes) {
5391                 my $volid = $local_volumes->{$opt};
5392                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5393                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5394                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5395             }
5396         }
5397
5398         if ($migratedfrom) {
5399             eval {
5400                 set_migration_caps($vmid);
5401             };
5402             warn $@ if $@;
5403
5404             if ($spice_port) {
5405                 print "spice listens on port $spice_port\n";
5406                 if ($spice_ticket) {
5407                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5408                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5409                 }
5410             }
5411
5412         } else {
5413             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5414                 if !$statefile && $conf->{balloon};
5415
5416             foreach my $opt (keys %$conf) {
5417                 next if $opt !~  m/^net\d+$/;
5418                 my $nicconf = parse_net($conf->{$opt});
5419                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5420             }
5421         }
5422
5423         vm_mon_cmd_nocheck($vmid, 'qom-set',
5424                     path => "machine/peripheral/balloon0",
5425                     property => "guest-stats-polling-interval",
5426                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5427
5428         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5429             delete $conf->@{qw(lock vmstate runningmachine)};
5430             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5431             PVE::Storage::vdisk_free($storecfg, $vmstate);
5432             PVE::QemuConfig->write_config($vmid, $conf);
5433         }
5434
5435         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5436     });
5437 }
5438
5439 sub vm_mon_cmd {
5440     my ($vmid, $execute, %params) = @_;
5441
5442     my $cmd = { execute => $execute, arguments => \%params };
5443     vm_qmp_command($vmid, $cmd);
5444 }
5445
5446 sub vm_mon_cmd_nocheck {
5447     my ($vmid, $execute, %params) = @_;
5448
5449     my $cmd = { execute => $execute, arguments => \%params };
5450     vm_qmp_command($vmid, $cmd, 1);
5451 }
5452
5453 sub vm_qmp_command {
5454     my ($vmid, $cmd, $nocheck) = @_;
5455
5456     my $res;
5457
5458     my $timeout;
5459     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5460         $timeout = $cmd->{arguments}->{timeout};
5461         delete $cmd->{arguments}->{timeout};
5462     }
5463
5464     eval {
5465         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5466         my $sname = qmp_socket($vmid);
5467         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5468             my $qmpclient = PVE::QMPClient->new();
5469
5470             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5471         } else {
5472             die "unable to open monitor socket\n";
5473         }
5474     };
5475     if (my $err = $@) {
5476         syslog("err", "VM $vmid qmp command failed - $err");
5477         die $err;
5478     }
5479
5480     return $res;
5481 }
5482
5483 sub vm_human_monitor_command {
5484     my ($vmid, $cmdline) = @_;
5485
5486     my $res;
5487
5488     my $cmd = {
5489         execute => 'human-monitor-command',
5490         arguments => { 'command-line' => $cmdline},
5491     };
5492
5493     return vm_qmp_command($vmid, $cmd);
5494 }
5495
5496 sub vm_commandline {
5497     my ($storecfg, $vmid, $snapname) = @_;
5498
5499     my $conf = PVE::QemuConfig->load_config($vmid);
5500
5501     if ($snapname) {
5502         my $snapshot = $conf->{snapshots}->{$snapname};
5503         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5504
5505         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5506
5507         $conf = $snapshot;
5508     }
5509
5510     my $defaults = load_defaults();
5511
5512     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5513
5514     return PVE::Tools::cmd2string($cmd);
5515 }
5516
5517 sub vm_reset {
5518     my ($vmid, $skiplock) = @_;
5519
5520     PVE::QemuConfig->lock_config($vmid, sub {
5521
5522         my $conf = PVE::QemuConfig->load_config($vmid);
5523
5524         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5525
5526         vm_mon_cmd($vmid, "system_reset");
5527     });
5528 }
5529
5530 sub get_vm_volumes {
5531     my ($conf) = @_;
5532
5533     my $vollist = [];
5534     foreach_volid($conf, sub {
5535         my ($volid, $attr) = @_;
5536
5537         return if $volid =~ m|^/|;
5538
5539         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5540         return if !$sid;
5541
5542         push @$vollist, $volid;
5543     });
5544
5545     return $vollist;
5546 }
5547
5548 sub vm_stop_cleanup {
5549     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5550
5551     eval {
5552
5553         if (!$keepActive) {
5554             my $vollist = get_vm_volumes($conf);
5555             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5556         }
5557
5558         foreach my $ext (qw(mon qmp pid vnc qga)) {
5559             unlink "/var/run/qemu-server/${vmid}.$ext";
5560         }
5561
5562         if ($conf->{ivshmem}) {
5563             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5564             # just delete it for now, VMs which have this already open do not
5565             # are affected, but new VMs will get a separated one. If this
5566             # becomes an issue we either add some sort of ref-counting or just
5567             # add a "don't delete on stop" flag to the ivshmem format.
5568             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5569         }
5570
5571         foreach my $key (keys %$conf) {
5572             next if $key !~ m/^hostpci(\d+)$/;
5573             my $hostpciindex = $1;
5574             my $d = parse_hostpci($conf->{$key});
5575             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5576
5577             foreach my $pci (@{$d->{pciid}}) {
5578                 my $pciid = $pci->{id};
5579                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5580             }
5581         }
5582
5583         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5584     };
5585     warn $@ if $@; # avoid errors - just warn
5586 }
5587
5588 # Note: use $nockeck to skip tests if VM configuration file exists.
5589 # We need that when migration VMs to other nodes (files already moved)
5590 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5591 sub vm_stop {
5592     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5593
5594     $force = 1 if !defined($force) && !$shutdown;
5595
5596     if ($migratedfrom){
5597         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5598         kill 15, $pid if $pid;
5599         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5600         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5601         return;
5602     }
5603
5604     PVE::QemuConfig->lock_config($vmid, sub {
5605
5606         my $pid = check_running($vmid, $nocheck);
5607         return if !$pid;
5608
5609         my $conf;
5610         if (!$nocheck) {
5611             $conf = PVE::QemuConfig->load_config($vmid);
5612             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5613             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5614                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5615                 $timeout = $opts->{down} if $opts->{down};
5616             }
5617             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5618         }
5619
5620         $timeout = 60 if !defined($timeout);
5621
5622         eval {
5623             if ($shutdown) {
5624                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5625                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5626                 } else {
5627                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5628                 }
5629             } else {
5630                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5631             }
5632         };
5633         my $err = $@;
5634
5635         if (!$err) {
5636             my $count = 0;
5637             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5638                 $count++;
5639                 sleep 1;
5640             }
5641
5642             if ($count >= $timeout) {
5643                 if ($force) {
5644                     warn "VM still running - terminating now with SIGTERM\n";
5645                     kill 15, $pid;
5646                 } else {
5647                     die "VM quit/powerdown failed - got timeout\n";
5648                 }
5649             } else {
5650                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5651                 return;
5652             }
5653         } else {
5654             if ($force) {
5655                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5656                 kill 15, $pid;
5657             } else {
5658                 die "VM quit/powerdown failed\n";
5659             }
5660         }
5661
5662         # wait again
5663         $timeout = 10;
5664
5665         my $count = 0;
5666         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5667             $count++;
5668             sleep 1;
5669         }
5670
5671         if ($count >= $timeout) {
5672             warn "VM still running - terminating now with SIGKILL\n";
5673             kill 9, $pid;
5674             sleep 1;
5675         }
5676
5677         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5678    });
5679 }
5680
5681 sub vm_suspend {
5682     my ($vmid, $skiplock, $includestate) = @_;
5683
5684     my $conf;
5685     my $path;
5686     my $storecfg;
5687     my $vmstate;
5688
5689     PVE::QemuConfig->lock_config($vmid, sub {
5690
5691         $conf = PVE::QemuConfig->load_config($vmid);
5692
5693         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5694         PVE::QemuConfig->check_lock($conf)
5695             if !($skiplock || $is_backing_up);
5696
5697         die "cannot suspend to disk during backup\n"
5698             if $is_backing_up && $includestate;
5699
5700         if ($includestate) {
5701             $conf->{lock} = 'suspending';
5702             my $date = strftime("%Y-%m-%d", localtime(time()));
5703             $storecfg = PVE::Storage::config();
5704             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, 1);
5705             $path = PVE::Storage::path($storecfg, $vmstate);
5706             PVE::QemuConfig->write_config($vmid, $conf);
5707         } else {
5708             vm_mon_cmd($vmid, "stop");
5709         }
5710     });
5711
5712     if ($includestate) {
5713         # save vm state
5714         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5715
5716         eval {
5717             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5718             for(;;) {
5719                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5720                 if (!$state->{status}) {
5721                     die "savevm not active\n";
5722                 } elsif ($state->{status} eq 'active') {
5723                     sleep(1);
5724                     next;
5725                 } elsif ($state->{status} eq 'completed') {
5726                     last;
5727                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5728                     die "query-savevm failed with error '$state->{error}'\n"
5729                 } else {
5730                     die "query-savevm returned status '$state->{status}'\n";
5731                 }
5732             }
5733         };
5734         my $err = $@;
5735
5736         PVE::QemuConfig->lock_config($vmid, sub {
5737             $conf = PVE::QemuConfig->load_config($vmid);
5738             if ($err) {
5739                 # cleanup, but leave suspending lock, to indicate something went wrong
5740                 eval {
5741                     vm_mon_cmd($vmid, "savevm-end");
5742                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5743                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5744                     delete $conf->@{qw(vmstate runningmachine)};
5745                     PVE::QemuConfig->write_config($vmid, $conf);
5746                 };
5747                 warn $@ if $@;
5748                 die $err;
5749             }
5750
5751             die "lock changed unexpectedly\n"
5752                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5753
5754             vm_qmp_command($vmid, { execute => "quit" });
5755             $conf->{lock} = 'suspended';
5756             PVE::QemuConfig->write_config($vmid, $conf);
5757         });
5758     }
5759 }
5760
5761 sub vm_resume {
5762     my ($vmid, $skiplock, $nocheck) = @_;
5763
5764     PVE::QemuConfig->lock_config($vmid, sub {
5765
5766         my $res = vm_mon_cmd($vmid, 'query-status');
5767         my $resume_cmd = 'cont';
5768
5769         if ($res->{status} && $res->{status} eq 'suspended') {
5770             $resume_cmd = 'system_wakeup';
5771         }
5772
5773         if (!$nocheck) {
5774
5775             my $conf = PVE::QemuConfig->load_config($vmid);
5776
5777             PVE::QemuConfig->check_lock($conf)
5778                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5779
5780             vm_mon_cmd($vmid, $resume_cmd);
5781
5782         } else {
5783             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5784         }
5785     });
5786 }
5787
5788 sub vm_sendkey {
5789     my ($vmid, $skiplock, $key) = @_;
5790
5791     PVE::QemuConfig->lock_config($vmid, sub {
5792
5793         my $conf = PVE::QemuConfig->load_config($vmid);
5794
5795         # there is no qmp command, so we use the human monitor command
5796         vm_human_monitor_command($vmid, "sendkey $key");
5797     });
5798 }
5799
5800 sub vm_destroy {
5801     my ($storecfg, $vmid, $skiplock) = @_;
5802
5803     PVE::QemuConfig->lock_config($vmid, sub {
5804
5805         my $conf = PVE::QemuConfig->load_config($vmid);
5806
5807         if (!check_running($vmid)) {
5808             destroy_vm($storecfg, $vmid, undef, $skiplock);
5809         } else {
5810             die "VM $vmid is running - destroy failed\n";
5811         }
5812     });
5813 }
5814
5815 # vzdump restore implementaion
5816
5817 sub tar_archive_read_firstfile {
5818     my $archive = shift;
5819
5820     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5821
5822     # try to detect archive type first
5823     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5824         die "unable to open file '$archive'\n";
5825     my $firstfile = <$fh>;
5826     kill 15, $pid;
5827     close $fh;
5828
5829     die "ERROR: archive contaions no data\n" if !$firstfile;
5830     chomp $firstfile;
5831
5832     return $firstfile;
5833 }
5834
5835 sub tar_restore_cleanup {
5836     my ($storecfg, $statfile) = @_;
5837
5838     print STDERR "starting cleanup\n";
5839
5840     if (my $fd = IO::File->new($statfile, "r")) {
5841         while (defined(my $line = <$fd>)) {
5842             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5843                 my $volid = $2;
5844                 eval {
5845                     if ($volid =~ m|^/|) {
5846                         unlink $volid || die 'unlink failed\n';
5847                     } else {
5848                         PVE::Storage::vdisk_free($storecfg, $volid);
5849                     }
5850                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5851                 };
5852                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5853             } else {
5854                 print STDERR "unable to parse line in statfile - $line";
5855             }
5856         }
5857         $fd->close();
5858     }
5859 }
5860
5861 sub restore_archive {
5862     my ($archive, $vmid, $user, $opts) = @_;
5863
5864     my $format = $opts->{format};
5865     my $comp;
5866
5867     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5868         $format = 'tar' if !$format;
5869         $comp = 'gzip';
5870     } elsif ($archive =~ m/\.tar$/) {
5871         $format = 'tar' if !$format;
5872     } elsif ($archive =~ m/.tar.lzo$/) {
5873         $format = 'tar' if !$format;
5874         $comp = 'lzop';
5875     } elsif ($archive =~ m/\.vma$/) {
5876         $format = 'vma' if !$format;
5877     } elsif ($archive =~ m/\.vma\.gz$/) {
5878         $format = 'vma' if !$format;
5879         $comp = 'gzip';
5880     } elsif ($archive =~ m/\.vma\.lzo$/) {
5881         $format = 'vma' if !$format;
5882         $comp = 'lzop';
5883     } else {
5884         $format = 'vma' if !$format; # default
5885     }
5886
5887     # try to detect archive format
5888     if ($format eq 'tar') {
5889         return restore_tar_archive($archive, $vmid, $user, $opts);
5890     } else {
5891         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5892     }
5893 }
5894
5895 sub restore_update_config_line {
5896     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5897
5898     return if $line =~ m/^\#qmdump\#/;
5899     return if $line =~ m/^\#vzdump\#/;
5900     return if $line =~ m/^lock:/;
5901     return if $line =~ m/^unused\d+:/;
5902     return if $line =~ m/^parent:/;
5903     return if $line =~ m/^template:/; # restored VM is never a template
5904
5905     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5906     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5907         # try to convert old 1.X settings
5908         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5909         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5910             my ($model, $macaddr) = split(/\=/, $devconfig);
5911             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5912             my $net = {
5913                 model => $model,
5914                 bridge => "vmbr$ind",
5915                 macaddr => $macaddr,
5916             };
5917             my $netstr = print_net($net);
5918
5919             print $outfd "net$cookie->{netcount}: $netstr\n";
5920             $cookie->{netcount}++;
5921         }
5922     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5923         my ($id, $netstr) = ($1, $2);
5924         my $net = parse_net($netstr);
5925         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5926         $netstr = print_net($net);
5927         print $outfd "$id: $netstr\n";
5928     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5929         my $virtdev = $1;
5930         my $value = $3;
5931         my $di = parse_drive($virtdev, $value);
5932         if (defined($di->{backup}) && !$di->{backup}) {
5933             print $outfd "#$line";
5934         } elsif ($map->{$virtdev}) {
5935             delete $di->{format}; # format can change on restore
5936             $di->{file} = $map->{$virtdev};
5937             $value = print_drive($vmid, $di);
5938             print $outfd "$virtdev: $value\n";
5939         } else {
5940             print $outfd $line;
5941         }
5942     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5943         my $vmgenid = $1;
5944         if ($vmgenid ne '0') {
5945             # always generate a new vmgenid if there was a valid one setup
5946             $vmgenid = generate_uuid();
5947         }
5948         print $outfd "vmgenid: $vmgenid\n";
5949     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5950         my ($uuid, $uuid_str);
5951         UUID::generate($uuid);
5952         UUID::unparse($uuid, $uuid_str);
5953         my $smbios1 = parse_smbios1($2);
5954         $smbios1->{uuid} = $uuid_str;
5955         print $outfd $1.print_smbios1($smbios1)."\n";
5956     } else {
5957         print $outfd $line;
5958     }
5959 }
5960
5961 sub scan_volids {
5962     my ($cfg, $vmid) = @_;
5963
5964     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5965
5966     my $volid_hash = {};
5967     foreach my $storeid (keys %$info) {
5968         foreach my $item (@{$info->{$storeid}}) {
5969             next if !($item->{volid} && $item->{size});
5970             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5971             $volid_hash->{$item->{volid}} = $item;
5972         }
5973     }
5974
5975     return $volid_hash;
5976 }
5977
5978 sub is_volume_in_use {
5979     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5980
5981     my $path = PVE::Storage::path($storecfg, $volid);
5982
5983     my $scan_config = sub {
5984         my ($cref, $snapname) = @_;
5985
5986         foreach my $key (keys %$cref) {
5987             my $value = $cref->{$key};
5988             if (is_valid_drivename($key)) {
5989                 next if $skip_drive && $key eq $skip_drive;
5990                 my $drive = parse_drive($key, $value);
5991                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5992                 return 1 if $volid eq $drive->{file};
5993                 if ($drive->{file} =~ m!^/!) {
5994                     return 1 if $drive->{file} eq $path;
5995                 } else {
5996                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5997                     next if !$storeid;
5998                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5999                     next if !$scfg;
6000                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6001                 }
6002             }
6003         }
6004
6005         return 0;
6006     };
6007
6008     return 1 if &$scan_config($conf);
6009
6010     undef $skip_drive;
6011
6012     foreach my $snapname (keys %{$conf->{snapshots}}) {
6013         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6014     }
6015
6016     return 0;
6017 }
6018
6019 sub update_disksize {
6020     my ($vmid, $conf, $volid_hash) = @_;
6021
6022     my $changes;
6023     my $prefix = "VM $vmid:";
6024
6025     # used and unused disks
6026     my $referenced = {};
6027
6028     # Note: it is allowed to define multiple storages with same path (alias), so
6029     # we need to check both 'volid' and real 'path' (two different volid can point
6030     # to the same path).
6031
6032     my $referencedpath = {};
6033
6034     # update size info
6035     foreach my $opt (keys %$conf) {
6036         if (is_valid_drivename($opt)) {
6037             my $drive = parse_drive($opt, $conf->{$opt});
6038             my $volid = $drive->{file};
6039             next if !$volid;
6040
6041             $referenced->{$volid} = 1;
6042             if ($volid_hash->{$volid} &&
6043                 (my $path = $volid_hash->{$volid}->{path})) {
6044                 $referencedpath->{$path} = 1;
6045             }
6046
6047             next if drive_is_cdrom($drive);
6048             next if !$volid_hash->{$volid};
6049
6050             $drive->{size} = $volid_hash->{$volid}->{size};
6051             my $new = print_drive($vmid, $drive);
6052             if ($new ne $conf->{$opt}) {
6053                 $changes = 1;
6054                 $conf->{$opt} = $new;
6055                 print "$prefix update disk '$opt' information.\n";
6056             }
6057         }
6058     }
6059
6060     # remove 'unusedX' entry if volume is used
6061     foreach my $opt (keys %$conf) {
6062         next if $opt !~ m/^unused\d+$/;
6063         my $volid = $conf->{$opt};
6064         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6065         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6066             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6067             $changes = 1;
6068             delete $conf->{$opt};
6069         }
6070
6071         $referenced->{$volid} = 1;
6072         $referencedpath->{$path} = 1 if $path;
6073     }
6074
6075     foreach my $volid (sort keys %$volid_hash) {
6076         next if $volid =~ m/vm-$vmid-state-/;
6077         next if $referenced->{$volid};
6078         my $path = $volid_hash->{$volid}->{path};
6079         next if !$path; # just to be sure
6080         next if $referencedpath->{$path};
6081         $changes = 1;
6082         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6083         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6084         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6085     }
6086
6087     return $changes;
6088 }
6089
6090 sub rescan {
6091     my ($vmid, $nolock, $dryrun) = @_;
6092
6093     my $cfg = PVE::Storage::config();
6094
6095     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6096     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6097     foreach my $stor (keys %{$cfg->{ids}}) {
6098         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6099     }
6100
6101     print "rescan volumes...\n";
6102     my $volid_hash = scan_volids($cfg, $vmid);
6103
6104     my $updatefn =  sub {
6105         my ($vmid) = @_;
6106
6107         my $conf = PVE::QemuConfig->load_config($vmid);
6108
6109         PVE::QemuConfig->check_lock($conf);
6110
6111         my $vm_volids = {};
6112         foreach my $volid (keys %$volid_hash) {
6113             my $info = $volid_hash->{$volid};
6114             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6115         }
6116
6117         my $changes = update_disksize($vmid, $conf, $vm_volids);
6118
6119         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6120     };
6121
6122     if (defined($vmid)) {
6123         if ($nolock) {
6124             &$updatefn($vmid);
6125         } else {
6126             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6127         }
6128     } else {
6129         my $vmlist = config_list();
6130         foreach my $vmid (keys %$vmlist) {
6131             if ($nolock) {
6132                 &$updatefn($vmid);
6133             } else {
6134                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6135             }
6136         }
6137     }
6138 }
6139
6140 sub restore_vma_archive {
6141     my ($archive, $vmid, $user, $opts, $comp) = @_;
6142
6143     my $readfrom = $archive;
6144
6145     my $cfg = PVE::Storage::config();
6146     my $commands = [];
6147     my $bwlimit = $opts->{bwlimit};
6148
6149     my $dbg_cmdstring = '';
6150     my $add_pipe = sub {
6151         my ($cmd) = @_;
6152         push @$commands, $cmd;
6153         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6154         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6155         $readfrom = '-';
6156     };
6157
6158     my $input = undef;
6159     if ($archive eq '-') {
6160         $input = '<&STDIN';
6161     } else {
6162         # If we use a backup from a PVE defined storage we also consider that
6163         # storage's rate limit:
6164         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6165         if (defined($volid)) {
6166             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6167             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6168             if ($readlimit) {
6169                 print STDERR "applying read rate limit: $readlimit\n";
6170                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6171                 $add_pipe->($cstream);
6172             }
6173         }
6174     }
6175
6176     if ($comp) {
6177         my $cmd;
6178         if ($comp eq 'gzip') {
6179             $cmd = ['zcat', $readfrom];
6180         } elsif ($comp eq 'lzop') {
6181             $cmd = ['lzop', '-d', '-c', $readfrom];
6182         } else {
6183             die "unknown compression method '$comp'\n";
6184         }
6185         $add_pipe->($cmd);
6186     }
6187
6188     my $tmpdir = "/var/tmp/vzdumptmp$$";
6189     rmtree $tmpdir;
6190
6191     # disable interrupts (always do cleanups)
6192     local $SIG{INT} =
6193         local $SIG{TERM} =
6194         local $SIG{QUIT} =
6195         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6196
6197     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6198     POSIX::mkfifo($mapfifo, 0600);
6199     my $fifofh;
6200
6201     my $openfifo = sub {
6202         open($fifofh, '>', $mapfifo) || die $!;
6203     };
6204
6205     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6206
6207     my $oldtimeout;
6208     my $timeout = 5;
6209
6210     my $devinfo = {};
6211
6212     my $rpcenv = PVE::RPCEnvironment::get();
6213
6214     my $conffile = PVE::QemuConfig->config_file($vmid);
6215     my $tmpfn = "$conffile.$$.tmp";
6216
6217     # Note: $oldconf is undef if VM does not exists
6218     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6219     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6220
6221     my %storage_limits;
6222
6223     my $print_devmap = sub {
6224         my $virtdev_hash = {};
6225
6226         my $cfgfn = "$tmpdir/qemu-server.conf";
6227
6228         # we can read the config - that is already extracted
6229         my $fh = IO::File->new($cfgfn, "r") ||
6230             "unable to read qemu-server.conf - $!\n";
6231
6232         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6233         if (-f $fwcfgfn) {
6234             my $pve_firewall_dir = '/etc/pve/firewall';
6235             mkdir $pve_firewall_dir; # make sure the dir exists
6236             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6237         }
6238
6239         while (defined(my $line = <$fh>)) {
6240             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6241                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6242                 die "archive does not contain data for drive '$virtdev'\n"
6243                     if !$devinfo->{$devname};
6244                 if (defined($opts->{storage})) {
6245                     $storeid = $opts->{storage} || 'local';
6246                 } elsif (!$storeid) {
6247                     $storeid = 'local';
6248                 }
6249                 $format = 'raw' if !$format;
6250                 $devinfo->{$devname}->{devname} = $devname;
6251                 $devinfo->{$devname}->{virtdev} = $virtdev;
6252                 $devinfo->{$devname}->{format} = $format;
6253                 $devinfo->{$devname}->{storeid} = $storeid;
6254
6255                 # check permission on storage
6256                 my $pool = $opts->{pool}; # todo: do we need that?
6257                 if ($user ne 'root@pam') {
6258                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6259                 }
6260
6261                 $storage_limits{$storeid} = $bwlimit;
6262
6263                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6264             }
6265         }
6266
6267         foreach my $key (keys %storage_limits) {
6268             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6269             next if !$limit;
6270             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6271             $storage_limits{$key} = $limit * 1024;
6272         }
6273
6274         foreach my $devname (keys %$devinfo) {
6275             die "found no device mapping information for device '$devname'\n"
6276                 if !$devinfo->{$devname}->{virtdev};
6277         }
6278
6279         # create empty/temp config
6280         if ($oldconf) {
6281             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6282             foreach_drive($oldconf, sub {
6283                 my ($ds, $drive) = @_;
6284
6285                 return if drive_is_cdrom($drive);
6286
6287                 my $volid = $drive->{file};
6288
6289                 return if !$volid || $volid =~ m|^/|;
6290
6291                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6292                 return if !$path || !$owner || ($owner != $vmid);
6293
6294                 # Note: only delete disk we want to restore
6295                 # other volumes will become unused
6296                 if ($virtdev_hash->{$ds}) {
6297                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6298                     if (my $err = $@) {
6299                         warn $err;
6300                     }
6301                 }
6302             });
6303
6304             # delete vmstate files
6305             # since after the restore we have no snapshots anymore
6306             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6307                 my $snap = $oldconf->{snapshots}->{$snapname};
6308                 if ($snap->{vmstate}) {
6309                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6310                     if (my $err = $@) {
6311                         warn $err;
6312                     }
6313                 }
6314             }
6315         }
6316
6317         my $map = {};
6318         foreach my $virtdev (sort keys %$virtdev_hash) {
6319             my $d = $virtdev_hash->{$virtdev};
6320             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6321             my $storeid = $d->{storeid};
6322             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6323
6324             my $map_opts = '';
6325             if (my $limit = $storage_limits{$storeid}) {
6326                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6327             }
6328
6329             # test if requested format is supported
6330             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6331             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6332             $d->{format} = $defFormat if !$supported;
6333
6334             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6335                                                   $d->{format}, undef, $alloc_size);
6336             print STDERR "new volume ID is '$volid'\n";
6337             $d->{volid} = $volid;
6338             my $path = PVE::Storage::path($cfg, $volid);
6339
6340             PVE::Storage::activate_volumes($cfg,[$volid]);
6341
6342             my $write_zeros = 1;
6343             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6344                 $write_zeros = 0;
6345             }
6346
6347             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6348
6349             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6350             $map->{$virtdev} = $volid;
6351         }
6352
6353         $fh->seek(0, 0) || die "seek failed - $!\n";
6354
6355         my $outfd = new IO::File ($tmpfn, "w") ||
6356             die "unable to write config for VM $vmid\n";
6357
6358         my $cookie = { netcount => 0 };
6359         while (defined(my $line = <$fh>)) {
6360             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6361         }
6362
6363         $fh->close();
6364         $outfd->close();
6365     };
6366
6367     eval {
6368         # enable interrupts
6369         local $SIG{INT} =
6370             local $SIG{TERM} =
6371             local $SIG{QUIT} =
6372             local $SIG{HUP} =
6373             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6374         local $SIG{ALRM} = sub { die "got timeout\n"; };
6375
6376         $oldtimeout = alarm($timeout);
6377
6378         my $parser = sub {
6379             my $line = shift;
6380
6381             print "$line\n";
6382
6383             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6384                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6385                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6386             } elsif ($line =~ m/^CTIME: /) {
6387                 # we correctly received the vma config, so we can disable
6388                 # the timeout now for disk allocation (set to 10 minutes, so
6389                 # that we always timeout if something goes wrong)
6390                 alarm(600);
6391                 &$print_devmap();
6392                 print $fifofh "done\n";
6393                 my $tmp = $oldtimeout || 0;
6394                 $oldtimeout = undef;
6395                 alarm($tmp);
6396                 close($fifofh);
6397             }
6398         };
6399
6400         print "restore vma archive: $dbg_cmdstring\n";
6401         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6402     };
6403     my $err = $@;
6404
6405     alarm($oldtimeout) if $oldtimeout;
6406
6407     my $vollist = [];
6408     foreach my $devname (keys %$devinfo) {
6409         my $volid = $devinfo->{$devname}->{volid};
6410         push @$vollist, $volid if $volid;
6411     }
6412
6413     PVE::Storage::deactivate_volumes($cfg, $vollist);
6414
6415     unlink $mapfifo;
6416
6417     if ($err) {
6418         rmtree $tmpdir;
6419         unlink $tmpfn;
6420
6421         foreach my $devname (keys %$devinfo) {
6422             my $volid = $devinfo->{$devname}->{volid};
6423             next if !$volid;
6424             eval {
6425                 if ($volid =~ m|^/|) {
6426                     unlink $volid || die 'unlink failed\n';
6427                 } else {
6428                     PVE::Storage::vdisk_free($cfg, $volid);
6429                 }
6430                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6431             };
6432             print STDERR "unable to cleanup '$volid' - $@" if $@;
6433         }
6434         die $err;
6435     }
6436
6437     rmtree $tmpdir;
6438
6439     rename($tmpfn, $conffile) ||
6440         die "unable to commit configuration file '$conffile'\n";
6441
6442     PVE::Cluster::cfs_update(); # make sure we read new file
6443
6444     eval { rescan($vmid, 1); };
6445     warn $@ if $@;
6446 }
6447
6448 sub restore_tar_archive {
6449     my ($archive, $vmid, $user, $opts) = @_;
6450
6451     if ($archive ne '-') {
6452         my $firstfile = tar_archive_read_firstfile($archive);
6453         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6454             if $firstfile ne 'qemu-server.conf';
6455     }
6456
6457     my $storecfg = PVE::Storage::config();
6458
6459     # destroy existing data - keep empty config
6460     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6461     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6462
6463     my $tocmd = "/usr/lib/qemu-server/qmextract";
6464
6465     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6466     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6467     $tocmd .= ' --prealloc' if $opts->{prealloc};
6468     $tocmd .= ' --info' if $opts->{info};
6469
6470     # tar option "xf" does not autodetect compression when read from STDIN,
6471     # so we pipe to zcat
6472     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6473         PVE::Tools::shellquote("--to-command=$tocmd");
6474
6475     my $tmpdir = "/var/tmp/vzdumptmp$$";
6476     mkpath $tmpdir;
6477
6478     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6479     local $ENV{VZDUMP_VMID} = $vmid;
6480     local $ENV{VZDUMP_USER} = $user;
6481
6482     my $conffile = PVE::QemuConfig->config_file($vmid);
6483     my $tmpfn = "$conffile.$$.tmp";
6484
6485     # disable interrupts (always do cleanups)
6486     local $SIG{INT} =
6487         local $SIG{TERM} =
6488         local $SIG{QUIT} =
6489         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6490
6491     eval {
6492         # enable interrupts
6493         local $SIG{INT} =
6494             local $SIG{TERM} =
6495             local $SIG{QUIT} =
6496             local $SIG{HUP} =
6497             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6498
6499         if ($archive eq '-') {
6500             print "extracting archive from STDIN\n";
6501             run_command($cmd, input => "<&STDIN");
6502         } else {
6503             print "extracting archive '$archive'\n";
6504             run_command($cmd);
6505         }
6506
6507         return if $opts->{info};
6508
6509         # read new mapping
6510         my $map = {};
6511         my $statfile = "$tmpdir/qmrestore.stat";
6512         if (my $fd = IO::File->new($statfile, "r")) {
6513             while (defined (my $line = <$fd>)) {
6514                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6515                     $map->{$1} = $2 if $1;
6516                 } else {
6517                     print STDERR "unable to parse line in statfile - $line\n";
6518                 }
6519             }
6520             $fd->close();
6521         }
6522
6523         my $confsrc = "$tmpdir/qemu-server.conf";
6524
6525         my $srcfd = new IO::File($confsrc, "r") ||
6526             die "unable to open file '$confsrc'\n";
6527
6528         my $outfd = new IO::File ($tmpfn, "w") ||
6529             die "unable to write config for VM $vmid\n";
6530
6531         my $cookie = { netcount => 0 };
6532         while (defined (my $line = <$srcfd>)) {
6533             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6534         }
6535
6536         $srcfd->close();
6537         $outfd->close();
6538     };
6539     my $err = $@;
6540
6541     if ($err) {
6542
6543         unlink $tmpfn;
6544
6545         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6546
6547         die $err;
6548     }
6549
6550     rmtree $tmpdir;
6551
6552     rename $tmpfn, $conffile ||
6553         die "unable to commit configuration file '$conffile'\n";
6554
6555     PVE::Cluster::cfs_update(); # make sure we read new file
6556
6557     eval { rescan($vmid, 1); };
6558     warn $@ if $@;
6559 };
6560
6561 sub foreach_storage_used_by_vm {
6562     my ($conf, $func) = @_;
6563
6564     my $sidhash = {};
6565
6566     foreach_drive($conf, sub {
6567         my ($ds, $drive) = @_;
6568         return if drive_is_cdrom($drive);
6569
6570         my $volid = $drive->{file};
6571
6572         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6573         $sidhash->{$sid} = $sid if $sid;
6574     });
6575
6576     foreach my $sid (sort keys %$sidhash) {
6577         &$func($sid);
6578     }
6579 }
6580
6581 sub do_snapshots_with_qemu {
6582     my ($storecfg, $volid) = @_;
6583
6584     my $storage_name = PVE::Storage::parse_volume_id($volid);
6585
6586     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6587         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6588         return 1;
6589     }
6590
6591     if ($volid =~ m/\.(qcow2|qed)$/){
6592         return 1;
6593     }
6594
6595     return undef;
6596 }
6597
6598 sub qga_check_running {
6599     my ($vmid, $nowarn) = @_;
6600
6601     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6602     if ($@) {
6603         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6604         return 0;
6605     }
6606     return 1;
6607 }
6608
6609 sub template_create {
6610     my ($vmid, $conf, $disk) = @_;
6611
6612     my $storecfg = PVE::Storage::config();
6613
6614     foreach_drive($conf, sub {
6615         my ($ds, $drive) = @_;
6616
6617         return if drive_is_cdrom($drive);
6618         return if $disk && $ds ne $disk;
6619
6620         my $volid = $drive->{file};
6621         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6622
6623         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6624         $drive->{file} = $voliddst;
6625         $conf->{$ds} = print_drive($vmid, $drive);
6626         PVE::QemuConfig->write_config($vmid, $conf);
6627     });
6628 }
6629
6630 sub convert_iscsi_path {
6631     my ($path) = @_;
6632
6633     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6634         my $portal = $1;
6635         my $target = $2;
6636         my $lun = $3;
6637
6638         my $initiator_name = get_initiator_name();
6639
6640         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6641                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6642     }
6643
6644     die "cannot convert iscsi path '$path', unkown format\n";
6645 }
6646
6647 sub qemu_img_convert {
6648     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6649
6650     my $storecfg = PVE::Storage::config();
6651     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6652     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6653
6654     if ($src_storeid && $dst_storeid) {
6655
6656         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6657
6658         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6659         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6660
6661         my $src_format = qemu_img_format($src_scfg, $src_volname);
6662         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6663
6664         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6665         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6666
6667         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6668         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6669
6670         my $cmd = [];
6671         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6672         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6673         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6674         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6675
6676         if ($src_is_iscsi) {
6677             push @$cmd, '--image-opts';
6678             $src_path = convert_iscsi_path($src_path);
6679         } else {
6680             push @$cmd, '-f', $src_format;
6681         }
6682
6683         if ($dst_is_iscsi) {
6684             push @$cmd, '--target-image-opts';
6685             $dst_path = convert_iscsi_path($dst_path);
6686         } else {
6687             push @$cmd, '-O', $dst_format;
6688         }
6689
6690         push @$cmd, $src_path;
6691
6692         if (!$dst_is_iscsi && $is_zero_initialized) {
6693             push @$cmd, "zeroinit:$dst_path";
6694         } else {
6695             push @$cmd, $dst_path;
6696         }
6697
6698         my $parser = sub {
6699             my $line = shift;
6700             if($line =~ m/\((\S+)\/100\%\)/){
6701                 my $percent = $1;
6702                 my $transferred = int($size * $percent / 100);
6703                 my $remaining = $size - $transferred;
6704
6705                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6706             }
6707
6708         };
6709
6710         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6711         my $err = $@;
6712         die "copy failed: $err" if $err;
6713     }
6714 }
6715
6716 sub qemu_img_format {
6717     my ($scfg, $volname) = @_;
6718
6719     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6720         return $1;
6721     } else {
6722         return "raw";
6723     }
6724 }
6725
6726 sub qemu_drive_mirror {
6727     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6728
6729     $jobs = {} if !$jobs;
6730
6731     my $qemu_target;
6732     my $format;
6733     $jobs->{"drive-$drive"} = {};
6734
6735     if ($dst_volid =~ /^nbd:/) {
6736         $qemu_target = $dst_volid;
6737         $format = "nbd";
6738     } else {
6739         my $storecfg = PVE::Storage::config();
6740         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6741
6742         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6743
6744         $format = qemu_img_format($dst_scfg, $dst_volname);
6745
6746         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6747
6748         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6749     }
6750
6751     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6752     $opts->{format} = $format if $format;
6753
6754     print "drive mirror is starting for drive-$drive\n";
6755
6756     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6757
6758     if (my $err = $@) {
6759         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6760         die "mirroring error: $err";
6761     }
6762
6763     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6764 }
6765
6766 sub qemu_drive_mirror_monitor {
6767     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6768
6769     eval {
6770         my $err_complete = 0;
6771
6772         while (1) {
6773             die "storage migration timed out\n" if $err_complete > 300;
6774
6775             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6776
6777             my $running_mirror_jobs = {};
6778             foreach my $stat (@$stats) {
6779                 next if $stat->{type} ne 'mirror';
6780                 $running_mirror_jobs->{$stat->{device}} = $stat;
6781             }
6782
6783             my $readycounter = 0;
6784
6785             foreach my $job (keys %$jobs) {
6786
6787                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6788                     print "$job : finished\n";
6789                     delete $jobs->{$job};
6790                     next;
6791                 }
6792
6793                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6794
6795                 my $busy = $running_mirror_jobs->{$job}->{busy};
6796                 my $ready = $running_mirror_jobs->{$job}->{ready};
6797                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6798                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6799                     my $remaining = $total - $transferred;
6800                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6801
6802                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6803                 }
6804
6805                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6806             }
6807
6808             last if scalar(keys %$jobs) == 0;
6809
6810             if ($readycounter == scalar(keys %$jobs)) {
6811                 print "all mirroring jobs are ready \n";
6812                 last if $skipcomplete; #do the complete later
6813
6814                 if ($vmiddst && $vmiddst != $vmid) {
6815                     my $agent_running = $qga && qga_check_running($vmid);
6816                     if ($agent_running) {
6817                         print "freeze filesystem\n";
6818                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6819                     } else {
6820                         print "suspend vm\n";
6821                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6822                     }
6823
6824                     # if we clone a disk for a new target vm, we don't switch the disk
6825                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6826
6827                     if ($agent_running) {
6828                         print "unfreeze filesystem\n";
6829                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6830                     } else {
6831                         print "resume vm\n";
6832                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6833                     }
6834
6835                     last;
6836                 } else {
6837
6838                     foreach my $job (keys %$jobs) {
6839                         # try to switch the disk if source and destination are on the same guest
6840                         print "$job: Completing block job...\n";
6841
6842                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6843                         if ($@ =~ m/cannot be completed/) {
6844                             print "$job: Block job cannot be completed, try again.\n";
6845                             $err_complete++;
6846                         }else {
6847                             print "$job: Completed successfully.\n";
6848                             $jobs->{$job}->{complete} = 1;
6849                         }
6850                     }
6851                 }
6852             }
6853             sleep 1;
6854         }
6855     };
6856     my $err = $@;
6857
6858     if ($err) {
6859         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6860         die "mirroring error: $err";
6861     }
6862
6863 }
6864
6865 sub qemu_blockjobs_cancel {
6866     my ($vmid, $jobs) = @_;
6867
6868     foreach my $job (keys %$jobs) {
6869         print "$job: Cancelling block job\n";
6870         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6871         $jobs->{$job}->{cancel} = 1;
6872     }
6873
6874     while (1) {
6875         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6876
6877         my $running_jobs = {};
6878         foreach my $stat (@$stats) {
6879             $running_jobs->{$stat->{device}} = $stat;
6880         }
6881
6882         foreach my $job (keys %$jobs) {
6883
6884             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6885                 print "$job: Done.\n";
6886                 delete $jobs->{$job};
6887             }
6888         }
6889
6890         last if scalar(keys %$jobs) == 0;
6891
6892         sleep 1;
6893     }
6894 }
6895
6896 sub clone_disk {
6897     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6898         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6899
6900     my $newvolid;
6901
6902     if (!$full) {
6903         print "create linked clone of drive $drivename ($drive->{file})\n";
6904         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6905         push @$newvollist, $newvolid;
6906     } else {
6907
6908         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6909         $storeid = $storage if $storage;
6910
6911         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6912         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6913
6914         print "create full clone of drive $drivename ($drive->{file})\n";
6915         my $name = undef;
6916         if (drive_is_cloudinit($drive)) {
6917             $name = "vm-$newvmid-cloudinit";
6918             $snapname = undef;
6919             # cloudinit only supports raw and qcow2 atm:
6920             if ($dst_format eq 'qcow2') {
6921                 $name .= '.qcow2';
6922             } elsif ($dst_format ne 'raw') {
6923                 die "clone: unhandled format for cloudinit image\n";
6924             }
6925         }
6926         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6927         push @$newvollist, $newvolid;
6928
6929         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6930
6931         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6932         if (!$running || $snapname) {
6933             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6934         } else {
6935
6936             my $kvmver = get_running_qemu_version ($vmid);
6937             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6938                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6939                     if $drive->{iothread};
6940             }
6941
6942             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6943         }
6944     }
6945
6946     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6947
6948     my $disk = $drive;
6949     $disk->{format} = undef;
6950     $disk->{file} = $newvolid;
6951     $disk->{size} = $size;
6952
6953     return $disk;
6954 }
6955
6956 # this only works if VM is running
6957 sub get_current_qemu_machine {
6958     my ($vmid) = @_;
6959
6960     my $cmd = { execute => 'query-machines', arguments => {} };
6961     my $res = vm_qmp_command($vmid, $cmd);
6962
6963     my ($current, $default);
6964     foreach my $e (@$res) {
6965         $default = $e->{name} if $e->{'is-default'};
6966         $current = $e->{name} if $e->{'is-current'};
6967     }
6968
6969     # fallback to the default machine if current is not supported by qemu
6970     return $current || $default || 'pc';
6971 }
6972
6973 sub get_running_qemu_version {
6974     my ($vmid) = @_;
6975     my $cmd = { execute => 'query-version', arguments => {} };
6976     my $res = vm_qmp_command($vmid, $cmd);
6977     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6978 }
6979
6980 sub qemu_machine_feature_enabled {
6981     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6982
6983     my $current_major;
6984     my $current_minor;
6985
6986     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6987
6988         $current_major = $3;
6989         $current_minor = $4;
6990
6991     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6992
6993         $current_major = $1;
6994         $current_minor = $2;
6995     }
6996
6997     return 1 if $current_major > $version_major ||
6998                 ($current_major == $version_major &&
6999                  $current_minor >= $version_minor);
7000 }
7001
7002 sub qemu_machine_pxe {
7003     my ($vmid, $conf, $machine) = @_;
7004
7005     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7006
7007     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7008         $machine .= '.pxe';
7009     }
7010
7011     return $machine;
7012 }
7013
7014 sub qemu_use_old_bios_files {
7015     my ($machine_type) = @_;
7016
7017     return if !$machine_type;
7018
7019     my $use_old_bios_files = undef;
7020
7021     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7022         $machine_type = $1;
7023         $use_old_bios_files = 1;
7024     } else {
7025         my $kvmver = kvm_user_version();
7026         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7027         # load new efi bios files on migration. So this hack is required to allow
7028         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7029         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7030         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7031     }
7032
7033     return ($use_old_bios_files, $machine_type);
7034 }
7035
7036 sub create_efidisk($$$$$) {
7037     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7038
7039     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7040     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7041
7042     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7043     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7044     PVE::Storage::activate_volumes($storecfg, [$volid]);
7045
7046     my $path = PVE::Storage::path($storecfg, $volid);
7047     eval {
7048         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7049     };
7050     die "Copying EFI vars image failed: $@" if $@;
7051
7052     return ($volid, $vars_size);
7053 }
7054
7055 sub vm_iothreads_list {
7056     my ($vmid) = @_;
7057
7058     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7059
7060     my $iothreads = {};
7061     foreach my $iothread (@$res) {
7062         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7063     }
7064
7065     return $iothreads;
7066 }
7067
7068 sub scsihw_infos {
7069     my ($conf, $drive) = @_;
7070
7071     my $maxdev = 0;
7072
7073     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7074         $maxdev = 7;
7075     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7076         $maxdev = 1;
7077     } else {
7078         $maxdev = 256;
7079     }
7080
7081     my $controller = int($drive->{index} / $maxdev);
7082     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7083
7084     return ($maxdev, $controller, $controller_prefix);
7085 }
7086
7087 sub add_hyperv_enlightenments {
7088     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7089
7090     return if $winversion < 6;
7091     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7092
7093     if ($gpu_passthrough || defined($hv_vendor_id)) {
7094         $hv_vendor_id //= 'proxmox';
7095         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7096     }
7097
7098     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7099         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7100         push @$cpuFlags , 'hv_vapic';
7101         push @$cpuFlags , 'hv_time';
7102     } else {
7103         push @$cpuFlags , 'hv_spinlocks=0xffff';
7104     }
7105
7106     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7107         push @$cpuFlags , 'hv_reset';
7108         push @$cpuFlags , 'hv_vpindex';
7109         push @$cpuFlags , 'hv_runtime';
7110     }
7111
7112     if ($winversion >= 7) {
7113         push @$cpuFlags , 'hv_relaxed';
7114
7115         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7116             push @$cpuFlags , 'hv_synic';
7117             push @$cpuFlags , 'hv_stimer';
7118         }
7119     }
7120 }
7121
7122 sub windows_version {
7123     my ($ostype) = @_;
7124
7125     return 0 if !$ostype;
7126
7127     my $winversion = 0;
7128
7129     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7130         $winversion = 5;
7131     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7132         $winversion = 6;
7133     } elsif ($ostype =~ m/^win(\d+)$/) {
7134         $winversion = $1;
7135     }
7136
7137     return $winversion;
7138 }
7139
7140 sub resolve_dst_disk_format {
7141         my ($storecfg, $storeid, $src_volname, $format) = @_;
7142         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7143
7144         if (!$format) {
7145             # if no target format is specified, use the source disk format as hint
7146             if ($src_volname) {
7147                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7148                 $format = qemu_img_format($scfg, $src_volname);
7149             } else {
7150                 return $defFormat;
7151             }
7152         }
7153
7154         # test if requested format is supported - else use default
7155         my $supported = grep { $_ eq $format } @$validFormats;
7156         $format = $defFormat if !$supported;
7157         return $format;
7158 }
7159
7160 sub resolve_first_disk {
7161     my $conf = shift;
7162     my @disks = PVE::QemuServer::valid_drive_names();
7163     my $firstdisk;
7164     foreach my $ds (reverse @disks) {
7165         next if !$conf->{$ds};
7166         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7167         next if PVE::QemuServer::drive_is_cdrom($disk);
7168         $firstdisk = $ds;
7169     }
7170     return $firstdisk;
7171 }
7172
7173 sub generate_uuid {
7174     my ($uuid, $uuid_str);
7175     UUID::generate($uuid);
7176     UUID::unparse($uuid, $uuid_str);
7177     return $uuid_str;
7178 }
7179
7180 sub generate_smbios1_uuid {
7181     return "uuid=".generate_uuid();
7182 }
7183
7184 sub nbd_stop {
7185     my ($vmid) = @_;
7186
7187     vm_mon_cmd($vmid, 'nbd-server-stop');
7188 }
7189
7190 # bash completion helper
7191
7192 sub complete_backup_archives {
7193     my ($cmdname, $pname, $cvalue) = @_;
7194
7195     my $cfg = PVE::Storage::config();
7196
7197     my $storeid;
7198
7199     if ($cvalue =~ m/^([^:]+):/) {
7200         $storeid = $1;
7201     }
7202
7203     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7204
7205     my $res = [];
7206     foreach my $id (keys %$data) {
7207         foreach my $item (@{$data->{$id}}) {
7208             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7209             push @$res, $item->{volid} if defined($item->{volid});
7210         }
7211     }
7212
7213     return $res;
7214 }
7215
7216 my $complete_vmid_full = sub {
7217     my ($running) = @_;
7218
7219     my $idlist = vmstatus();
7220
7221     my $res = [];
7222
7223     foreach my $id (keys %$idlist) {
7224         my $d = $idlist->{$id};
7225         if (defined($running)) {
7226             next if $d->{template};
7227             next if $running && $d->{status} ne 'running';
7228             next if !$running && $d->{status} eq 'running';
7229         }
7230         push @$res, $id;
7231
7232     }
7233     return $res;
7234 };
7235
7236 sub complete_vmid {
7237     return &$complete_vmid_full();
7238 }
7239
7240 sub complete_vmid_stopped {
7241     return &$complete_vmid_full(0);
7242 }
7243
7244 sub complete_vmid_running {
7245     return &$complete_vmid_full(1);
7246 }
7247
7248 sub complete_storage {
7249
7250     my $cfg = PVE::Storage::config();
7251     my $ids = $cfg->{ids};
7252
7253     my $res = [];
7254     foreach my $sid (keys %$ids) {
7255         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7256         next if !$ids->{$sid}->{content}->{images};
7257         push @$res, $sid;
7258     }
7259
7260     return $res;
7261 }
7262
7263 1;