PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_IDE_DISKS = 4;
 735 my $MAX_SCSI_DISKS = 14;
 736 my $MAX_VIRTIO_DISKS = 16;
 737 my $MAX_SATA_DISKS = 6;
 738 my $MAX_USB_DEVICES = 5;
 739 my $MAX_NETS = 32;
 740 my $MAX_UNUSED_DISKS = 256;
 741 my $MAX_HOSTPCI_DEVICES = 4;
 742 my $MAX_SERIAL_PORTS = 4;
 743 my $MAX_PARALLEL_PORTS = 3;
 744 my $MAX_NUMA = 8;
 745
 746 my $numa_fmt = {
 747     cpus => {
 748         type => "string",
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => "CPUs accessing this NUMA node.",
 751         format_description => "id[-id];...",
 752     },
 753     memory => {
 754         type => "number",
 755         description => "Amount of memory this NUMA node provides.",
 756         optional => 1,
 757     },
 758     hostnodes => {
 759         type => "string",
 760         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 761         description => "Host NUMA nodes to use.",
 762         format_description => "id[-id];...",
 763         optional => 1,
 764     },
 765     policy => {
 766         type => 'string',
 767         enum => [qw(preferred bind interleave)],
 768         description => "NUMA allocation policy.",
 769         optional => 1,
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 773 my $numadesc = {
 774     optional => 1,
 775     type => 'string', format => $numa_fmt,
 776     description => "NUMA topology.",
 777 };
 778 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 779
 780 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 781     $confdesc->{"numa$i"} = $numadesc;
 782 }
 783
 784 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 785                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 786                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 787 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 788
 789 my $net_fmt_bridge_descr = <<__EOD__;
 790 Bridge to attach the network device to. The Proxmox VE standard bridge
 791 is called 'vmbr0'.
 792
 793 If you do not specify a bridge, we create a kvm user (NATed) network
 794 device, which provides DHCP and DNS services. The following addresses
 795 are used:
 796
 797  10.0.2.2   Gateway
 798  10.0.2.3   DNS Server
 799  10.0.2.4   SMB Server
 800
 801 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 802 __EOD__
 803
 804 my $net_fmt = {
 805     macaddr => {
 806         type => 'string',
 807         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 808         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 809         format_description => "XX:XX:XX:XX:XX:XX",
 810         optional => 1,
 811     },
 812     model => {
 813         type => 'string',
 814         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 815         enum => $nic_model_list,
 816         default_key => 1,
 817     },
 818     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 819     bridge => {
 820         type => 'string',
 821         description => $net_fmt_bridge_descr,
 822         format_description => 'bridge',
 823         optional => 1,
 824     },
 825     queues => {
 826         type => 'integer',
 827         minimum => 0, maximum => 16,
 828         description => 'Number of packet queues to be used on the device.',
 829         optional => 1,
 830     },
 831     rate => {
 832         type => 'number',
 833         minimum => 0,
 834         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 835         optional => 1,
 836     },
 837     tag => {
 838         type => 'integer',
 839         minimum => 1, maximum => 4094,
 840         description => 'VLAN tag to apply to packets on this interface.',
 841         optional => 1,
 842     },
 843     trunks => {
 844         type => 'string',
 845         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 846         description => 'VLAN trunks to pass through this interface.',
 847         format_description => 'vlanid[;vlanid...]',
 848         optional => 1,
 849     },
 850     firewall => {
 851         type => 'boolean',
 852         description => 'Whether this interface should be protected by the firewall.',
 853         optional => 1,
 854     },
 855     link_down => {
 856         type => 'boolean',
 857         description => 'Whether this interface should be disconnected (like pulling the plug).',
 858         optional => 1,
 859     },
 860 };
 861
 862 my $netdesc = {
 863     optional => 1,
 864     type => 'string', format => $net_fmt,
 865     description => "Specify network devices.",
 866 };
 867
 868 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 869
 870 my $ipconfig_fmt = {
 871     ip => {
 872         type => 'string',
 873         format => 'pve-ipv4-config',
 874         format_description => 'IPv4Format/CIDR',
 875         description => 'IPv4 address in CIDR format.',
 876         optional => 1,
 877         default => 'dhcp',
 878     },
 879     gw => {
 880         type => 'string',
 881         format => 'ipv4',
 882         format_description => 'GatewayIPv4',
 883         description => 'Default gateway for IPv4 traffic.',
 884         optional => 1,
 885         requires => 'ip',
 886     },
 887     ip6 => {
 888         type => 'string',
 889         format => 'pve-ipv6-config',
 890         format_description => 'IPv6Format/CIDR',
 891         description => 'IPv6 address in CIDR format.',
 892         optional => 1,
 893         default => 'dhcp',
 894     },
 895     gw6 => {
 896         type => 'string',
 897         format => 'ipv6',
 898         format_description => 'GatewayIPv6',
 899         description => 'Default gateway for IPv6 traffic.',
 900         optional => 1,
 901         requires => 'ip6',
 902     },
 903 };
 904 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 905 my $ipconfigdesc = {
 906     optional => 1,
 907     type => 'string', format => 'pve-qm-ipconfig',
 908     description => <<'EODESCR',
 909 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 910
 911 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 912
 913 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 914 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 915
 916 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 917 EODESCR
 918 };
 919 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 920
 921 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 922     $confdesc->{"net$i"} = $netdesc;
 923     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 924 }
 925
 926 foreach my $key (keys %$confdesc_cloudinit) {
 927     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 928 }
 929
 930 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 931 sub verify_volume_id_or_qm_path {
 932     my ($volid, $noerr) = @_;
 933
 934     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 935         return $volid;
 936     }
 937
 938     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 939     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 940     if ($@) {
 941         return undef if $noerr;
 942         die $@;
 943     }
 944     return $volid;
 945 }
 946
 947 my $drivename_hash;
 948
 949 my %drivedesc_base = (
 950     volume => { alias => 'file' },
 951     file => {
 952         type => 'string',
 953         format => 'pve-volume-id-or-qm-path',
 954         default_key => 1,
 955         format_description => 'volume',
 956         description => "The drive's backing volume.",
 957     },
 958     media => {
 959         type => 'string',
 960         enum => [qw(cdrom disk)],
 961         description => "The drive's media type.",
 962         default => 'disk',
 963         optional => 1
 964     },
 965     cyls => {
 966         type => 'integer',
 967         description => "Force the drive's physical geometry to have a specific cylinder count.",
 968         optional => 1
 969     },
 970     heads => {
 971         type => 'integer',
 972         description => "Force the drive's physical geometry to have a specific head count.",
 973         optional => 1
 974     },
 975     secs => {
 976         type => 'integer',
 977         description => "Force the drive's physical geometry to have a specific sector count.",
 978         optional => 1
 979     },
 980     trans => {
 981         type => 'string',
 982         enum => [qw(none lba auto)],
 983         description => "Force disk geometry bios translation mode.",
 984         optional => 1,
 985     },
 986     snapshot => {
 987         type => 'boolean',
 988         description => "Controls qemu's snapshot mode feature."
 989             . " If activated, changes made to the disk are temporary and will"
 990             . " be discarded when the VM is shutdown.",
 991         optional => 1,
 992     },
 993     cache => {
 994         type => 'string',
 995         enum => [qw(none writethrough writeback unsafe directsync)],
 996         description => "The drive's cache mode",
 997         optional => 1,
 998     },
 999     format => get_standard_option('pve-qm-image-format'),
1000     size => {
1001         type => 'string',
1002         format => 'disk-size',
1003         format_description => 'DiskSize',
1004         description => "Disk size. This is purely informational and has no effect.",
1005         optional => 1,
1006     },
1007     backup => {
1008         type => 'boolean',
1009         description => "Whether the drive should be included when making backups.",
1010         optional => 1,
1011     },
1012     replicate => {
1013         type => 'boolean',
1014         description => 'Whether the drive should considered for replication jobs.',
1015         optional => 1,
1016         default => 1,
1017     },
1018     rerror => {
1019         type => 'string',
1020         enum => [qw(ignore report stop)],
1021         description => 'Read error action.',
1022         optional => 1,
1023     },
1024     werror => {
1025         type => 'string',
1026         enum => [qw(enospc ignore report stop)],
1027         description => 'Write error action.',
1028         optional => 1,
1029     },
1030     aio => {
1031         type => 'string',
1032         enum => [qw(native threads)],
1033         description => 'AIO type to use.',
1034         optional => 1,
1035     },
1036     discard => {
1037         type => 'string',
1038         enum => [qw(ignore on)],
1039         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1040         optional => 1,
1041     },
1042     detect_zeroes => {
1043         type => 'boolean',
1044         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1045         optional => 1,
1046     },
1047     serial => {
1048         type => 'string',
1049         format => 'urlencoded',
1050         format_description => 'serial',
1051         maxLength => 20*3, # *3 since it's %xx url enoded
1052         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1053         optional => 1,
1054     },
1055     shared => {
1056         type => 'boolean',
1057         description => 'Mark this locally-managed volume as available on all nodes',
1058         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1059         optional => 1,
1060         default => 0,
1061     }
1062 );
1063
1064 my %iothread_fmt = ( iothread => {
1065         type => 'boolean',
1066         description => "Whether to use iothreads for this drive",
1067         optional => 1,
1068 });
1069
1070 my %model_fmt = (
1071     model => {
1072         type => 'string',
1073         format => 'urlencoded',
1074         format_description => 'model',
1075         maxLength => 40*3, # *3 since it's %xx url enoded
1076         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1077         optional => 1,
1078     },
1079 );
1080
1081 my %queues_fmt = (
1082     queues => {
1083         type => 'integer',
1084         description => "Number of queues.",
1085         minimum => 2,
1086         optional => 1
1087     }
1088 );
1089
1090 my %scsiblock_fmt = (
1091     scsiblock => {
1092         type => 'boolean',
1093         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1094         optional => 1,
1095         default => 0,
1096     },
1097 );
1098
1099 my %ssd_fmt = (
1100     ssd => {
1101         type => 'boolean',
1102         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1103         optional => 1,
1104     },
1105 );
1106
1107 my %wwn_fmt = (
1108     wwn => {
1109         type => 'string',
1110         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1111         format_description => 'wwn',
1112         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1113         optional => 1,
1114     },
1115 );
1116
1117 my $add_throttle_desc = sub {
1118     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1119     my $d = {
1120         type => $type,
1121         format_description => $unit,
1122         description => "Maximum $what in $longunit.",
1123         optional => 1,
1124     };
1125     $d->{minimum} = $minimum if defined($minimum);
1126     $drivedesc_base{$key} = $d;
1127 };
1128 # throughput: (leaky bucket)
1129 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1130 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1131 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1132 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1138
1139 # pools: (pool of IO before throttling starts taking effect)
1140 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1141 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1142 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1143 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1144 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1145 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1146
1147 # burst lengths
1148 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1149 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1150 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1151 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1152 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1153 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1154
1155 # legacy support
1156 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1157 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1158 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1159 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1160
1161 my $ide_fmt = {
1162     %drivedesc_base,
1163     %model_fmt,
1164     %ssd_fmt,
1165     %wwn_fmt,
1166 };
1167 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1168
1169 my $idedesc = {
1170     optional => 1,
1171     type => 'string', format => $ide_fmt,
1172     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1173 };
1174 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1175
1176 my $scsi_fmt = {
1177     %drivedesc_base,
1178     %iothread_fmt,
1179     %queues_fmt,
1180     %scsiblock_fmt,
1181     %ssd_fmt,
1182     %wwn_fmt,
1183 };
1184 my $scsidesc = {
1185     optional => 1,
1186     type => 'string', format => $scsi_fmt,
1187     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1188 };
1189 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1190
1191 my $sata_fmt = {
1192     %drivedesc_base,
1193     %ssd_fmt,
1194     %wwn_fmt,
1195 };
1196 my $satadesc = {
1197     optional => 1,
1198     type => 'string', format => $sata_fmt,
1199     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1200 };
1201 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1202
1203 my $virtio_fmt = {
1204     %drivedesc_base,
1205     %iothread_fmt,
1206 };
1207 my $virtiodesc = {
1208     optional => 1,
1209     type => 'string', format => $virtio_fmt,
1210     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1211 };
1212 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1213
1214 my $alldrive_fmt = {
1215     %drivedesc_base,
1216     %iothread_fmt,
1217     %model_fmt,
1218     %queues_fmt,
1219     %scsiblock_fmt,
1220     %ssd_fmt,
1221     %wwn_fmt,
1222 };
1223
1224 my $efidisk_fmt = {
1225     volume => { alias => 'file' },
1226     file => {
1227         type => 'string',
1228         format => 'pve-volume-id-or-qm-path',
1229         default_key => 1,
1230         format_description => 'volume',
1231         description => "The drive's backing volume.",
1232     },
1233     format => get_standard_option('pve-qm-image-format'),
1234     size => {
1235         type => 'string',
1236         format => 'disk-size',
1237         format_description => 'DiskSize',
1238         description => "Disk size. This is purely informational and has no effect.",
1239         optional => 1,
1240     },
1241 };
1242
1243 my $efidisk_desc = {
1244     optional => 1,
1245     type => 'string', format => $efidisk_fmt,
1246     description => "Configure a Disk for storing EFI vars",
1247 };
1248
1249 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1250
1251 my $usb_fmt = {
1252     host => {
1253         default_key => 1,
1254         type => 'string', format => 'pve-qm-usb-device',
1255         format_description => 'HOSTUSBDEVICE|spice',
1256         description => <<EODESCR,
1257 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1258
1259  'bus-port(.port)*' (decimal numbers) or
1260  'vendor_id:product_id' (hexadeciaml numbers) or
1261  'spice'
1262
1263 You can use the 'lsusb -t' command to list existing usb devices.
1264
1265 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1266
1267 The value 'spice' can be used to add a usb redirection devices for spice.
1268 EODESCR
1269     },
1270     usb3 => {
1271         optional => 1,
1272         type => 'boolean',
1273         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1274         default => 0,
1275     },
1276 };
1277
1278 my $usbdesc = {
1279     optional => 1,
1280     type => 'string', format => $usb_fmt,
1281     description => "Configure an USB device (n is 0 to 4).",
1282 };
1283 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1284
1285 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1286 my $hostpci_fmt = {
1287     host => {
1288         default_key => 1,
1289         type => 'string',
1290         pattern => qr/$PCIRE(;$PCIRE)*/,
1291         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1292         description => <<EODESCR,
1293 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1294 of PCI virtual functions of the host. HOSTPCIID syntax is:
1295
1296 'bus:dev.func' (hexadecimal numbers)
1297
1298 You can us the 'lspci' command to list existing PCI devices.
1299 EODESCR
1300     },
1301     rombar => {
1302         type => 'boolean',
1303         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1304         optional => 1,
1305         default => 1,
1306     },
1307     romfile => {
1308         type => 'string',
1309         pattern => '[^,;]+',
1310         format_description => 'string',
1311         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1312         optional => 1,
1313     },
1314     pcie => {
1315         type => 'boolean',
1316         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1317         optional => 1,
1318         default => 0,
1319     },
1320     'x-vga' => {
1321         type => 'boolean',
1322         description =>  "Enable vfio-vga device support.",
1323         optional => 1,
1324         default => 0,
1325     },
1326     'mdev' => {
1327         type => 'string',
1328         format_description => 'string',
1329         pattern => '[^/\.:]+',
1330         optional => 1,
1331         description => <<EODESCR
1332 The type of mediated device to use.
1333 An instance of this type will be created on startup of the VM and
1334 will be cleaned up when the VM stops.
1335 EODESCR
1336     }
1337 };
1338 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1339
1340 my $hostpcidesc = {
1341         optional => 1,
1342         type => 'string', format => 'pve-qm-hostpci',
1343         description => "Map host PCI devices into guest.",
1344         verbose_description =>  <<EODESCR,
1345 Map host PCI devices into guest.
1346
1347 NOTE: This option allows direct access to host hardware. So it is no longer
1348 possible to migrate such machines - use with special care.
1349
1350 CAUTION: Experimental! User reported problems with this option.
1351 EODESCR
1352 };
1353 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1354
1355 my $serialdesc = {
1356         optional => 1,
1357         type => 'string',
1358         pattern => '(/dev/.+|socket)',
1359         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1360         verbose_description =>  <<EODESCR,
1361 Create a serial device inside the VM (n is 0 to 3), and pass through a
1362 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1363 host side (use 'qm terminal' to open a terminal connection).
1364
1365 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1366
1367 CAUTION: Experimental! User reported problems with this option.
1368 EODESCR
1369 };
1370
1371 my $paralleldesc= {
1372         optional => 1,
1373         type => 'string',
1374         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1375         description =>  "Map host parallel devices (n is 0 to 2).",
1376         verbose_description =>  <<EODESCR,
1377 Map host parallel devices (n is 0 to 2).
1378
1379 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1380
1381 CAUTION: Experimental! User reported problems with this option.
1382 EODESCR
1383 };
1384
1385 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1386     $confdesc->{"parallel$i"} = $paralleldesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1390     $confdesc->{"serial$i"} = $serialdesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1394     $confdesc->{"hostpci$i"} = $hostpcidesc;
1395 }
1396
1397 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1398     $drivename_hash->{"ide$i"} = 1;
1399     $confdesc->{"ide$i"} = $idedesc;
1400 }
1401
1402 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1403     $drivename_hash->{"sata$i"} = 1;
1404     $confdesc->{"sata$i"} = $satadesc;
1405 }
1406
1407 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1408     $drivename_hash->{"scsi$i"} = 1;
1409     $confdesc->{"scsi$i"} = $scsidesc ;
1410 }
1411
1412 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1413     $drivename_hash->{"virtio$i"} = 1;
1414     $confdesc->{"virtio$i"} = $virtiodesc;
1415 }
1416
1417 $drivename_hash->{efidisk0} = 1;
1418 $confdesc->{efidisk0} = $efidisk_desc;
1419
1420 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1421     $confdesc->{"usb$i"} = $usbdesc;
1422 }
1423
1424 my $unuseddesc = {
1425     optional => 1,
1426     type => 'string', format => 'pve-volume-id',
1427     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1428 };
1429
1430 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1431     $confdesc->{"unused$i"} = $unuseddesc;
1432 }
1433
1434 my $kvm_api_version = 0;
1435
1436 sub kvm_version {
1437     return $kvm_api_version if $kvm_api_version;
1438
1439     open my $fh, '<', '/dev/kvm'
1440         or return undef;
1441
1442     # 0xae00 => KVM_GET_API_VERSION
1443     $kvm_api_version = ioctl($fh, 0xae00, 0);
1444
1445     return $kvm_api_version;
1446 }
1447
1448 my $kvm_user_version;
1449
1450 sub kvm_user_version {
1451
1452     return $kvm_user_version if $kvm_user_version;
1453
1454     $kvm_user_version = 'unknown';
1455
1456     my $code = sub {
1457         my $line = shift;
1458         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1459             $kvm_user_version = $2;
1460         }
1461     };
1462
1463     eval { run_command("kvm -version", outfunc => $code); };
1464     warn $@ if $@;
1465
1466     return $kvm_user_version;
1467
1468 }
1469
1470 sub kernel_has_vhost_net {
1471     return -c '/dev/vhost-net';
1472 }
1473
1474 sub valid_drive_names {
1475     # order is important - used to autoselect boot disk
1476     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1477             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1478             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1479             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1480             'efidisk0');
1481 }
1482
1483 sub is_valid_drivename {
1484     my $dev = shift;
1485
1486     return defined($drivename_hash->{$dev});
1487 }
1488
1489 sub option_exists {
1490     my $key = shift;
1491     return defined($confdesc->{$key});
1492 }
1493
1494 sub nic_models {
1495     return $nic_model_list;
1496 }
1497
1498 sub os_list_description {
1499
1500     return {
1501         other => 'Other',
1502         wxp => 'Windows XP',
1503         w2k => 'Windows 2000',
1504         w2k3 =>, 'Windows 2003',
1505         w2k8 => 'Windows 2008',
1506         wvista => 'Windows Vista',
1507         win7 => 'Windows 7',
1508         win8 => 'Windows 8/2012',
1509         win10 => 'Windows 10/2016',
1510         l24 => 'Linux 2.4',
1511         l26 => 'Linux 2.6',
1512     };
1513 }
1514
1515 my $cdrom_path;
1516
1517 sub get_cdrom_path {
1518
1519     return  $cdrom_path if $cdrom_path;
1520
1521     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1522     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1523     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1524 }
1525
1526 sub get_iso_path {
1527     my ($storecfg, $vmid, $cdrom) = @_;
1528
1529     if ($cdrom eq 'cdrom') {
1530         return get_cdrom_path();
1531     } elsif ($cdrom eq 'none') {
1532         return '';
1533     } elsif ($cdrom =~ m|^/|) {
1534         return $cdrom;
1535     } else {
1536         return PVE::Storage::path($storecfg, $cdrom);
1537     }
1538 }
1539
1540 # try to convert old style file names to volume IDs
1541 sub filename_to_volume_id {
1542     my ($vmid, $file, $media) = @_;
1543
1544      if (!($file eq 'none' || $file eq 'cdrom' ||
1545           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1546
1547         return undef if $file =~ m|/|;
1548
1549         if ($media && $media eq 'cdrom') {
1550             $file = "local:iso/$file";
1551         } else {
1552             $file = "local:$vmid/$file";
1553         }
1554     }
1555
1556     return $file;
1557 }
1558
1559 sub verify_media_type {
1560     my ($opt, $vtype, $media) = @_;
1561
1562     return if !$media;
1563
1564     my $etype;
1565     if ($media eq 'disk') {
1566         $etype = 'images';
1567     } elsif ($media eq 'cdrom') {
1568         $etype = 'iso';
1569     } else {
1570         die "internal error";
1571     }
1572
1573     return if ($vtype eq $etype);
1574
1575     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1576 }
1577
1578 sub cleanup_drive_path {
1579     my ($opt, $storecfg, $drive) = @_;
1580
1581     # try to convert filesystem paths to volume IDs
1582
1583     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1584         ($drive->{file} !~ m|^/dev/.+|) &&
1585         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1586         ($drive->{file} !~ m/^\d+$/)) {
1587         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1588         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1589         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1590         verify_media_type($opt, $vtype, $drive->{media});
1591         $drive->{file} = $volid;
1592     }
1593
1594     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1595 }
1596
1597 sub parse_hotplug_features {
1598     my ($data) = @_;
1599
1600     my $res = {};
1601
1602     return $res if $data eq '0';
1603
1604     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1605
1606     foreach my $feature (PVE::Tools::split_list($data)) {
1607         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1608             $res->{$1} = 1;
1609         } else {
1610             die "invalid hotplug feature '$feature'\n";
1611         }
1612     }
1613     return $res;
1614 }
1615
1616 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1617 sub pve_verify_hotplug_features {
1618     my ($value, $noerr) = @_;
1619
1620     return $value if parse_hotplug_features($value);
1621
1622     return undef if $noerr;
1623
1624     die "unable to parse hotplug option\n";
1625 }
1626
1627 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1628 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1629 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1630 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1631 #        [,iothread=on][,serial=serial][,model=model]
1632
1633 sub parse_drive {
1634     my ($key, $data) = @_;
1635
1636     my ($interface, $index);
1637
1638     if ($key =~ m/^([^\d]+)(\d+)$/) {
1639         $interface = $1;
1640         $index = $2;
1641     } else {
1642         return undef;
1643     }
1644
1645     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1646                                      : $confdesc->{$key}->{format};
1647     if (!$desc) {
1648         warn "invalid drive key: $key\n";
1649         return undef;
1650     }
1651     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1652     return undef if !$res;
1653     $res->{interface} = $interface;
1654     $res->{index} = $index;
1655
1656     my $error = 0;
1657     foreach my $opt (qw(bps bps_rd bps_wr)) {
1658         if (my $bps = defined(delete $res->{$opt})) {
1659             if (defined($res->{"m$opt"})) {
1660                 warn "both $opt and m$opt specified\n";
1661                 ++$error;
1662                 next;
1663             }
1664             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1665         }
1666     }
1667
1668     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1669     for my $requirement (
1670         [mbps_max => 'mbps'],
1671         [mbps_rd_max => 'mbps_rd'],
1672         [mbps_wr_max => 'mbps_wr'],
1673         [miops_max => 'miops'],
1674         [miops_rd_max => 'miops_rd'],
1675         [miops_wr_max => 'miops_wr'],
1676         [bps_max_length => 'mbps_max'],
1677         [bps_rd_max_length => 'mbps_rd_max'],
1678         [bps_wr_max_length => 'mbps_wr_max'],
1679         [iops_max_length => 'iops_max'],
1680         [iops_rd_max_length => 'iops_rd_max'],
1681         [iops_wr_max_length => 'iops_wr_max']) {
1682         my ($option, $requires) = @$requirement;
1683         if ($res->{$option} && !$res->{$requires}) {
1684             warn "$option requires $requires\n";
1685             ++$error;
1686         }
1687     }
1688
1689     return undef if $error;
1690
1691     return undef if $res->{mbps_rd} && $res->{mbps};
1692     return undef if $res->{mbps_wr} && $res->{mbps};
1693     return undef if $res->{iops_rd} && $res->{iops};
1694     return undef if $res->{iops_wr} && $res->{iops};
1695
1696     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1697         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1698         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1699         return undef if $res->{interface} eq 'virtio';
1700     }
1701
1702     if (my $size = $res->{size}) {
1703         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1704     }
1705
1706     return $res;
1707 }
1708
1709 sub print_drive {
1710     my ($vmid, $drive) = @_;
1711     my $data = { %$drive };
1712     delete $data->{$_} for qw(index interface);
1713     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1714 }
1715
1716 sub scsi_inquiry {
1717     my($fh, $noerr) = @_;
1718
1719     my $SG_IO = 0x2285;
1720     my $SG_GET_VERSION_NUM = 0x2282;
1721
1722     my $versionbuf = "\x00" x 8;
1723     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1724     if (!$ret) {
1725         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1726         return undef;
1727     }
1728     my $version = unpack("I", $versionbuf);
1729     if ($version < 30000) {
1730         die "scsi generic interface too old\n"  if !$noerr;
1731         return undef;
1732     }
1733
1734     my $buf = "\x00" x 36;
1735     my $sensebuf = "\x00" x 8;
1736     my $cmd = pack("C x3 C x1", 0x12, 36);
1737
1738     # see /usr/include/scsi/sg.h
1739     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1740
1741     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1742                       length($sensebuf), 0, length($buf), $buf,
1743                       $cmd, $sensebuf, 6000);
1744
1745     $ret = ioctl($fh, $SG_IO, $packet);
1746     if (!$ret) {
1747         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1748         return undef;
1749     }
1750
1751     my @res = unpack($sg_io_hdr_t, $packet);
1752     if ($res[17] || $res[18]) {
1753         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1754         return undef;
1755     }
1756
1757     my $res = {};
1758     (my $byte0, my $byte1, $res->{vendor},
1759      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1760
1761     $res->{removable} = $byte1 & 128 ? 1 : 0;
1762     $res->{type} = $byte0 & 31;
1763
1764     return $res;
1765 }
1766
1767 sub path_is_scsi {
1768     my ($path) = @_;
1769
1770     my $fh = IO::File->new("+<$path") || return undef;
1771     my $res = scsi_inquiry($fh, 1);
1772     close($fh);
1773
1774     return $res;
1775 }
1776
1777 sub machine_type_is_q35 {
1778     my ($conf) = @_;
1779
1780     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1781 }
1782
1783 sub print_tabletdevice_full {
1784     my ($conf, $arch) = @_;
1785
1786     my $q35 = machine_type_is_q35($conf);
1787
1788     # we use uhci for old VMs because tablet driver was buggy in older qemu
1789     my $usbbus;
1790     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1791         $usbbus = 'ehci';
1792     } else {
1793         $usbbus = 'uhci';
1794     }
1795
1796     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1797 }
1798
1799 sub print_keyboarddevice_full {
1800     my ($conf, $arch, $machine) = @_;
1801
1802     return undef if $arch ne 'aarch64';
1803
1804     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1805 }
1806
1807 sub print_drivedevice_full {
1808     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1809
1810     my $device = '';
1811     my $maxdev = 0;
1812
1813     if ($drive->{interface} eq 'virtio') {
1814         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1815         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1816         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1817     } elsif ($drive->{interface} eq 'scsi') {
1818
1819         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1820         my $unit = $drive->{index} % $maxdev;
1821         my $devicetype = 'hd';
1822         my $path = '';
1823         if (drive_is_cdrom($drive)) {
1824             $devicetype = 'cd';
1825         } else {
1826             if ($drive->{file} =~ m|^/|) {
1827                 $path = $drive->{file};
1828                 if (my $info = path_is_scsi($path)) {
1829                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1830                         $devicetype = 'block';
1831                     } elsif ($info->{type} == 1) { # tape
1832                         $devicetype = 'generic';
1833                     }
1834                 }
1835             } else {
1836                  $path = PVE::Storage::path($storecfg, $drive->{file});
1837             }
1838
1839             if($path =~ m/^iscsi\:\/\//){
1840                 $devicetype = 'generic';
1841             }
1842         }
1843
1844         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1845             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1846         } else {
1847             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1848         }
1849
1850         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1851             $device .= ",rotation_rate=1";
1852         }
1853         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1854
1855     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1856         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1857         my $controller = int($drive->{index} / $maxdev);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1860
1861         $device = "ide-$devicetype";
1862         if ($drive->{interface} eq 'ide') {
1863             $device .= ",bus=ide.$controller,unit=$unit";
1864         } else {
1865             $device .= ",bus=ahci$controller.$unit";
1866         }
1867         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1868
1869         if ($devicetype eq 'hd') {
1870             if (my $model = $drive->{model}) {
1871                 $model = URI::Escape::uri_unescape($model);
1872                 $device .= ",model=$model";
1873             }
1874             if ($drive->{ssd}) {
1875                 $device .= ",rotation_rate=1";
1876             }
1877         }
1878         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1879     } elsif ($drive->{interface} eq 'usb') {
1880         die "implement me";
1881         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1882     } else {
1883         die "unsupported interface type";
1884     }
1885
1886     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1887
1888     if (my $serial = $drive->{serial}) {
1889         $serial = URI::Escape::uri_unescape($serial);
1890         $device .= ",serial=$serial";
1891     }
1892
1893
1894     return $device;
1895 }
1896
1897 sub get_initiator_name {
1898     my $initiator;
1899
1900     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1901     while (defined(my $line = <$fh>)) {
1902         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1903         $initiator = $1;
1904         last;
1905     }
1906     $fh->close();
1907
1908     return $initiator;
1909 }
1910
1911 sub print_drive_full {
1912     my ($storecfg, $vmid, $drive) = @_;
1913
1914     my $path;
1915     my $volid = $drive->{file};
1916     my $format;
1917
1918     if (drive_is_cdrom($drive)) {
1919         $path = get_iso_path($storecfg, $vmid, $volid);
1920     } else {
1921         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1922         if ($storeid) {
1923             $path = PVE::Storage::path($storecfg, $volid);
1924             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1925             $format = qemu_img_format($scfg, $volname);
1926         } else {
1927             $path = $volid;
1928             $format = "raw";
1929         }
1930    }
1931
1932     my $opts = '';
1933     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1934     foreach my $o (@qemu_drive_options) {
1935         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1936     }
1937
1938     # snapshot only accepts on|off
1939     if (defined($drive->{snapshot})) {
1940         my $v = $drive->{snapshot} ? 'on' : 'off';
1941         $opts .= ",snapshot=$v";
1942     }
1943
1944     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1945         my ($dir, $qmpname) = @$type;
1946         if (my $v = $drive->{"mbps$dir"}) {
1947             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1948         }
1949         if (my $v = $drive->{"mbps${dir}_max"}) {
1950             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1951         }
1952         if (my $v = $drive->{"bps${dir}_max_length"}) {
1953             $opts .= ",throttling.bps$qmpname-max-length=$v";
1954         }
1955         if (my $v = $drive->{"iops${dir}"}) {
1956             $opts .= ",throttling.iops$qmpname=$v";
1957         }
1958         if (my $v = $drive->{"iops${dir}_max"}) {
1959             $opts .= ",throttling.iops$qmpname-max=$v";
1960         }
1961         if (my $v = $drive->{"iops${dir}_max_length"}) {
1962             $opts .= ",throttling.iops$qmpname-max-length=$v";
1963         }
1964     }
1965
1966     $opts .= ",format=$format" if $format && !$drive->{format};
1967
1968     my $cache_direct = 0;
1969
1970     if (my $cache = $drive->{cache}) {
1971         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1972     } elsif (!drive_is_cdrom($drive)) {
1973         $opts .= ",cache=none";
1974         $cache_direct = 1;
1975     }
1976
1977     # aio native works only with O_DIRECT
1978     if (!$drive->{aio}) {
1979         if($cache_direct) {
1980             $opts .= ",aio=native";
1981         } else {
1982             $opts .= ",aio=threads";
1983         }
1984     }
1985
1986     if (!drive_is_cdrom($drive)) {
1987         my $detectzeroes;
1988         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1989             $detectzeroes = 'off';
1990         } elsif ($drive->{discard}) {
1991             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1992         } else {
1993             # This used to be our default with discard not being specified:
1994             $detectzeroes = 'on';
1995         }
1996         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1997     }
1998
1999     my $pathinfo = $path ? "file=$path," : '';
2000
2001     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2002 }
2003
2004 sub print_netdevice_full {
2005     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2006
2007     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2008
2009     my $device = $net->{model};
2010     if ($net->{model} eq 'virtio') {
2011          $device = 'virtio-net-pci';
2012      };
2013
2014     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2015     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2016     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2017         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2018         my $vectors = $net->{queues} * 2 + 2;
2019         $tmpstr .= ",vectors=$vectors,mq=on";
2020     }
2021     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2022
2023     if ($use_old_bios_files) {
2024         my $romfile;
2025         if ($device eq 'virtio-net-pci') {
2026             $romfile = 'pxe-virtio.rom';
2027         } elsif ($device eq 'e1000') {
2028             $romfile = 'pxe-e1000.rom';
2029         } elsif ($device eq 'ne2k') {
2030             $romfile = 'pxe-ne2k_pci.rom';
2031         } elsif ($device eq 'pcnet') {
2032             $romfile = 'pxe-pcnet.rom';
2033         } elsif ($device eq 'rtl8139') {
2034             $romfile = 'pxe-rtl8139.rom';
2035         }
2036         $tmpstr .= ",romfile=$romfile" if $romfile;
2037     }
2038
2039     return $tmpstr;
2040 }
2041
2042 sub print_netdev_full {
2043     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2044
2045     my $i = '';
2046     if ($netid =~ m/^net(\d+)$/) {
2047         $i = int($1);
2048     }
2049
2050     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2051
2052     my $ifname = "tap${vmid}i$i";
2053
2054     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2055     die "interface name '$ifname' is too long (max 15 character)\n"
2056         if length($ifname) >= 16;
2057
2058     my $vhostparam = '';
2059     if (is_native($arch)) {
2060         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2061     }
2062
2063     my $vmname = $conf->{name} || "vm$vmid";
2064
2065     my $netdev = "";
2066     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2067
2068     if ($net->{bridge}) {
2069         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2070     } else {
2071         $netdev = "type=user,id=$netid,hostname=$vmname";
2072     }
2073
2074     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2075
2076     return $netdev;
2077 }
2078
2079
2080 sub print_cpu_device {
2081     my ($conf, $id) = @_;
2082
2083     my $kvm = $conf->{kvm} // 1;
2084     my $cpu = $kvm ? "kvm64" : "qemu64";
2085     if (my $cputype = $conf->{cpu}) {
2086         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2087             or die "Cannot parse cpu description: $cputype\n";
2088         $cpu = $cpuconf->{cputype};
2089     }
2090
2091     my $cores = $conf->{cores} || 1;
2092
2093     my $current_core = ($id - 1) % $cores;
2094     my $current_socket = int(($id - 1 - $current_core)/$cores);
2095
2096     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2097 }
2098
2099 my $vga_map = {
2100     'cirrus' => 'cirrus-vga',
2101     'std' => 'VGA',
2102     'vmware' => 'vmware-svga',
2103     'virtio' => 'virtio-vga',
2104 };
2105
2106 sub print_vga_device {
2107     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2108
2109     my $type = $vga_map->{$vga->{type}};
2110     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2111         $type = 'virtio-gpu';
2112     }
2113     my $vgamem_mb = $vga->{memory};
2114     if ($qxlnum) {
2115         $type = $id ? 'qxl' : 'qxl-vga';
2116     }
2117     die "no devicetype for $vga->{type}\n" if !$type;
2118
2119     my $memory = "";
2120     if ($vgamem_mb) {
2121         if ($vga->{type} eq 'virtio') {
2122             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2123             $memory = ",max_hostmem=$bytes";
2124         } elsif ($qxlnum) {
2125             # from https://www.spice-space.org/multiple-monitors.html
2126             $memory = ",vgamem_mb=$vga->{memory}";
2127             my $ram = $vgamem_mb * 4;
2128             my $vram = $vgamem_mb * 2;
2129             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2130         } else {
2131             $memory = ",vgamem_mb=$vga->{memory}";
2132         }
2133     } elsif ($qxlnum && $id) {
2134         $memory = ",ram_size=67108864,vram_size=33554432";
2135     }
2136
2137     my $q35 = machine_type_is_q35($conf);
2138     my $vgaid = "vga" . ($id // '');
2139     my $pciaddr;
2140
2141     if ($q35 && $vgaid eq 'vga') {
2142         # the first display uses pcie.0 bus on q35 machines
2143         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2144     } else {
2145         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2146     }
2147
2148     return "$type,id=${vgaid}${memory}${pciaddr}";
2149 }
2150
2151 sub drive_is_cloudinit {
2152     my ($drive) = @_;
2153     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2154 }
2155
2156 sub drive_is_cdrom {
2157     my ($drive, $exclude_cloudinit) = @_;
2158
2159     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2160
2161     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2162
2163 }
2164
2165 sub parse_number_sets {
2166     my ($set) = @_;
2167     my $res = [];
2168     foreach my $part (split(/;/, $set)) {
2169         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2170             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2171             push @$res, [ $1, $2 ];
2172         } else {
2173             die "invalid range: $part\n";
2174         }
2175     }
2176     return $res;
2177 }
2178
2179 sub parse_numa {
2180     my ($data) = @_;
2181
2182     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2183     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2184     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2185     return $res;
2186 }
2187
2188 sub parse_hostpci {
2189     my ($value) = @_;
2190
2191     return undef if !$value;
2192
2193     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2194
2195     my @idlist = split(/;/, $res->{host});
2196     delete $res->{host};
2197     foreach my $id (@idlist) {
2198         if ($id =~ m/\./) { # full id 00:00.1
2199             push @{$res->{pciid}}, {
2200                 id => $id,
2201             };
2202         } else { # partial id 00:00
2203             $res->{pciid} = PVE::SysFSTools::lspci($id);
2204         }
2205     }
2206     return $res;
2207 }
2208
2209 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2210 sub parse_net {
2211     my ($data) = @_;
2212
2213     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2214     if ($@) {
2215         warn $@;
2216         return undef;
2217     }
2218     if (!defined($res->{macaddr})) {
2219         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2220         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2221     }
2222     return $res;
2223 }
2224
2225 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2226 sub parse_ipconfig {
2227     my ($data) = @_;
2228
2229     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2230     if ($@) {
2231         warn $@;
2232         return undef;
2233     }
2234
2235     if ($res->{gw} && !$res->{ip}) {
2236         warn 'gateway specified without specifying an IP address';
2237         return undef;
2238     }
2239     if ($res->{gw6} && !$res->{ip6}) {
2240         warn 'IPv6 gateway specified without specifying an IPv6 address';
2241         return undef;
2242     }
2243     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2244         warn 'gateway specified together with DHCP';
2245         return undef;
2246     }
2247     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2248         # gw6 + auto/dhcp
2249         warn "IPv6 gateway specified together with $res->{ip6} address";
2250         return undef;
2251     }
2252
2253     if (!$res->{ip} && !$res->{ip6}) {
2254         return { ip => 'dhcp', ip6 => 'dhcp' };
2255     }
2256
2257     return $res;
2258 }
2259
2260 sub print_net {
2261     my $net = shift;
2262
2263     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2264 }
2265
2266 sub add_random_macs {
2267     my ($settings) = @_;
2268
2269     foreach my $opt (keys %$settings) {
2270         next if $opt !~ m/^net(\d+)$/;
2271         my $net = parse_net($settings->{$opt});
2272         next if !$net;
2273         $settings->{$opt} = print_net($net);
2274     }
2275 }
2276
2277 sub vm_is_volid_owner {
2278     my ($storecfg, $vmid, $volid) = @_;
2279
2280     if ($volid !~  m|^/|) {
2281         my ($path, $owner);
2282         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2283         if ($owner && ($owner == $vmid)) {
2284             return 1;
2285         }
2286     }
2287
2288     return undef;
2289 }
2290
2291 sub split_flagged_list {
2292     my $text = shift || '';
2293     $text =~ s/[,;]/ /g;
2294     $text =~ s/^\s+//;
2295     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2296 }
2297
2298 sub join_flagged_list {
2299     my ($how, $lst) = @_;
2300     join $how, map { $lst->{$_} . $_ } keys %$lst;
2301 }
2302
2303 sub vmconfig_delete_pending_option {
2304     my ($conf, $key, $force) = @_;
2305
2306     delete $conf->{pending}->{$key};
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     $pending_delete_hash->{$key} = $force ? '!' : '';
2309     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2310 }
2311
2312 sub vmconfig_undelete_pending_option {
2313     my ($conf, $key) = @_;
2314
2315     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2316     delete $pending_delete_hash->{$key};
2317
2318     if (%$pending_delete_hash) {
2319         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2320     } else {
2321         delete $conf->{pending}->{delete};
2322     }
2323 }
2324
2325 sub vmconfig_register_unused_drive {
2326     my ($storecfg, $vmid, $conf, $drive) = @_;
2327
2328     if (drive_is_cloudinit($drive)) {
2329         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2330         warn $@ if $@;
2331     } elsif (!drive_is_cdrom($drive)) {
2332         my $volid = $drive->{file};
2333         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2334             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2335         }
2336     }
2337 }
2338
2339 sub vmconfig_cleanup_pending {
2340     my ($conf) = @_;
2341
2342     # remove pending changes when nothing changed
2343     my $changes;
2344     foreach my $opt (keys %{$conf->{pending}}) {
2345         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2346             $changes = 1;
2347             delete $conf->{pending}->{$opt};
2348         }
2349     }
2350
2351     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2352     my $pending_delete_hash = {};
2353     while (my ($opt, $force) = each %$current_delete_hash) {
2354         if (defined($conf->{$opt})) {
2355             $pending_delete_hash->{$opt} = $force;
2356         } else {
2357             $changes = 1;
2358         }
2359     }
2360
2361     if (%$pending_delete_hash) {
2362         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2363     } else {
2364         delete $conf->{pending}->{delete};
2365     }
2366
2367     return $changes;
2368 }
2369
2370 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2371 my $smbios1_fmt = {
2372     uuid => {
2373         type => 'string',
2374         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2375         format_description => 'UUID',
2376         description => "Set SMBIOS1 UUID.",
2377         optional => 1,
2378     },
2379     version => {
2380         type => 'string',
2381         pattern => '\S+',
2382         format_description => 'string',
2383         description => "Set SMBIOS1 version.",
2384         optional => 1,
2385     },
2386     serial => {
2387         type => 'string',
2388         pattern => '\S+',
2389         format_description => 'string',
2390         description => "Set SMBIOS1 serial number.",
2391         optional => 1,
2392     },
2393     manufacturer => {
2394         type => 'string',
2395         pattern => '\S+',
2396         format_description => 'string',
2397         description => "Set SMBIOS1 manufacturer.",
2398         optional => 1,
2399     },
2400     product => {
2401         type => 'string',
2402         pattern => '\S+',
2403         format_description => 'string',
2404         description => "Set SMBIOS1 product ID.",
2405         optional => 1,
2406     },
2407     sku => {
2408         type => 'string',
2409         pattern => '\S+',
2410         format_description => 'string',
2411         description => "Set SMBIOS1 SKU string.",
2412         optional => 1,
2413     },
2414     family => {
2415         type => 'string',
2416         pattern => '\S+',
2417         format_description => 'string',
2418         description => "Set SMBIOS1 family string.",
2419         optional => 1,
2420     },
2421 };
2422
2423 sub parse_smbios1 {
2424     my ($data) = @_;
2425
2426     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2427     warn $@ if $@;
2428     return $res;
2429 }
2430
2431 sub print_smbios1 {
2432     my ($smbios1) = @_;
2433     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2434 }
2435
2436 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2437
2438 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2439 sub verify_bootdisk {
2440     my ($value, $noerr) = @_;
2441
2442     return $value if is_valid_drivename($value);
2443
2444     return undef if $noerr;
2445
2446     die "invalid boot disk '$value'\n";
2447 }
2448
2449 sub parse_watchdog {
2450     my ($value) = @_;
2451
2452     return undef if !$value;
2453
2454     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2455     warn $@ if $@;
2456     return $res;
2457 }
2458
2459 sub parse_guest_agent {
2460     my ($value) = @_;
2461
2462     return {} if !defined($value->{agent});
2463
2464     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2465     warn $@ if $@;
2466
2467     # if the agent is disabled ignore the other potentially set properties
2468     return {} if !$res->{enabled};
2469     return $res;
2470 }
2471
2472 sub parse_vga {
2473     my ($value) = @_;
2474
2475     return {} if !$value;
2476     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2477     warn $@ if $@;
2478     return $res;
2479 }
2480
2481 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2482 sub verify_usb_device {
2483     my ($value, $noerr) = @_;
2484
2485     return $value if parse_usb_device($value);
2486
2487     return undef if $noerr;
2488
2489     die "unable to parse usb device\n";
2490 }
2491
2492 # add JSON properties for create and set function
2493 sub json_config_properties {
2494     my $prop = shift;
2495
2496     foreach my $opt (keys %$confdesc) {
2497         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2498         $prop->{$opt} = $confdesc->{$opt};
2499     }
2500
2501     return $prop;
2502 }
2503
2504 # return copy of $confdesc_cloudinit to generate documentation
2505 sub cloudinit_config_properties {
2506
2507     return dclone($confdesc_cloudinit);
2508 }
2509
2510 sub check_type {
2511     my ($key, $value) = @_;
2512
2513     die "unknown setting '$key'\n" if !$confdesc->{$key};
2514
2515     my $type = $confdesc->{$key}->{type};
2516
2517     if (!defined($value)) {
2518         die "got undefined value\n";
2519     }
2520
2521     if ($value =~ m/[\n\r]/) {
2522         die "property contains a line feed\n";
2523     }
2524
2525     if ($type eq 'boolean') {
2526         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2527         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2528         die "type check ('boolean') failed - got '$value'\n";
2529     } elsif ($type eq 'integer') {
2530         return int($1) if $value =~ m/^(\d+)$/;
2531         die "type check ('integer') failed - got '$value'\n";
2532     } elsif ($type eq 'number') {
2533         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2534         die "type check ('number') failed - got '$value'\n";
2535     } elsif ($type eq 'string') {
2536         if (my $fmt = $confdesc->{$key}->{format}) {
2537             PVE::JSONSchema::check_format($fmt, $value);
2538             return $value;
2539         }
2540         $value =~ s/^\"(.*)\"$/$1/;
2541         return $value;
2542     } else {
2543         die "internal error"
2544     }
2545 }
2546
2547 sub touch_config {
2548     my ($vmid) = @_;
2549
2550     my $conf = PVE::QemuConfig->config_file($vmid);
2551     utime undef, undef, $conf;
2552 }
2553
2554 sub destroy_vm {
2555     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2556
2557     my $conffile = PVE::QemuConfig->config_file($vmid);
2558
2559     my $conf = PVE::QemuConfig->load_config($vmid);
2560
2561     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2562
2563     if ($conf->{template}) {
2564         # check if any base image is still used by a linked clone
2565         foreach_drive($conf, sub {
2566                 my ($ds, $drive) = @_;
2567
2568                 return if drive_is_cdrom($drive);
2569
2570                 my $volid = $drive->{file};
2571
2572                 return if !$volid || $volid =~ m|^/|;
2573
2574                 die "base volume '$volid' is still in use by linked cloned\n"
2575                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2576
2577         });
2578     }
2579
2580     # only remove disks owned by this VM
2581     foreach_drive($conf, sub {
2582         my ($ds, $drive) = @_;
2583
2584         return if drive_is_cdrom($drive, 1);
2585
2586         my $volid = $drive->{file};
2587
2588         return if !$volid || $volid =~ m|^/|;
2589
2590         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2591         return if !$path || !$owner || ($owner != $vmid);
2592
2593         eval {
2594             PVE::Storage::vdisk_free($storecfg, $volid);
2595         };
2596         warn "Could not remove disk '$volid', check manually: $@" if $@;
2597
2598     });
2599
2600     if ($keep_empty_config) {
2601         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2602     } else {
2603         unlink $conffile;
2604     }
2605
2606     # also remove unused disk
2607     eval {
2608         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2609
2610         eval {
2611             PVE::Storage::foreach_volid($dl, sub {
2612                 my ($volid, $sid, $volname, $d) = @_;
2613                 PVE::Storage::vdisk_free($storecfg, $volid);
2614             });
2615         };
2616         warn $@ if $@;
2617
2618     };
2619     warn $@ if $@;
2620 }
2621
2622 sub parse_vm_config {
2623     my ($filename, $raw) = @_;
2624
2625     return undef if !defined($raw);
2626
2627     my $res = {
2628         digest => Digest::SHA::sha1_hex($raw),
2629         snapshots => {},
2630         pending => {},
2631     };
2632
2633     $filename =~ m|/qemu-server/(\d+)\.conf$|
2634         || die "got strange filename '$filename'";
2635
2636     my $vmid = $1;
2637
2638     my $conf = $res;
2639     my $descr;
2640     my $section = '';
2641
2642     my @lines = split(/\n/, $raw);
2643     foreach my $line (@lines) {
2644         next if $line =~ m/^\s*$/;
2645
2646         if ($line =~ m/^\[PENDING\]\s*$/i) {
2647             $section = 'pending';
2648             if (defined($descr)) {
2649                 $descr =~ s/\s+$//;
2650                 $conf->{description} = $descr;
2651             }
2652             $descr = undef;
2653             $conf = $res->{$section} = {};
2654             next;
2655
2656         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2657             $section = $1;
2658             if (defined($descr)) {
2659                 $descr =~ s/\s+$//;
2660                 $conf->{description} = $descr;
2661             }
2662             $descr = undef;
2663             $conf = $res->{snapshots}->{$section} = {};
2664             next;
2665         }
2666
2667         if ($line =~ m/^\#(.*)\s*$/) {
2668             $descr = '' if !defined($descr);
2669             $descr .= PVE::Tools::decode_text($1) . "\n";
2670             next;
2671         }
2672
2673         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2674             $descr = '' if !defined($descr);
2675             $descr .= PVE::Tools::decode_text($2);
2676         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2677             $conf->{snapstate} = $1;
2678         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2679             my $key = $1;
2680             my $value = $2;
2681             $conf->{$key} = $value;
2682         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2683             my $value = $1;
2684             if ($section eq 'pending') {
2685                 $conf->{delete} = $value; # we parse this later
2686             } else {
2687                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2688             }
2689         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2690             my $key = $1;
2691             my $value = $2;
2692             eval { $value = check_type($key, $value); };
2693             if ($@) {
2694                 warn "vm $vmid - unable to parse value of '$key' - $@";
2695             } else {
2696                 $key = 'ide2' if $key eq 'cdrom';
2697                 my $fmt = $confdesc->{$key}->{format};
2698                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2699                     my $v = parse_drive($key, $value);
2700                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2701                         $v->{file} = $volid;
2702                         $value = print_drive($vmid, $v);
2703                     } else {
2704                         warn "vm $vmid - unable to parse value of '$key'\n";
2705                         next;
2706                     }
2707                 }
2708
2709                 $conf->{$key} = $value;
2710             }
2711         }
2712     }
2713
2714     if (defined($descr)) {
2715         $descr =~ s/\s+$//;
2716         $conf->{description} = $descr;
2717     }
2718     delete $res->{snapstate}; # just to be sure
2719
2720     return $res;
2721 }
2722
2723 sub write_vm_config {
2724     my ($filename, $conf) = @_;
2725
2726     delete $conf->{snapstate}; # just to be sure
2727
2728     if ($conf->{cdrom}) {
2729         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2730         $conf->{ide2} = $conf->{cdrom};
2731         delete $conf->{cdrom};
2732     }
2733
2734     # we do not use 'smp' any longer
2735     if ($conf->{sockets}) {
2736         delete $conf->{smp};
2737     } elsif ($conf->{smp}) {
2738         $conf->{sockets} = $conf->{smp};
2739         delete $conf->{cores};
2740         delete $conf->{smp};
2741     }
2742
2743     my $used_volids = {};
2744
2745     my $cleanup_config = sub {
2746         my ($cref, $pending, $snapname) = @_;
2747
2748         foreach my $key (keys %$cref) {
2749             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2750                 $key eq 'snapstate' || $key eq 'pending';
2751             my $value = $cref->{$key};
2752             if ($key eq 'delete') {
2753                 die "propertry 'delete' is only allowed in [PENDING]\n"
2754                     if !$pending;
2755                 # fixme: check syntax?
2756                 next;
2757             }
2758             eval { $value = check_type($key, $value); };
2759             die "unable to parse value of '$key' - $@" if $@;
2760
2761             $cref->{$key} = $value;
2762
2763             if (!$snapname && is_valid_drivename($key)) {
2764                 my $drive = parse_drive($key, $value);
2765                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2766             }
2767         }
2768     };
2769
2770     &$cleanup_config($conf);
2771
2772     &$cleanup_config($conf->{pending}, 1);
2773
2774     foreach my $snapname (keys %{$conf->{snapshots}}) {
2775         die "internal error" if $snapname eq 'pending';
2776         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2777     }
2778
2779     # remove 'unusedX' settings if we re-add a volume
2780     foreach my $key (keys %$conf) {
2781         my $value = $conf->{$key};
2782         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2783             delete $conf->{$key};
2784         }
2785     }
2786
2787     my $generate_raw_config = sub {
2788         my ($conf, $pending) = @_;
2789
2790         my $raw = '';
2791
2792         # add description as comment to top of file
2793         if (defined(my $descr = $conf->{description})) {
2794             if ($descr) {
2795                 foreach my $cl (split(/\n/, $descr)) {
2796                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2797                 }
2798             } else {
2799                 $raw .= "#\n" if $pending;
2800             }
2801         }
2802
2803         foreach my $key (sort keys %$conf) {
2804             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2805             $raw .= "$key: $conf->{$key}\n";
2806         }
2807         return $raw;
2808     };
2809
2810     my $raw = &$generate_raw_config($conf);
2811
2812     if (scalar(keys %{$conf->{pending}})){
2813         $raw .= "\n[PENDING]\n";
2814         $raw .= &$generate_raw_config($conf->{pending}, 1);
2815     }
2816
2817     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2818         $raw .= "\n[$snapname]\n";
2819         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2820     }
2821
2822     return $raw;
2823 }
2824
2825 sub load_defaults {
2826
2827     my $res = {};
2828
2829     # we use static defaults from our JSON schema configuration
2830     foreach my $key (keys %$confdesc) {
2831         if (defined(my $default = $confdesc->{$key}->{default})) {
2832             $res->{$key} = $default;
2833         }
2834     }
2835
2836     return $res;
2837 }
2838
2839 sub config_list {
2840     my $vmlist = PVE::Cluster::get_vmlist();
2841     my $res = {};
2842     return $res if !$vmlist || !$vmlist->{ids};
2843     my $ids = $vmlist->{ids};
2844
2845     foreach my $vmid (keys %$ids) {
2846         my $d = $ids->{$vmid};
2847         next if !$d->{node} || $d->{node} ne $nodename;
2848         next if !$d->{type} || $d->{type} ne 'qemu';
2849         $res->{$vmid}->{exists} = 1;
2850     }
2851     return $res;
2852 }
2853
2854 # test if VM uses local resources (to prevent migration)
2855 sub check_local_resources {
2856     my ($conf, $noerr) = @_;
2857
2858     my $loc_res = 0;
2859
2860     $loc_res = 1 if $conf->{hostusb}; # old syntax
2861     $loc_res = 1 if $conf->{hostpci}; # old syntax
2862
2863     $loc_res = 1 if $conf->{ivshmem};
2864
2865     foreach my $k (keys %$conf) {
2866         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2867         # sockets are safe: they will recreated be on the target side post-migrate
2868         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2869         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2870     }
2871
2872     die "VM uses local resources\n" if $loc_res && !$noerr;
2873
2874     return $loc_res;
2875 }
2876
2877 # check if used storages are available on all nodes (use by migrate)
2878 sub check_storage_availability {
2879     my ($storecfg, $conf, $node) = @_;
2880
2881     foreach_drive($conf, sub {
2882         my ($ds, $drive) = @_;
2883
2884         my $volid = $drive->{file};
2885         return if !$volid;
2886
2887         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2888         return if !$sid;
2889
2890         # check if storage is available on both nodes
2891         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2892         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2893    });
2894 }
2895
2896 # list nodes where all VM images are available (used by has_feature API)
2897 sub shared_nodes {
2898     my ($conf, $storecfg) = @_;
2899
2900     my $nodelist = PVE::Cluster::get_nodelist();
2901     my $nodehash = { map { $_ => 1 } @$nodelist };
2902     my $nodename = PVE::INotify::nodename();
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913             if ($scfg->{disable}) {
2914                 $nodehash = {};
2915             } elsif (my $avail = $scfg->{nodes}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if !$avail->{$node};
2918                 }
2919             } elsif (!$scfg->{shared}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     delete $nodehash->{$node} if $node ne $nodename
2922                 }
2923             }
2924         }
2925     });
2926
2927     return $nodehash
2928 }
2929
2930 sub check_cmdline {
2931     my ($pidfile, $pid) = @_;
2932
2933     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2934     if (defined($fh)) {
2935         my $line = <$fh>;
2936         $fh->close;
2937         return undef if !$line;
2938         my @param = split(/\0/, $line);
2939
2940         my $cmd = $param[0];
2941         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2942
2943         for (my $i = 0; $i < scalar (@param); $i++) {
2944             my $p = $param[$i];
2945             next if !$p;
2946             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2947                 my $p = $param[$i+1];
2948                 return 1 if $p && ($p eq $pidfile);
2949                 return undef;
2950             }
2951         }
2952     }
2953     return undef;
2954 }
2955
2956 sub check_running {
2957     my ($vmid, $nocheck, $node) = @_;
2958
2959     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2960
2961     die "unable to find configuration file for VM $vmid - no such machine\n"
2962         if !$nocheck && ! -f $filename;
2963
2964     my $pidfile = pidfile_name($vmid);
2965
2966     if (my $fd = IO::File->new("<$pidfile")) {
2967         my $st = stat($fd);
2968         my $line = <$fd>;
2969         close($fd);
2970
2971         my $mtime = $st->mtime;
2972         if ($mtime > time()) {
2973             warn "file '$filename' modified in future\n";
2974         }
2975
2976         if ($line =~ m/^(\d+)$/) {
2977             my $pid = $1;
2978             if (check_cmdline($pidfile, $pid)) {
2979                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2980                     return $pid;
2981                 }
2982             }
2983         }
2984     }
2985
2986     return undef;
2987 }
2988
2989 sub vzlist {
2990
2991     my $vzlist = config_list();
2992
2993     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2994
2995     while (defined(my $de = $fd->read)) {
2996         next if $de !~ m/^(\d+)\.pid$/;
2997         my $vmid = $1;
2998         next if !defined($vzlist->{$vmid});
2999         if (my $pid = check_running($vmid)) {
3000             $vzlist->{$vmid}->{pid} = $pid;
3001         }
3002     }
3003
3004     return $vzlist;
3005 }
3006
3007 sub disksize {
3008     my ($storecfg, $conf) = @_;
3009
3010     my $bootdisk = $conf->{bootdisk};
3011     return undef if !$bootdisk;
3012     return undef if !is_valid_drivename($bootdisk);
3013
3014     return undef if !$conf->{$bootdisk};
3015
3016     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3017     return undef if !defined($drive);
3018
3019     return undef if drive_is_cdrom($drive);
3020
3021     my $volid = $drive->{file};
3022     return undef if !$volid;
3023
3024     return $drive->{size};
3025 }
3026
3027 our $vmstatus_return_properties = {
3028     vmid => get_standard_option('pve-vmid'),
3029     status => {
3030         description => "Qemu process status.",
3031         type => 'string',
3032         enum => ['stopped', 'running'],
3033     },
3034     maxmem => {
3035         description => "Maximum memory in bytes.",
3036         type => 'integer',
3037         optional => 1,
3038         renderer => 'bytes',
3039     },
3040     maxdisk => {
3041         description => "Root disk size in bytes.",
3042         type => 'integer',
3043         optional => 1,
3044         renderer => 'bytes',
3045     },
3046     name => {
3047         description => "VM name.",
3048         type => 'string',
3049         optional => 1,
3050     },
3051     qmpstatus => {
3052         description => "Qemu QMP agent status.",
3053         type => 'string',
3054         optional => 1,
3055     },
3056     pid => {
3057         description => "PID of running qemu process.",
3058         type => 'integer',
3059         optional => 1,
3060     },
3061     uptime => {
3062         description => "Uptime.",
3063         type => 'integer',
3064         optional => 1,
3065         renderer => 'duration',
3066     },
3067     cpus => {
3068         description => "Maximum usable CPUs.",
3069         type => 'number',
3070         optional => 1,
3071     },
3072 };
3073
3074 my $last_proc_pid_stat;
3075
3076 # get VM status information
3077 # This must be fast and should not block ($full == false)
3078 # We only query KVM using QMP if $full == true (this can be slow)
3079 sub vmstatus {
3080     my ($opt_vmid, $full) = @_;
3081
3082     my $res = {};
3083
3084     my $storecfg = PVE::Storage::config();
3085
3086     my $list = vzlist();
3087     my $defaults = load_defaults();
3088
3089     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3090
3091     my $cpucount = $cpuinfo->{cpus} || 1;
3092
3093     foreach my $vmid (keys %$list) {
3094         next if $opt_vmid && ($vmid ne $opt_vmid);
3095
3096         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3097         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3098
3099         my $d = { vmid => $vmid };
3100         $d->{pid} = $list->{$vmid}->{pid};
3101
3102         # fixme: better status?
3103         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3104
3105         my $size = disksize($storecfg, $conf);
3106         if (defined($size)) {
3107             $d->{disk} = 0; # no info available
3108             $d->{maxdisk} = $size;
3109         } else {
3110             $d->{disk} = 0;
3111             $d->{maxdisk} = 0;
3112         }
3113
3114         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3115             * ($conf->{cores} || $defaults->{cores});
3116         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3117         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3118
3119         $d->{name} = $conf->{name} || "VM $vmid";
3120         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3121             : $defaults->{memory}*(1024*1024);
3122
3123         if ($conf->{balloon}) {
3124             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3125             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3126                 : $defaults->{shares};
3127         }
3128
3129         $d->{uptime} = 0;
3130         $d->{cpu} = 0;
3131         $d->{mem} = 0;
3132
3133         $d->{netout} = 0;
3134         $d->{netin} = 0;
3135
3136         $d->{diskread} = 0;
3137         $d->{diskwrite} = 0;
3138
3139         $d->{template} = PVE::QemuConfig->is_template($conf);
3140
3141         $d->{serial} = 1 if conf_has_serial($conf);
3142
3143         $res->{$vmid} = $d;
3144     }
3145
3146     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3147     foreach my $dev (keys %$netdev) {
3148         next if $dev !~ m/^tap([1-9]\d*)i/;
3149         my $vmid = $1;
3150         my $d = $res->{$vmid};
3151         next if !$d;
3152
3153         $d->{netout} += $netdev->{$dev}->{receive};
3154         $d->{netin} += $netdev->{$dev}->{transmit};
3155
3156         if ($full) {
3157             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3158             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3159         }
3160
3161     }
3162
3163     my $ctime = gettimeofday;
3164
3165     foreach my $vmid (keys %$list) {
3166
3167         my $d = $res->{$vmid};
3168         my $pid = $d->{pid};
3169         next if !$pid;
3170
3171         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3172         next if !$pstat; # not running
3173
3174         my $used = $pstat->{utime} + $pstat->{stime};
3175
3176         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3177
3178         if ($pstat->{vsize}) {
3179             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3180         }
3181
3182         my $old = $last_proc_pid_stat->{$pid};
3183         if (!$old) {
3184             $last_proc_pid_stat->{$pid} = {
3185                 time => $ctime,
3186                 used => $used,
3187                 cpu => 0,
3188             };
3189             next;
3190         }
3191
3192         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3193
3194         if ($dtime > 1000) {
3195             my $dutime = $used -  $old->{used};
3196
3197             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3198             $last_proc_pid_stat->{$pid} = {
3199                 time => $ctime,
3200                 used => $used,
3201                 cpu => $d->{cpu},
3202             };
3203         } else {
3204             $d->{cpu} = $old->{cpu};
3205         }
3206     }
3207
3208     return $res if !$full;
3209
3210     my $qmpclient = PVE::QMPClient->new();
3211
3212     my $ballooncb = sub {
3213         my ($vmid, $resp) = @_;
3214
3215         my $info = $resp->{'return'};
3216         return if !$info->{max_mem};
3217
3218         my $d = $res->{$vmid};
3219
3220         # use memory assigned to VM
3221         $d->{maxmem} = $info->{max_mem};
3222         $d->{balloon} = $info->{actual};
3223
3224         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3225             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3226             $d->{freemem} = $info->{free_mem};
3227         }
3228
3229         $d->{ballooninfo} = $info;
3230     };
3231
3232     my $blockstatscb = sub {
3233         my ($vmid, $resp) = @_;
3234         my $data = $resp->{'return'} || [];
3235         my $totalrdbytes = 0;
3236         my $totalwrbytes = 0;
3237
3238         for my $blockstat (@$data) {
3239             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3240             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3241
3242             $blockstat->{device} =~ s/drive-//;
3243             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3244         }
3245         $res->{$vmid}->{diskread} = $totalrdbytes;
3246         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3247     };
3248
3249     my $statuscb = sub {
3250         my ($vmid, $resp) = @_;
3251
3252         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3253         # this fails if ballon driver is not loaded, so this must be
3254         # the last commnand (following command are aborted if this fails).
3255         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3256
3257         my $status = 'unknown';
3258         if (!defined($status = $resp->{'return'}->{status})) {
3259             warn "unable to get VM status\n";
3260             return;
3261         }
3262
3263         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3264     };
3265
3266     foreach my $vmid (keys %$list) {
3267         next if $opt_vmid && ($vmid ne $opt_vmid);
3268         next if !$res->{$vmid}->{pid}; # not running
3269         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3270     }
3271
3272     $qmpclient->queue_execute(undef, 2);
3273
3274     foreach my $vmid (keys %$list) {
3275         next if $opt_vmid && ($vmid ne $opt_vmid);
3276         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3277     }
3278
3279     return $res;
3280 }
3281
3282 sub foreach_drive {
3283     my ($conf, $func, @param) = @_;
3284
3285     foreach my $ds (valid_drive_names()) {
3286         next if !defined($conf->{$ds});
3287
3288         my $drive = parse_drive($ds, $conf->{$ds});
3289         next if !$drive;
3290
3291         &$func($ds, $drive, @param);
3292     }
3293 }
3294
3295 sub foreach_volid {
3296     my ($conf, $func, @param) = @_;
3297
3298     my $volhash = {};
3299
3300     my $test_volid = sub {
3301         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3302
3303         return if !$volid;
3304
3305         $volhash->{$volid}->{cdrom} //= 1;
3306         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3307
3308         $volhash->{$volid}->{replicate} //= 0;
3309         $volhash->{$volid}->{replicate} = 1 if $replicate;
3310
3311         $volhash->{$volid}->{shared} //= 0;
3312         $volhash->{$volid}->{shared} = 1 if $shared;
3313
3314         $volhash->{$volid}->{referenced_in_config} //= 0;
3315         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3316
3317         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3318             if defined($snapname);
3319     };
3320
3321     foreach_drive($conf, sub {
3322         my ($ds, $drive) = @_;
3323         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3324     });
3325
3326     foreach my $snapname (keys %{$conf->{snapshots}}) {
3327         my $snap = $conf->{snapshots}->{$snapname};
3328         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3329         foreach_drive($snap, sub {
3330             my ($ds, $drive) = @_;
3331             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3332         });
3333     }
3334
3335     foreach my $volid (keys %$volhash) {
3336         &$func($volid, $volhash->{$volid}, @param);
3337     }
3338 }
3339
3340 sub conf_has_serial {
3341     my ($conf) = @_;
3342
3343     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3344         if ($conf->{"serial$i"}) {
3345             return 1;
3346         }
3347     }
3348
3349     return 0;
3350 }
3351
3352 sub vga_conf_has_spice {
3353     my ($vga) = @_;
3354
3355     my $vgaconf = parse_vga($vga);
3356     my $vgatype = $vgaconf->{type};
3357     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3358
3359     return $1 || 1;
3360 }
3361
3362 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3363 sub get_host_arch() {
3364     $host_arch = (POSIX::uname())[4] if !$host_arch;
3365     return $host_arch;
3366 }
3367
3368 sub is_native($) {
3369     my ($arch) = @_;
3370     return get_host_arch() eq $arch;
3371 }
3372
3373 my $default_machines = {
3374     x86_64 => 'pc',
3375     aarch64 => 'virt',
3376 };
3377
3378 sub get_basic_machine_info {
3379     my ($conf, $forcemachine) = @_;
3380
3381     my $arch = $conf->{arch} // get_host_arch();
3382     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3383     return ($arch, $machine);
3384 }
3385
3386 sub get_ovmf_files($) {
3387     my ($arch) = @_;
3388
3389     my $ovmf = $OVMF->{$arch}
3390         or die "no OVMF images known for architecture '$arch'\n";
3391
3392     return @$ovmf;
3393 }
3394
3395 my $Arch2Qemu = {
3396     aarch64 => '/usr/bin/qemu-system-aarch64',
3397     x86_64 => '/usr/bin/qemu-system-x86_64',
3398 };
3399 sub get_command_for_arch($) {
3400     my ($arch) = @_;
3401     return '/usr/bin/kvm' if is_native($arch);
3402
3403     my $cmd = $Arch2Qemu->{$arch}
3404         or die "don't know how to emulate architecture '$arch'\n";
3405     return $cmd;
3406 }
3407
3408 sub get_cpu_options {
3409     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3410
3411     my $cpuFlags = [];
3412     my $ostype = $conf->{ostype};
3413
3414     my $cpu = $kvm ? "kvm64" : "qemu64";
3415     if ($arch eq 'aarch64') {
3416         $cpu = 'cortex-a57';
3417     }
3418     my $hv_vendor_id;
3419     if (my $cputype = $conf->{cpu}) {
3420         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3421             or die "Cannot parse cpu description: $cputype\n";
3422         $cpu = $cpuconf->{cputype};
3423         $kvm_off = 1 if $cpuconf->{hidden};
3424         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3425
3426         if (defined(my $flags = $cpuconf->{flags})) {
3427             push @$cpuFlags, split(";", $flags);
3428         }
3429     }
3430
3431     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3432
3433     push @$cpuFlags , '-x2apic'
3434         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3435
3436     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3437
3438     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3439
3440     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3441
3442         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3443         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3444     }
3445
3446     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3447
3448     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3449
3450     push @$cpuFlags, 'kvm=off' if $kvm_off;
3451
3452     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3453         push @$cpuFlags, "vendor=${cpu_vendor}"
3454             if $cpu_vendor ne 'default';
3455     } elsif ($arch ne 'aarch64') {
3456         die "internal error"; # should not happen
3457     }
3458
3459     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3460
3461     return ('-cpu', $cpu);
3462 }
3463
3464 sub config_to_command {
3465     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3466
3467     my $cmd = [];
3468     my $globalFlags = [];
3469     my $machineFlags = [];
3470     my $rtcFlags = [];
3471     my $devices = [];
3472     my $pciaddr = '';
3473     my $bridges = {};
3474     my $kvmver = kvm_user_version();
3475     my $vernum = 0; # unknown
3476     my $ostype = $conf->{ostype};
3477     my $winversion = windows_version($ostype);
3478     my $kvm = $conf->{kvm};
3479
3480     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3481     $kvm //= 1 if is_native($arch);
3482
3483     if ($kvm) {
3484         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3485             if !defined kvm_version();
3486     }
3487
3488     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3489         $vernum = $1*1000000+$2*1000;
3490     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3491         $vernum = $1*1000000+$2*1000+$3;
3492     }
3493
3494     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3495
3496     my $have_ovz = -f '/proc/vz/vestat';
3497
3498     my $q35 = machine_type_is_q35($conf);
3499     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3500     my $use_old_bios_files = undef;
3501     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3502
3503     my $cpuunits = defined($conf->{cpuunits}) ?
3504             $conf->{cpuunits} : $defaults->{cpuunits};
3505
3506     push @$cmd, get_command_for_arch($arch);
3507
3508     push @$cmd, '-id', $vmid;
3509
3510     my $vmname = $conf->{name} || "vm$vmid";
3511
3512     push @$cmd, '-name', $vmname;
3513
3514     my $use_virtio = 0;
3515
3516     my $qmpsocket = qmp_socket($vmid);
3517     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3518     push @$cmd, '-mon', "chardev=qmp,mode=control";
3519
3520     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3521         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3522         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3523     }
3524
3525     push @$cmd, '-pidfile' , pidfile_name($vmid);
3526
3527     push @$cmd, '-daemonize';
3528
3529     if ($conf->{smbios1}) {
3530         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3531     }
3532
3533     if ($conf->{vmgenid}) {
3534         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3535     }
3536
3537     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3538     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3539         die "uefi base image not found\n" if ! -f $ovmf_code;
3540
3541         my $path;
3542         my $format;
3543         if (my $efidisk = $conf->{efidisk0}) {
3544             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3545             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3546             $format = $d->{format};
3547             if ($storeid) {
3548                 $path = PVE::Storage::path($storecfg, $d->{file});
3549                 if (!defined($format)) {
3550                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3551                     $format = qemu_img_format($scfg, $volname);
3552                 }
3553             } else {
3554                 $path = $d->{file};
3555                 die "efidisk format must be specified\n"
3556                     if !defined($format);
3557             }
3558         } else {
3559             warn "no efidisk configured! Using temporary efivars disk.\n";
3560             $path = "/tmp/$vmid-ovmf.fd";
3561             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3562             $format = 'raw';
3563         }
3564
3565         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3566         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3567     }
3568
3569
3570     # add usb controllers
3571     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3572     push @$devices, @usbcontrollers if @usbcontrollers;
3573     my $vga = parse_vga($conf->{vga});
3574
3575     my $qxlnum = vga_conf_has_spice($conf->{vga});
3576     $vga->{type} = 'qxl' if $qxlnum;
3577
3578     if (!$vga->{type}) {
3579         if ($arch eq 'aarch64') {
3580             $vga->{type} = 'virtio';
3581         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3582             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3583         } else {
3584             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3585         }
3586     }
3587
3588     # enable absolute mouse coordinates (needed by vnc)
3589     my $tablet;
3590     if (defined($conf->{tablet})) {
3591         $tablet = $conf->{tablet};
3592     } else {
3593         $tablet = $defaults->{tablet};
3594         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3595         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3596     }
3597
3598     if ($tablet) {
3599         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3600         my $kbd = print_keyboarddevice_full($conf, $arch);
3601         push @$devices, '-device', $kbd if defined($kbd);
3602     }
3603
3604     my $kvm_off = 0;
3605     my $gpu_passthrough;
3606
3607     # host pci devices
3608     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3609         my $d = parse_hostpci($conf->{"hostpci$i"});
3610         next if !$d;
3611
3612         my $pcie = $d->{pcie};
3613         if ($pcie) {
3614             die "q35 machine model is not enabled" if !$q35;
3615             # win7 wants to have the pcie devices directly on the pcie bus
3616             # instead of in the root port
3617             if ($winversion == 7) {
3618                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3619             } else {
3620                 $pciaddr = print_pcie_addr("hostpci$i");
3621             }
3622         } else {
3623             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3624         }
3625
3626         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3627         my $romfile = $d->{romfile};
3628
3629         my $xvga = '';
3630         if ($d->{'x-vga'}) {
3631             $xvga = ',x-vga=on';
3632             $kvm_off = 1;
3633             $vga->{type} = 'none' if !defined($conf->{vga});
3634             $gpu_passthrough = 1;
3635
3636             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3637                 $xvga = "";
3638             }
3639         }
3640         my $pcidevices = $d->{pciid};
3641         my $multifunction = 1 if @$pcidevices > 1;
3642         my $sysfspath;
3643         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3644             my $id = $pcidevices->[0]->{id};
3645             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3646             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3647         } elsif ($d->{mdev}) {
3648             warn "ignoring mediated device with multifunction device\n";
3649         }
3650
3651         my $j=0;
3652         foreach my $pcidevice (@$pcidevices) {
3653
3654             my $id = "hostpci$i";
3655             $id .= ".$j" if $multifunction;
3656             my $addr = $pciaddr;
3657             $addr .= ".$j" if $multifunction;
3658             my $devicestr = "vfio-pci";
3659             if ($sysfspath) {
3660                 $devicestr .= ",sysfsdev=$sysfspath";
3661             } else {
3662                 $devicestr .= ",host=$pcidevice->{id}";
3663             }
3664             $devicestr .= ",id=$id$addr";
3665
3666             if($j == 0){
3667                 $devicestr .= "$rombar$xvga";
3668                 $devicestr .= ",multifunction=on" if $multifunction;
3669                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3670             }
3671
3672             push @$devices, '-device', $devicestr;
3673             $j++;
3674         }
3675     }
3676
3677     # usb devices
3678     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3679     push @$devices, @usbdevices if @usbdevices;
3680     # serial devices
3681     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3682         if (my $path = $conf->{"serial$i"}) {
3683             if ($path eq 'socket') {
3684                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3685                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3686                 # On aarch64, serial0 is the UART device. Qemu only allows
3687                 # connecting UART devices via the '-serial' command line, as
3688                 # the device has a fixed slot on the hardware...
3689                 if ($arch eq 'aarch64' && $i == 0) {
3690                     push @$devices, '-serial', "chardev:serial$i";
3691                 } else {
3692                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3693                 }
3694             } else {
3695                 die "no such serial device\n" if ! -c $path;
3696                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3697                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3698             }
3699         }
3700     }
3701
3702     # parallel devices
3703     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3704         if (my $path = $conf->{"parallel$i"}) {
3705             die "no such parallel device\n" if ! -c $path;
3706             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3707             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3708             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3709         }
3710     }
3711
3712
3713     my $sockets = 1;
3714     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3715     $sockets = $conf->{sockets} if  $conf->{sockets};
3716
3717     my $cores = $conf->{cores} || 1;
3718
3719     my $maxcpus = $sockets * $cores;
3720
3721     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3722
3723     my $allowed_vcpus = $cpuinfo->{cpus};
3724
3725     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3726         if ($allowed_vcpus < $maxcpus);
3727
3728     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3729
3730         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3731         for (my $i = 2; $i <= $vcpus; $i++)  {
3732             my $cpustr = print_cpu_device($conf,$i);
3733             push @$cmd, '-device', $cpustr;
3734         }
3735
3736     } else {
3737
3738         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3739     }
3740     push @$cmd, '-nodefaults';
3741
3742     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3743
3744     my $bootindex_hash = {};
3745     my $i = 1;
3746     foreach my $o (split(//, $bootorder)) {
3747         $bootindex_hash->{$o} = $i*100;
3748         $i++;
3749     }
3750
3751     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3752
3753     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3754
3755     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3756
3757     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3758         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3759         my $socket = vnc_socket($vmid);
3760         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3761     } else {
3762         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3763         push @$cmd, '-nographic';
3764     }
3765
3766     # time drift fix
3767     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3768
3769     my $useLocaltime = $conf->{localtime};
3770
3771     if ($winversion >= 5) { # windows
3772         $useLocaltime = 1 if !defined($conf->{localtime});
3773
3774         # use time drift fix when acpi is enabled
3775         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3776             $tdf = 1 if !defined($conf->{tdf});
3777         }
3778     }
3779
3780     if ($winversion >= 6) {
3781         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3782         push @$cmd, '-no-hpet';
3783     }
3784
3785     push @$rtcFlags, 'driftfix=slew' if $tdf;
3786
3787     if (!$kvm) {
3788         push @$machineFlags, 'accel=tcg';
3789     }
3790
3791     if ($machine_type) {
3792         push @$machineFlags, "type=${machine_type}";
3793     }
3794
3795     if ($conf->{startdate}) {
3796         push @$rtcFlags, "base=$conf->{startdate}";
3797     } elsif ($useLocaltime) {
3798         push @$rtcFlags, 'base=localtime';
3799     }
3800
3801     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3802
3803     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3804
3805     push @$cmd, '-S' if $conf->{freeze};
3806
3807     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3808
3809     # enable sound
3810     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3811     #push @$cmd, '-soundhw', 'es1370';
3812     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3813
3814     if (parse_guest_agent($conf)->{enabled}) {
3815         my $qgasocket = qmp_socket($vmid, 1);
3816         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3817         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3818         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3819         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3820     }
3821
3822     my $spice_port;
3823
3824     if ($qxlnum) {
3825         if ($qxlnum > 1) {
3826             if ($winversion){
3827                 for(my $i = 1; $i < $qxlnum; $i++){
3828                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3829                 }
3830             } else {
3831                 # assume other OS works like Linux
3832                 my ($ram, $vram) = ("134217728", "67108864");
3833                 if ($vga->{memory}) {
3834                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3835                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3836                 }
3837                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3838                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3839             }
3840         }
3841
3842         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3843
3844         my $nodename = PVE::INotify::nodename();
3845         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3846         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3847         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3848         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3849         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3850
3851         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3852
3853         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3854         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3855         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3856     }
3857
3858     # enable balloon by default, unless explicitly disabled
3859     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3860         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3861         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3862     }
3863
3864     if ($conf->{watchdog}) {
3865         my $wdopts = parse_watchdog($conf->{watchdog});
3866         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3867         my $watchdog = $wdopts->{model} || 'i6300esb';
3868         push @$devices, '-device', "$watchdog$pciaddr";
3869         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3870     }
3871
3872     my $vollist = [];
3873     my $scsicontroller = {};
3874     my $ahcicontroller = {};
3875     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3876
3877     # Add iscsi initiator name if available
3878     if (my $initiator = get_initiator_name()) {
3879         push @$devices, '-iscsi', "initiator-name=$initiator";
3880     }
3881
3882     foreach_drive($conf, sub {
3883         my ($ds, $drive) = @_;
3884
3885         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3886             push @$vollist, $drive->{file};
3887         }
3888
3889         # ignore efidisk here, already added in bios/fw handling code above
3890         return if $drive->{interface} eq 'efidisk';
3891
3892         $use_virtio = 1 if $ds =~ m/^virtio/;
3893
3894         if (drive_is_cdrom ($drive)) {
3895             if ($bootindex_hash->{d}) {
3896                 $drive->{bootindex} = $bootindex_hash->{d};
3897                 $bootindex_hash->{d} += 1;
3898             }
3899         } else {
3900             if ($bootindex_hash->{c}) {
3901                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3902                 $bootindex_hash->{c} += 1;
3903             }
3904         }
3905
3906         if($drive->{interface} eq 'virtio'){
3907            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3908         }
3909
3910         if ($drive->{interface} eq 'scsi') {
3911
3912             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3913
3914             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3915             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3916
3917             my $iothread = '';
3918             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3919                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3920                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3921             } elsif ($drive->{iothread}) {
3922                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3923             }
3924
3925             my $queues = '';
3926             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3927                 $queues = ",num_queues=$drive->{queues}";
3928             }
3929
3930             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3931             $scsicontroller->{$controller}=1;
3932         }
3933
3934         if ($drive->{interface} eq 'sata') {
3935            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3936            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3937            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3938            $ahcicontroller->{$controller}=1;
3939         }
3940
3941         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3942         push @$devices, '-drive',$drive_cmd;
3943         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3944     });
3945
3946     for (my $i = 0; $i < $MAX_NETS; $i++) {
3947          next if !$conf->{"net$i"};
3948          my $d = parse_net($conf->{"net$i"});
3949          next if !$d;
3950
3951          $use_virtio = 1 if $d->{model} eq 'virtio';
3952
3953          if ($bootindex_hash->{n}) {
3954             $d->{bootindex} = $bootindex_hash->{n};
3955             $bootindex_hash->{n} += 1;
3956          }
3957
3958          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3959          push @$devices, '-netdev', $netdevfull;
3960
3961          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3962          push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3980     }
3981
3982     if (!$q35) {
3983         # add pci bridges
3984         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3985            $bridges->{1} = 1;
3986            $bridges->{2} = 1;
3987         }
3988
3989         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3990
3991         while (my ($k, $v) = each %$bridges) {
3992             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3993             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3994         }
3995     }
3996
3997     push @$cmd, @$devices;
3998     push @$cmd, '-rtc', join(',', @$rtcFlags)
3999         if scalar(@$rtcFlags);
4000     push @$cmd, '-machine', join(',', @$machineFlags)
4001         if scalar(@$machineFlags);
4002     push @$cmd, '-global', join(',', @$globalFlags)
4003         if scalar(@$globalFlags);
4004
4005     # add custom args
4006     if ($conf->{args}) {
4007         my $aa = PVE::Tools::split_args($conf->{args});
4008         push @$cmd, @$aa;
4009     }
4010
4011     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4012 }
4013
4014 sub vnc_socket {
4015     my ($vmid) = @_;
4016     return "${var_run_tmpdir}/$vmid.vnc";
4017 }
4018
4019 sub spice_port {
4020     my ($vmid) = @_;
4021
4022     my $res = vm_mon_cmd($vmid, 'query-spice');
4023
4024     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4025 }
4026
4027 sub qmp_socket {
4028     my ($vmid, $qga, $name) = @_;
4029     my $sockettype = $qga ? 'qga' : 'qmp';
4030     my $ext = $name ? '-'.$name : '';
4031     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4032 }
4033
4034 sub pidfile_name {
4035     my ($vmid) = @_;
4036     return "${var_run_tmpdir}/$vmid.pid";
4037 }
4038
4039 sub vm_devices_list {
4040     my ($vmid) = @_;
4041
4042     my $res = vm_mon_cmd($vmid, 'query-pci');
4043     my $devices_to_check = [];
4044     my $devices = {};
4045     foreach my $pcibus (@$res) {
4046         push @$devices_to_check, @{$pcibus->{devices}},
4047     }
4048
4049     while (@$devices_to_check) {
4050         my $to_check = [];
4051         for my $d (@$devices_to_check) {
4052             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4053             next if !$d->{'pci_bridge'};
4054
4055             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4056             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4057         }
4058         $devices_to_check = $to_check;
4059     }
4060
4061     my $resblock = vm_mon_cmd($vmid, 'query-block');
4062     foreach my $block (@$resblock) {
4063         if($block->{device} =~ m/^drive-(\S+)/){
4064                 $devices->{$1} = 1;
4065         }
4066     }
4067
4068     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4069     foreach my $mice (@$resmice) {
4070         if ($mice->{name} eq 'QEMU HID Tablet') {
4071             $devices->{tablet} = 1;
4072             last;
4073         }
4074     }
4075
4076     # for usb devices there is no query-usb
4077     # but we can iterate over the entries in
4078     # qom-list path=/machine/peripheral
4079     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4080     foreach my $per (@$resperipheral) {
4081         if ($per->{name} =~ m/^usb\d+$/) {
4082             $devices->{$per->{name}} = 1;
4083         }
4084     }
4085
4086     return $devices;
4087 }
4088
4089 sub vm_deviceplug {
4090     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4091
4092     my $q35 = machine_type_is_q35($conf);
4093
4094     my $devices_list = vm_devices_list($vmid);
4095     return 1 if defined($devices_list->{$deviceid});
4096
4097     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4098
4099     if ($deviceid eq 'tablet') {
4100
4101         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4102
4103     } elsif ($deviceid eq 'keyboard') {
4104
4105         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4106
4107     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4108
4109         die "usb hotplug currently not reliable\n";
4110         # since we can't reliably hot unplug all added usb devices
4111         # and usb passthrough disables live migration
4112         # we disable usb hotplugging for now
4113         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4114
4115     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4116
4117         qemu_iothread_add($vmid, $deviceid, $device);
4118
4119         qemu_driveadd($storecfg, $vmid, $device);
4120         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4121
4122         qemu_deviceadd($vmid, $devicefull);
4123         eval { qemu_deviceaddverify($vmid, $deviceid); };
4124         if (my $err = $@) {
4125             eval { qemu_drivedel($vmid, $deviceid); };
4126             warn $@ if $@;
4127             die $err;
4128         }
4129
4130     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4131
4132
4133         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4134         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4135         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4136
4137         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4138
4139         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4140             qemu_iothread_add($vmid, $deviceid, $device);
4141             $devicefull .= ",iothread=iothread-$deviceid";
4142         }
4143
4144         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4145             $devicefull .= ",num_queues=$device->{queues}";
4146         }
4147
4148         qemu_deviceadd($vmid, $devicefull);
4149         qemu_deviceaddverify($vmid, $deviceid);
4150
4151     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4152
4153         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4154         qemu_driveadd($storecfg, $vmid, $device);
4155
4156         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4157         eval { qemu_deviceadd($vmid, $devicefull); };
4158         if (my $err = $@) {
4159             eval { qemu_drivedel($vmid, $deviceid); };
4160             warn $@ if $@;
4161             die $err;
4162         }
4163
4164     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4165
4166         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4167
4168         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4169         my $use_old_bios_files = undef;
4170         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4171
4172         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4173         qemu_deviceadd($vmid, $netdevicefull);
4174         eval {
4175             qemu_deviceaddverify($vmid, $deviceid);
4176             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4177         };
4178         if (my $err = $@) {
4179             eval { qemu_netdevdel($vmid, $deviceid); };
4180             warn $@ if $@;
4181             die $err;
4182         }
4183
4184     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4185
4186         my $bridgeid = $2;
4187         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4188         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4189
4190         qemu_deviceadd($vmid, $devicefull);
4191         qemu_deviceaddverify($vmid, $deviceid);
4192
4193     } else {
4194         die "can't hotplug device '$deviceid'\n";
4195     }
4196
4197     return 1;
4198 }
4199
4200 # fixme: this should raise exceptions on error!
4201 sub vm_deviceunplug {
4202     my ($vmid, $conf, $deviceid) = @_;
4203
4204     my $devices_list = vm_devices_list($vmid);
4205     return 1 if !defined($devices_list->{$deviceid});
4206
4207     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4208
4209     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4210
4211         qemu_devicedel($vmid, $deviceid);
4212
4213     } elsif ($deviceid =~ m/^usb\d+$/) {
4214
4215         die "usb hotplug currently not reliable\n";
4216         # when unplugging usb devices this way,
4217         # there may be remaining usb controllers/hubs
4218         # so we disable it for now
4219         qemu_devicedel($vmid, $deviceid);
4220         qemu_devicedelverify($vmid, $deviceid);
4221
4222     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4223
4224         qemu_devicedel($vmid, $deviceid);
4225         qemu_devicedelverify($vmid, $deviceid);
4226         qemu_drivedel($vmid, $deviceid);
4227         qemu_iothread_del($conf, $vmid, $deviceid);
4228
4229     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4230
4231         qemu_devicedel($vmid, $deviceid);
4232         qemu_devicedelverify($vmid, $deviceid);
4233         qemu_iothread_del($conf, $vmid, $deviceid);
4234
4235     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4236
4237         qemu_devicedel($vmid, $deviceid);
4238         qemu_drivedel($vmid, $deviceid);
4239         qemu_deletescsihw($conf, $vmid, $deviceid);
4240
4241     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4242
4243         qemu_devicedel($vmid, $deviceid);
4244         qemu_devicedelverify($vmid, $deviceid);
4245         qemu_netdevdel($vmid, $deviceid);
4246
4247     } else {
4248         die "can't unplug device '$deviceid'\n";
4249     }
4250
4251     return 1;
4252 }
4253
4254 sub qemu_deviceadd {
4255     my ($vmid, $devicefull) = @_;
4256
4257     $devicefull = "driver=".$devicefull;
4258     my %options =  split(/[=,]/, $devicefull);
4259
4260     vm_mon_cmd($vmid, "device_add" , %options);
4261 }
4262
4263 sub qemu_devicedel {
4264     my ($vmid, $deviceid) = @_;
4265
4266     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4267 }
4268
4269 sub qemu_iothread_add {
4270     my($vmid, $deviceid, $device) = @_;
4271
4272     if ($device->{iothread}) {
4273         my $iothreads = vm_iothreads_list($vmid);
4274         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4275     }
4276 }
4277
4278 sub qemu_iothread_del {
4279     my($conf, $vmid, $deviceid) = @_;
4280
4281     my $confid = $deviceid;
4282     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4283         $confid = 'scsi' . $1;
4284     }
4285     my $device = parse_drive($confid, $conf->{$confid});
4286     if ($device->{iothread}) {
4287         my $iothreads = vm_iothreads_list($vmid);
4288         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4289     }
4290 }
4291
4292 sub qemu_objectadd {
4293     my($vmid, $objectid, $qomtype) = @_;
4294
4295     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4296
4297     return 1;
4298 }
4299
4300 sub qemu_objectdel {
4301     my($vmid, $objectid) = @_;
4302
4303     vm_mon_cmd($vmid, "object-del", id => $objectid);
4304
4305     return 1;
4306 }
4307
4308 sub qemu_driveadd {
4309     my ($storecfg, $vmid, $device) = @_;
4310
4311     my $drive = print_drive_full($storecfg, $vmid, $device);
4312     $drive =~ s/\\/\\\\/g;
4313     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4314
4315     # If the command succeeds qemu prints: "OK"
4316     return 1 if $ret =~ m/OK/s;
4317
4318     die "adding drive failed: $ret\n";
4319 }
4320
4321 sub qemu_drivedel {
4322     my($vmid, $deviceid) = @_;
4323
4324     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4325     $ret =~ s/^\s+//;
4326
4327     return 1 if $ret eq "";
4328
4329     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4330     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4331
4332     die "deleting drive $deviceid failed : $ret\n";
4333 }
4334
4335 sub qemu_deviceaddverify {
4336     my ($vmid, $deviceid) = @_;
4337
4338     for (my $i = 0; $i <= 5; $i++) {
4339          my $devices_list = vm_devices_list($vmid);
4340          return 1 if defined($devices_list->{$deviceid});
4341          sleep 1;
4342     }
4343
4344     die "error on hotplug device '$deviceid'\n";
4345 }
4346
4347
4348 sub qemu_devicedelverify {
4349     my ($vmid, $deviceid) = @_;
4350
4351     # need to verify that the device is correctly removed as device_del
4352     # is async and empty return is not reliable
4353
4354     for (my $i = 0; $i <= 5; $i++) {
4355          my $devices_list = vm_devices_list($vmid);
4356          return 1 if !defined($devices_list->{$deviceid});
4357          sleep 1;
4358     }
4359
4360     die "error on hot-unplugging device '$deviceid'\n";
4361 }
4362
4363 sub qemu_findorcreatescsihw {
4364     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4365
4366     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4367
4368     my $scsihwid="$controller_prefix$controller";
4369     my $devices_list = vm_devices_list($vmid);
4370
4371     if(!defined($devices_list->{$scsihwid})) {
4372         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4373     }
4374
4375     return 1;
4376 }
4377
4378 sub qemu_deletescsihw {
4379     my ($conf, $vmid, $opt) = @_;
4380
4381     my $device = parse_drive($opt, $conf->{$opt});
4382
4383     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4384         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4385         return 1;
4386     }
4387
4388     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4389
4390     my $devices_list = vm_devices_list($vmid);
4391     foreach my $opt (keys %{$devices_list}) {
4392         if (PVE::QemuServer::is_valid_drivename($opt)) {
4393             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4394             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4395                 return 1;
4396             }
4397         }
4398     }
4399
4400     my $scsihwid="scsihw$controller";
4401
4402     vm_deviceunplug($vmid, $conf, $scsihwid);
4403
4404     return 1;
4405 }
4406
4407 sub qemu_add_pci_bridge {
4408     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4409
4410     my $bridges = {};
4411
4412     my $bridgeid;
4413
4414     print_pci_addr($device, $bridges, $arch, $machine_type);
4415
4416     while (my ($k, $v) = each %$bridges) {
4417         $bridgeid = $k;
4418     }
4419     return 1 if !defined($bridgeid) || $bridgeid < 1;
4420
4421     my $bridge = "pci.$bridgeid";
4422     my $devices_list = vm_devices_list($vmid);
4423
4424     if (!defined($devices_list->{$bridge})) {
4425         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4426     }
4427
4428     return 1;
4429 }
4430
4431 sub qemu_set_link_status {
4432     my ($vmid, $device, $up) = @_;
4433
4434     vm_mon_cmd($vmid, "set_link", name => $device,
4435                up => $up ? JSON::true : JSON::false);
4436 }
4437
4438 sub qemu_netdevadd {
4439     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4440
4441     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4442     my %options =  split(/[=,]/, $netdev);
4443
4444     vm_mon_cmd($vmid, "netdev_add",  %options);
4445     return 1;
4446 }
4447
4448 sub qemu_netdevdel {
4449     my ($vmid, $deviceid) = @_;
4450
4451     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4452 }
4453
4454 sub qemu_usb_hotplug {
4455     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4456
4457     return if !$device;
4458
4459     # remove the old one first
4460     vm_deviceunplug($vmid, $conf, $deviceid);
4461
4462     # check if xhci controller is necessary and available
4463     if ($device->{usb3}) {
4464
4465         my $devicelist = vm_devices_list($vmid);
4466
4467         if (!$devicelist->{xhci}) {
4468             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4469             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4470         }
4471     }
4472     my $d = parse_usb_device($device->{host});
4473     $d->{usb3} = $device->{usb3};
4474
4475     # add the new one
4476     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4477 }
4478
4479 sub qemu_cpu_hotplug {
4480     my ($vmid, $conf, $vcpus) = @_;
4481
4482     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4483
4484     my $sockets = 1;
4485     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4486     $sockets = $conf->{sockets} if  $conf->{sockets};
4487     my $cores = $conf->{cores} || 1;
4488     my $maxcpus = $sockets * $cores;
4489
4490     $vcpus = $maxcpus if !$vcpus;
4491
4492     die "you can't add more vcpus than maxcpus\n"
4493         if $vcpus > $maxcpus;
4494
4495     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4496
4497     if ($vcpus < $currentvcpus) {
4498
4499         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4500
4501             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4502                 qemu_devicedel($vmid, "cpu$i");
4503                 my $retry = 0;
4504                 my $currentrunningvcpus = undef;
4505                 while (1) {
4506                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4507                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4508                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4509                     $retry++;
4510                     sleep 1;
4511                 }
4512                 #update conf after each succesfull cpu unplug
4513                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4514                 PVE::QemuConfig->write_config($vmid, $conf);
4515             }
4516         } else {
4517             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4518         }
4519
4520         return;
4521     }
4522
4523     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4524     die "vcpus in running vm does not match its configuration\n"
4525         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4526
4527     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4528
4529         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4530             my $cpustr = print_cpu_device($conf, $i);
4531             qemu_deviceadd($vmid, $cpustr);
4532
4533             my $retry = 0;
4534             my $currentrunningvcpus = undef;
4535             while (1) {
4536                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4537                 last if scalar(@{$currentrunningvcpus}) == $i;
4538                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4539                 sleep 1;
4540                 $retry++;
4541             }
4542             #update conf after each succesfull cpu hotplug
4543             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4544             PVE::QemuConfig->write_config($vmid, $conf);
4545         }
4546     } else {
4547
4548         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4549             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4550         }
4551     }
4552 }
4553
4554 sub qemu_block_set_io_throttle {
4555     my ($vmid, $deviceid,
4556         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4557         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4558         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4559         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4560
4561     return if !check_running($vmid) ;
4562
4563     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4564         bps => int($bps),
4565         bps_rd => int($bps_rd),
4566         bps_wr => int($bps_wr),
4567         iops => int($iops),
4568         iops_rd => int($iops_rd),
4569         iops_wr => int($iops_wr),
4570         bps_max => int($bps_max),
4571         bps_rd_max => int($bps_rd_max),
4572         bps_wr_max => int($bps_wr_max),
4573         iops_max => int($iops_max),
4574         iops_rd_max => int($iops_rd_max),
4575         iops_wr_max => int($iops_wr_max),
4576         bps_max_length => int($bps_max_length),
4577         bps_rd_max_length => int($bps_rd_max_length),
4578         bps_wr_max_length => int($bps_wr_max_length),
4579         iops_max_length => int($iops_max_length),
4580         iops_rd_max_length => int($iops_rd_max_length),
4581         iops_wr_max_length => int($iops_wr_max_length),
4582     );
4583
4584 }
4585
4586 # old code, only used to shutdown old VM after update
4587 sub __read_avail {
4588     my ($fh, $timeout) = @_;
4589
4590     my $sel = new IO::Select;
4591     $sel->add($fh);
4592
4593     my $res = '';
4594     my $buf;
4595
4596     my @ready;
4597     while (scalar (@ready = $sel->can_read($timeout))) {
4598         my $count;
4599         if ($count = $fh->sysread($buf, 8192)) {
4600             if ($buf =~ /^(.*)\(qemu\) $/s) {
4601                 $res .= $1;
4602                 last;
4603             } else {
4604                 $res .= $buf;
4605             }
4606         } else {
4607             if (!defined($count)) {
4608                 die "$!\n";
4609             }
4610             last;
4611         }
4612     }
4613
4614     die "monitor read timeout\n" if !scalar(@ready);
4615
4616     return $res;
4617 }
4618
4619 sub qemu_block_resize {
4620     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4621
4622     my $running = check_running($vmid);
4623
4624     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4625
4626     return if !$running;
4627
4628     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4629
4630 }
4631
4632 sub qemu_volume_snapshot {
4633     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4634
4635     my $running = check_running($vmid);
4636
4637     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4638         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4639     } else {
4640         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4641     }
4642 }
4643
4644 sub qemu_volume_snapshot_delete {
4645     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4646
4647     my $running = check_running($vmid);
4648
4649     if($running) {
4650
4651         $running = undef;
4652         my $conf = PVE::QemuConfig->load_config($vmid);
4653         foreach_drive($conf, sub {
4654             my ($ds, $drive) = @_;
4655             $running = 1 if $drive->{file} eq $volid;
4656         });
4657     }
4658
4659     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4660         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4661     } else {
4662         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4663     }
4664 }
4665
4666 sub set_migration_caps {
4667     my ($vmid) = @_;
4668
4669     my $cap_ref = [];
4670
4671     my $enabled_cap = {
4672         "auto-converge" => 1,
4673         "xbzrle" => 1,
4674         "x-rdma-pin-all" => 0,
4675         "zero-blocks" => 0,
4676         "compress" => 0
4677     };
4678
4679     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4680
4681     for my $supported_capability (@$supported_capabilities) {
4682         push @$cap_ref, {
4683             capability => $supported_capability->{capability},
4684             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4685         };
4686     }
4687
4688     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4689 }
4690
4691 my $fast_plug_option = {
4692     'lock' => 1,
4693     'name' => 1,
4694     'onboot' => 1,
4695     'shares' => 1,
4696     'startup' => 1,
4697     'description' => 1,
4698     'protection' => 1,
4699     'vmstatestorage' => 1,
4700     'hookscript' => 1,
4701 };
4702
4703 # hotplug changes in [PENDING]
4704 # $selection hash can be used to only apply specified options, for
4705 # example: { cores => 1 } (only apply changed 'cores')
4706 # $errors ref is used to return error messages
4707 sub vmconfig_hotplug_pending {
4708     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4709
4710     my $defaults = load_defaults();
4711     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4712
4713     # commit values which do not have any impact on running VM first
4714     # Note: those option cannot raise errors, we we do not care about
4715     # $selection and always apply them.
4716
4717     my $add_error = sub {
4718         my ($opt, $msg) = @_;
4719         $errors->{$opt} = "hotplug problem - $msg";
4720     };
4721
4722     my $changes = 0;
4723     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4724         if ($fast_plug_option->{$opt}) {
4725             $conf->{$opt} = $conf->{pending}->{$opt};
4726             delete $conf->{pending}->{$opt};
4727             $changes = 1;
4728         }
4729     }
4730
4731     if ($changes) {
4732         PVE::QemuConfig->write_config($vmid, $conf);
4733         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4734     }
4735
4736     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4737
4738     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4739     while (my ($opt, $force) = each %$pending_delete_hash) {
4740         next if $selection && !$selection->{$opt};
4741         eval {
4742             if ($opt eq 'hotplug') {
4743                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4744             } elsif ($opt eq 'tablet') {
4745                 die "skip\n" if !$hotplug_features->{usb};
4746                 if ($defaults->{tablet}) {
4747                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4748                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4749                         if $arch eq 'aarch64';
4750                 } else {
4751                     vm_deviceunplug($vmid, $conf, 'tablet');
4752                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4753                 }
4754             } elsif ($opt =~ m/^usb\d+/) {
4755                 die "skip\n";
4756                 # since we cannot reliably hot unplug usb devices
4757                 # we are disabling it
4758                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4759                 vm_deviceunplug($vmid, $conf, $opt);
4760             } elsif ($opt eq 'vcpus') {
4761                 die "skip\n" if !$hotplug_features->{cpu};
4762                 qemu_cpu_hotplug($vmid, $conf, undef);
4763             } elsif ($opt eq 'balloon') {
4764                 # enable balloon device is not hotpluggable
4765                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4766                 # here we reset the ballooning value to memory
4767                 my $balloon = $conf->{memory} || $defaults->{memory};
4768                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4769             } elsif ($fast_plug_option->{$opt}) {
4770                 # do nothing
4771             } elsif ($opt =~ m/^net(\d+)$/) {
4772                 die "skip\n" if !$hotplug_features->{network};
4773                 vm_deviceunplug($vmid, $conf, $opt);
4774             } elsif (is_valid_drivename($opt)) {
4775                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4776                 vm_deviceunplug($vmid, $conf, $opt);
4777                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4778             } elsif ($opt =~ m/^memory$/) {
4779                 die "skip\n" if !$hotplug_features->{memory};
4780                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4781             } elsif ($opt eq 'cpuunits') {
4782                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4783             } elsif ($opt eq 'cpulimit') {
4784                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4785             } else {
4786                 die "skip\n";
4787             }
4788         };
4789         if (my $err = $@) {
4790             &$add_error($opt, $err) if $err ne "skip\n";
4791         } else {
4792             # save new config if hotplug was successful
4793             delete $conf->{$opt};
4794             vmconfig_undelete_pending_option($conf, $opt);
4795             PVE::QemuConfig->write_config($vmid, $conf);
4796             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4797         }
4798     }
4799
4800     my $apply_pending_cloudinit;
4801     $apply_pending_cloudinit = sub {
4802         my ($key, $value) = @_;
4803         $apply_pending_cloudinit = sub {}; # once is enough
4804
4805         my @cloudinit_opts = keys %$confdesc_cloudinit;
4806         foreach my $opt (keys %{$conf->{pending}}) {
4807             next if !grep { $_ eq $opt } @cloudinit_opts;
4808             $conf->{$opt} = delete $conf->{pending}->{$opt};
4809         }
4810
4811         my $new_conf = { %$conf };
4812         $new_conf->{$key} = $value;
4813         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4814     };
4815
4816     foreach my $opt (keys %{$conf->{pending}}) {
4817         next if $selection && !$selection->{$opt};
4818         my $value = $conf->{pending}->{$opt};
4819         eval {
4820             if ($opt eq 'hotplug') {
4821                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4822             } elsif ($opt eq 'tablet') {
4823                 die "skip\n" if !$hotplug_features->{usb};
4824                 if ($value == 1) {
4825                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4826                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4827                         if $arch eq 'aarch64';
4828                 } elsif ($value == 0) {
4829                     vm_deviceunplug($vmid, $conf, 'tablet');
4830                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4831                 }
4832             } elsif ($opt =~ m/^usb\d+$/) {
4833                 die "skip\n";
4834                 # since we cannot reliably hot unplug usb devices
4835                 # we are disabling it
4836                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4837                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4838                 die "skip\n" if !$d;
4839                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4840             } elsif ($opt eq 'vcpus') {
4841                 die "skip\n" if !$hotplug_features->{cpu};
4842                 qemu_cpu_hotplug($vmid, $conf, $value);
4843             } elsif ($opt eq 'balloon') {
4844                 # enable/disable balloning device is not hotpluggable
4845                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4846                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4847                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4848
4849                 # allow manual ballooning if shares is set to zero
4850                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4851                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4852                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4853                 }
4854             } elsif ($opt =~ m/^net(\d+)$/) {
4855                 # some changes can be done without hotplug
4856                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4857                                     $vmid, $opt, $value, $arch, $machine_type);
4858             } elsif (is_valid_drivename($opt)) {
4859                 # some changes can be done without hotplug
4860                 my $drive = parse_drive($opt, $value);
4861                 if (drive_is_cloudinit($drive)) {
4862                     &$apply_pending_cloudinit($opt, $value);
4863                 }
4864                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4865                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4866             } elsif ($opt =~ m/^memory$/) { #dimms
4867                 die "skip\n" if !$hotplug_features->{memory};
4868                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4869             } elsif ($opt eq 'cpuunits') {
4870                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4871             } elsif ($opt eq 'cpulimit') {
4872                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4873                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4874             } else {
4875                 die "skip\n";  # skip non-hot-pluggable options
4876             }
4877         };
4878         if (my $err = $@) {
4879             &$add_error($opt, $err) if $err ne "skip\n";
4880         } else {
4881             # save new config if hotplug was successful
4882             $conf->{$opt} = $value;
4883             delete $conf->{pending}->{$opt};
4884             PVE::QemuConfig->write_config($vmid, $conf);
4885             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4886         }
4887     }
4888 }
4889
4890 sub try_deallocate_drive {
4891     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4892
4893     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4894         my $volid = $drive->{file};
4895         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4896             my $sid = PVE::Storage::parse_volume_id($volid);
4897             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4898
4899             # check if the disk is really unused
4900             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4901                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4902             PVE::Storage::vdisk_free($storecfg, $volid);
4903             return 1;
4904         } else {
4905             # If vm is not owner of this disk remove from config
4906             return 1;
4907         }
4908     }
4909
4910     return undef;
4911 }
4912
4913 sub vmconfig_delete_or_detach_drive {
4914     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4915
4916     my $drive = parse_drive($opt, $conf->{$opt});
4917
4918     my $rpcenv = PVE::RPCEnvironment::get();
4919     my $authuser = $rpcenv->get_user();
4920
4921     if ($force) {
4922         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4923         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4924     } else {
4925         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4926     }
4927 }
4928
4929 sub vmconfig_apply_pending {
4930     my ($vmid, $conf, $storecfg) = @_;
4931
4932     # cold plug
4933
4934     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4935     while (my ($opt, $force) = each %$pending_delete_hash) {
4936         die "internal error" if $opt =~ m/^unused/;
4937         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4938         if (!defined($conf->{$opt})) {
4939             vmconfig_undelete_pending_option($conf, $opt);
4940             PVE::QemuConfig->write_config($vmid, $conf);
4941         } elsif (is_valid_drivename($opt)) {
4942             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4943             vmconfig_undelete_pending_option($conf, $opt);
4944             delete $conf->{$opt};
4945             PVE::QemuConfig->write_config($vmid, $conf);
4946         } else {
4947             vmconfig_undelete_pending_option($conf, $opt);
4948             delete $conf->{$opt};
4949             PVE::QemuConfig->write_config($vmid, $conf);
4950         }
4951     }
4952
4953     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4954
4955     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4956         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4957
4958         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4959             # skip if nothing changed
4960         } elsif (is_valid_drivename($opt)) {
4961             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4962                 if defined($conf->{$opt});
4963             $conf->{$opt} = $conf->{pending}->{$opt};
4964         } else {
4965             $conf->{$opt} = $conf->{pending}->{$opt};
4966         }
4967
4968         delete $conf->{pending}->{$opt};
4969         PVE::QemuConfig->write_config($vmid, $conf);
4970     }
4971 }
4972
4973 my $safe_num_ne = sub {
4974     my ($a, $b) = @_;
4975
4976     return 0 if !defined($a) && !defined($b);
4977     return 1 if !defined($a);
4978     return 1 if !defined($b);
4979
4980     return $a != $b;
4981 };
4982
4983 my $safe_string_ne = sub {
4984     my ($a, $b) = @_;
4985
4986     return 0 if !defined($a) && !defined($b);
4987     return 1 if !defined($a);
4988     return 1 if !defined($b);
4989
4990     return $a ne $b;
4991 };
4992
4993 sub vmconfig_update_net {
4994     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4995
4996     my $newnet = parse_net($value);
4997
4998     if ($conf->{$opt}) {
4999         my $oldnet = parse_net($conf->{$opt});
5000
5001         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5002             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5003             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5004             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5005
5006             # for non online change, we try to hot-unplug
5007             die "skip\n" if !$hotplug;
5008             vm_deviceunplug($vmid, $conf, $opt);
5009         } else {
5010
5011             die "internal error" if $opt !~ m/net(\d+)/;
5012             my $iface = "tap${vmid}i$1";
5013
5014             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5015                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5016                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5017                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5018                 PVE::Network::tap_unplug($iface);
5019                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5020             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5021                 # Rate can be applied on its own but any change above needs to
5022                 # include the rate in tap_plug since OVS resets everything.
5023                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5024             }
5025
5026             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5027                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5028             }
5029
5030             return 1;
5031         }
5032     }
5033
5034     if ($hotplug) {
5035         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5036     } else {
5037         die "skip\n";
5038     }
5039 }
5040
5041 sub vmconfig_update_disk {
5042     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5043
5044     # fixme: do we need force?
5045
5046     my $drive = parse_drive($opt, $value);
5047
5048     if ($conf->{$opt}) {
5049
5050         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5051
5052             my $media = $drive->{media} || 'disk';
5053             my $oldmedia = $old_drive->{media} || 'disk';
5054             die "unable to change media type\n" if $media ne $oldmedia;
5055
5056             if (!drive_is_cdrom($old_drive)) {
5057
5058                 if ($drive->{file} ne $old_drive->{file}) {
5059
5060                     die "skip\n" if !$hotplug;
5061
5062                     # unplug and register as unused
5063                     vm_deviceunplug($vmid, $conf, $opt);
5064                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5065
5066                 } else {
5067                     # update existing disk
5068
5069                     # skip non hotpluggable value
5070                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5071                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5072                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5073                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5074                         die "skip\n";
5075                     }
5076
5077                     # apply throttle
5078                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5079                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5080                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5081                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5082                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5083                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5084                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5085                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5086                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5087                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5088                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5089                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5090                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5091                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5092                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5093                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5094                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5095                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5096
5097                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5098                                                    ($drive->{mbps} || 0)*1024*1024,
5099                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5100                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5101                                                    $drive->{iops} || 0,
5102                                                    $drive->{iops_rd} || 0,
5103                                                    $drive->{iops_wr} || 0,
5104                                                    ($drive->{mbps_max} || 0)*1024*1024,
5105                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5106                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5107                                                    $drive->{iops_max} || 0,
5108                                                    $drive->{iops_rd_max} || 0,
5109                                                    $drive->{iops_wr_max} || 0,
5110                                                    $drive->{bps_max_length} || 1,
5111                                                    $drive->{bps_rd_max_length} || 1,
5112                                                    $drive->{bps_wr_max_length} || 1,
5113                                                    $drive->{iops_max_length} || 1,
5114                                                    $drive->{iops_rd_max_length} || 1,
5115                                                    $drive->{iops_wr_max_length} || 1);
5116
5117                     }
5118
5119                     return 1;
5120                 }
5121
5122             } else { # cdrom
5123
5124                 if ($drive->{file} eq 'none') {
5125                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5126                     if (drive_is_cloudinit($old_drive)) {
5127                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5128                     }
5129                 } else {
5130                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5131                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5132                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5133                 }
5134
5135                 return 1;
5136             }
5137         }
5138     }
5139
5140     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5141     # hotplug new disks
5142     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5143     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5144 }
5145
5146 sub vm_start {
5147     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5148         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5149
5150     PVE::QemuConfig->lock_config($vmid, sub {
5151         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5152
5153         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5154
5155         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5156
5157         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5158
5159         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5160             vmconfig_apply_pending($vmid, $conf, $storecfg);
5161             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5162         }
5163
5164         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5165
5166         my $defaults = load_defaults();
5167
5168         # set environment variable useful inside network script
5169         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5170
5171         my $local_volumes = {};
5172
5173         if ($targetstorage) {
5174             foreach_drive($conf, sub {
5175                 my ($ds, $drive) = @_;
5176
5177                 return if drive_is_cdrom($drive);
5178
5179                 my $volid = $drive->{file};
5180
5181                 return if !$volid;
5182
5183                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5184
5185                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5186                 return if $scfg->{shared};
5187                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5188             });
5189
5190             my $format = undef;
5191
5192             foreach my $opt (sort keys %$local_volumes) {
5193
5194                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5195                 my $drive = parse_drive($opt, $conf->{$opt});
5196
5197                 #if remote storage is specified, use default format
5198                 if ($targetstorage && $targetstorage ne "1") {
5199                     $storeid = $targetstorage;
5200                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5201                     $format = $defFormat;
5202                 } else {
5203                     #else we use same format than original
5204                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5205                     $format = qemu_img_format($scfg, $volid);
5206                 }
5207
5208                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5209                 my $newdrive = $drive;
5210                 $newdrive->{format} = $format;
5211                 $newdrive->{file} = $newvolid;
5212                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5213                 $local_volumes->{$opt} = $drivestr;
5214                 #pass drive to conf for command line
5215                 $conf->{$opt} = $drivestr;
5216             }
5217         }
5218
5219         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5220
5221         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5222
5223         my $migrate_port = 0;
5224         my $migrate_uri;
5225         if ($statefile) {
5226             if ($statefile eq 'tcp') {
5227                 my $localip = "localhost";
5228                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5229                 my $nodename = PVE::INotify::nodename();
5230
5231                 if (!defined($migration_type)) {
5232                     if (defined($datacenterconf->{migration}->{type})) {
5233                         $migration_type = $datacenterconf->{migration}->{type};
5234                     } else {
5235                         $migration_type = 'secure';
5236                     }
5237                 }
5238
5239                 if ($migration_type eq 'insecure') {
5240                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5241                     if ($migrate_network_addr) {
5242                         $localip = $migrate_network_addr;
5243                     } else {
5244                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5245                     }
5246
5247                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5248                 }
5249
5250                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5251                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5252                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5253                 push @$cmd, '-incoming', $migrate_uri;
5254                 push @$cmd, '-S';
5255
5256             } elsif ($statefile eq 'unix') {
5257                 # should be default for secure migrations as a ssh TCP forward
5258                 # tunnel is not deterministic reliable ready and fails regurarly
5259                 # to set up in time, so use UNIX socket forwards
5260                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5261                 unlink $socket_addr;
5262
5263                 $migrate_uri = "unix:$socket_addr";
5264
5265                 push @$cmd, '-incoming', $migrate_uri;
5266                 push @$cmd, '-S';
5267
5268             } else {
5269                 push @$cmd, '-loadstate', $statefile;
5270             }
5271         } elsif ($paused) {
5272             push @$cmd, '-S';
5273         }
5274
5275         # host pci devices
5276         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5277           my $d = parse_hostpci($conf->{"hostpci$i"});
5278           next if !$d;
5279           my $pcidevices = $d->{pciid};
5280           foreach my $pcidevice (@$pcidevices) {
5281                 my $pciid = $pcidevice->{id};
5282
5283                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5284                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5285                 die "no pci device info for device '$pciid'\n" if !$info;
5286
5287                 if ($d->{mdev}) {
5288                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5289                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5290                 } else {
5291                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5292                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5293                     die "can't reset pci device '$pciid'\n"
5294                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5295                 }
5296           }
5297         }
5298
5299         PVE::Storage::activate_volumes($storecfg, $vollist);
5300
5301         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5302             eval {
5303                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5304                     outfunc => sub {}, errfunc => sub {});
5305             };
5306         }
5307
5308         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5309                                                   : $defaults->{cpuunits};
5310
5311         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5312         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5313
5314         my %properties = (
5315             Slice => 'qemu.slice',
5316             KillMode => 'none',
5317             CPUShares => $cpuunits
5318         );
5319
5320         if (my $cpulimit = $conf->{cpulimit}) {
5321             $properties{CPUQuota} = int($cpulimit * 100);
5322         }
5323         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5324
5325         my $run_qemu = sub {
5326             PVE::Tools::run_fork sub {
5327                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5328                 run_command($cmd, %run_params);
5329             };
5330         };
5331
5332         if ($conf->{hugepages}) {
5333
5334             my $code = sub {
5335                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5336                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5337
5338                 PVE::QemuServer::Memory::hugepages_mount();
5339                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5340
5341                 eval { $run_qemu->() };
5342                 if (my $err = $@) {
5343                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5344                     die $err;
5345                 }
5346
5347                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5348             };
5349             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5350
5351         } else {
5352             eval { $run_qemu->() };
5353         }
5354
5355         if (my $err = $@) {
5356             # deactivate volumes if start fails
5357             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5358             die "start failed: $err";
5359         }
5360
5361         print "migration listens on $migrate_uri\n" if $migrate_uri;
5362
5363         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5364             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5365             warn $@ if $@;
5366         }
5367
5368         #start nbd server for storage migration
5369         if ($targetstorage) {
5370             my $nodename = PVE::INotify::nodename();
5371             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5372             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5373             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5374             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5375
5376             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5377
5378             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5379
5380             foreach my $opt (sort keys %$local_volumes) {
5381                 my $volid = $local_volumes->{$opt};
5382                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5383                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5384                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5385             }
5386         }
5387
5388         if ($migratedfrom) {
5389             eval {
5390                 set_migration_caps($vmid);
5391             };
5392             warn $@ if $@;
5393
5394             if ($spice_port) {
5395                 print "spice listens on port $spice_port\n";
5396                 if ($spice_ticket) {
5397                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5398                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5399                 }
5400             }
5401
5402         } else {
5403             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5404                 if !$statefile && $conf->{balloon};
5405
5406             foreach my $opt (keys %$conf) {
5407                 next if $opt !~  m/^net\d+$/;
5408                 my $nicconf = parse_net($conf->{$opt});
5409                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5410             }
5411         }
5412
5413         vm_mon_cmd_nocheck($vmid, 'qom-set',
5414                     path => "machine/peripheral/balloon0",
5415                     property => "guest-stats-polling-interval",
5416                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5417
5418         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5419     });
5420 }
5421
5422 sub vm_mon_cmd {
5423     my ($vmid, $execute, %params) = @_;
5424
5425     my $cmd = { execute => $execute, arguments => \%params };
5426     vm_qmp_command($vmid, $cmd);
5427 }
5428
5429 sub vm_mon_cmd_nocheck {
5430     my ($vmid, $execute, %params) = @_;
5431
5432     my $cmd = { execute => $execute, arguments => \%params };
5433     vm_qmp_command($vmid, $cmd, 1);
5434 }
5435
5436 sub vm_qmp_command {
5437     my ($vmid, $cmd, $nocheck) = @_;
5438
5439     my $res;
5440
5441     my $timeout;
5442     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5443         $timeout = $cmd->{arguments}->{timeout};
5444         delete $cmd->{arguments}->{timeout};
5445     }
5446
5447     eval {
5448         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5449         my $sname = qmp_socket($vmid);
5450         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5451             my $qmpclient = PVE::QMPClient->new();
5452
5453             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5454         } else {
5455             die "unable to open monitor socket\n";
5456         }
5457     };
5458     if (my $err = $@) {
5459         syslog("err", "VM $vmid qmp command failed - $err");
5460         die $err;
5461     }
5462
5463     return $res;
5464 }
5465
5466 sub vm_human_monitor_command {
5467     my ($vmid, $cmdline) = @_;
5468
5469     my $res;
5470
5471     my $cmd = {
5472         execute => 'human-monitor-command',
5473         arguments => { 'command-line' => $cmdline},
5474     };
5475
5476     return vm_qmp_command($vmid, $cmd);
5477 }
5478
5479 sub vm_commandline {
5480     my ($storecfg, $vmid, $snapname) = @_;
5481
5482     my $conf = PVE::QemuConfig->load_config($vmid);
5483
5484     if ($snapname) {
5485         my $snapshot = $conf->{snapshots}->{$snapname};
5486         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5487
5488         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5489
5490         $conf = $snapshot;
5491     }
5492
5493     my $defaults = load_defaults();
5494
5495     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5496
5497     return PVE::Tools::cmd2string($cmd);
5498 }
5499
5500 sub vm_reset {
5501     my ($vmid, $skiplock) = @_;
5502
5503     PVE::QemuConfig->lock_config($vmid, sub {
5504
5505         my $conf = PVE::QemuConfig->load_config($vmid);
5506
5507         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5508
5509         vm_mon_cmd($vmid, "system_reset");
5510     });
5511 }
5512
5513 sub get_vm_volumes {
5514     my ($conf) = @_;
5515
5516     my $vollist = [];
5517     foreach_volid($conf, sub {
5518         my ($volid, $attr) = @_;
5519
5520         return if $volid =~ m|^/|;
5521
5522         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5523         return if !$sid;
5524
5525         push @$vollist, $volid;
5526     });
5527
5528     return $vollist;
5529 }
5530
5531 sub vm_stop_cleanup {
5532     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5533
5534     eval {
5535
5536         if (!$keepActive) {
5537             my $vollist = get_vm_volumes($conf);
5538             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5539         }
5540
5541         foreach my $ext (qw(mon qmp pid vnc qga)) {
5542             unlink "/var/run/qemu-server/${vmid}.$ext";
5543         }
5544
5545         if ($conf->{ivshmem}) {
5546             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5547             # just delete it for now, VMs which have this already open do not
5548             # are affected, but new VMs will get a separated one. If this
5549             # becomes an issue we either add some sort of ref-counting or just
5550             # add a "don't delete on stop" flag to the ivshmem format.
5551             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5552         }
5553
5554         foreach my $key (keys %$conf) {
5555             next if $key !~ m/^hostpci(\d+)$/;
5556             my $hostpciindex = $1;
5557             my $d = parse_hostpci($conf->{$key});
5558             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5559
5560             foreach my $pci (@{$d->{pciid}}) {
5561                 my $pciid = $pci->{id};
5562                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5563             }
5564         }
5565
5566         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5567     };
5568     warn $@ if $@; # avoid errors - just warn
5569 }
5570
5571 # Note: use $nockeck to skip tests if VM configuration file exists.
5572 # We need that when migration VMs to other nodes (files already moved)
5573 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5574 sub vm_stop {
5575     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5576
5577     $force = 1 if !defined($force) && !$shutdown;
5578
5579     if ($migratedfrom){
5580         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5581         kill 15, $pid if $pid;
5582         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5583         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5584         return;
5585     }
5586
5587     PVE::QemuConfig->lock_config($vmid, sub {
5588
5589         my $pid = check_running($vmid, $nocheck);
5590         return if !$pid;
5591
5592         my $conf;
5593         if (!$nocheck) {
5594             $conf = PVE::QemuConfig->load_config($vmid);
5595             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5596             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5597                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5598                 $timeout = $opts->{down} if $opts->{down};
5599             }
5600             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5601         }
5602
5603         $timeout = 60 if !defined($timeout);
5604
5605         eval {
5606             if ($shutdown) {
5607                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5608                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5609                 } else {
5610                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5611                 }
5612             } else {
5613                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5614             }
5615         };
5616         my $err = $@;
5617
5618         if (!$err) {
5619             my $count = 0;
5620             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5621                 $count++;
5622                 sleep 1;
5623             }
5624
5625             if ($count >= $timeout) {
5626                 if ($force) {
5627                     warn "VM still running - terminating now with SIGTERM\n";
5628                     kill 15, $pid;
5629                 } else {
5630                     die "VM quit/powerdown failed - got timeout\n";
5631                 }
5632             } else {
5633                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5634                 return;
5635             }
5636         } else {
5637             if ($force) {
5638                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5639                 kill 15, $pid;
5640             } else {
5641                 die "VM quit/powerdown failed\n";
5642             }
5643         }
5644
5645         # wait again
5646         $timeout = 10;
5647
5648         my $count = 0;
5649         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5650             $count++;
5651             sleep 1;
5652         }
5653
5654         if ($count >= $timeout) {
5655             warn "VM still running - terminating now with SIGKILL\n";
5656             kill 9, $pid;
5657             sleep 1;
5658         }
5659
5660         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5661    });
5662 }
5663
5664 sub vm_suspend {
5665     my ($vmid, $skiplock) = @_;
5666
5667     PVE::QemuConfig->lock_config($vmid, sub {
5668
5669         my $conf = PVE::QemuConfig->load_config($vmid);
5670
5671         PVE::QemuConfig->check_lock($conf)
5672             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5673
5674         vm_mon_cmd($vmid, "stop");
5675     });
5676 }
5677
5678 sub vm_resume {
5679     my ($vmid, $skiplock, $nocheck) = @_;
5680
5681     PVE::QemuConfig->lock_config($vmid, sub {
5682
5683         my $res = vm_mon_cmd($vmid, 'query-status');
5684         my $resume_cmd = 'cont';
5685
5686         if ($res->{status} && $res->{status} eq 'suspended') {
5687             $resume_cmd = 'system_wakeup';
5688         }
5689
5690         if (!$nocheck) {
5691
5692             my $conf = PVE::QemuConfig->load_config($vmid);
5693
5694             PVE::QemuConfig->check_lock($conf)
5695                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5696
5697             vm_mon_cmd($vmid, $resume_cmd);
5698
5699         } else {
5700             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5701         }
5702     });
5703 }
5704
5705 sub vm_sendkey {
5706     my ($vmid, $skiplock, $key) = @_;
5707
5708     PVE::QemuConfig->lock_config($vmid, sub {
5709
5710         my $conf = PVE::QemuConfig->load_config($vmid);
5711
5712         # there is no qmp command, so we use the human monitor command
5713         vm_human_monitor_command($vmid, "sendkey $key");
5714     });
5715 }
5716
5717 sub vm_destroy {
5718     my ($storecfg, $vmid, $skiplock) = @_;
5719
5720     PVE::QemuConfig->lock_config($vmid, sub {
5721
5722         my $conf = PVE::QemuConfig->load_config($vmid);
5723
5724         if (!check_running($vmid)) {
5725             destroy_vm($storecfg, $vmid, undef, $skiplock);
5726         } else {
5727             die "VM $vmid is running - destroy failed\n";
5728         }
5729     });
5730 }
5731
5732 # vzdump restore implementaion
5733
5734 sub tar_archive_read_firstfile {
5735     my $archive = shift;
5736
5737     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5738
5739     # try to detect archive type first
5740     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5741         die "unable to open file '$archive'\n";
5742     my $firstfile = <$fh>;
5743     kill 15, $pid;
5744     close $fh;
5745
5746     die "ERROR: archive contaions no data\n" if !$firstfile;
5747     chomp $firstfile;
5748
5749     return $firstfile;
5750 }
5751
5752 sub tar_restore_cleanup {
5753     my ($storecfg, $statfile) = @_;
5754
5755     print STDERR "starting cleanup\n";
5756
5757     if (my $fd = IO::File->new($statfile, "r")) {
5758         while (defined(my $line = <$fd>)) {
5759             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5760                 my $volid = $2;
5761                 eval {
5762                     if ($volid =~ m|^/|) {
5763                         unlink $volid || die 'unlink failed\n';
5764                     } else {
5765                         PVE::Storage::vdisk_free($storecfg, $volid);
5766                     }
5767                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5768                 };
5769                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5770             } else {
5771                 print STDERR "unable to parse line in statfile - $line";
5772             }
5773         }
5774         $fd->close();
5775     }
5776 }
5777
5778 sub restore_archive {
5779     my ($archive, $vmid, $user, $opts) = @_;
5780
5781     my $format = $opts->{format};
5782     my $comp;
5783
5784     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5785         $format = 'tar' if !$format;
5786         $comp = 'gzip';
5787     } elsif ($archive =~ m/\.tar$/) {
5788         $format = 'tar' if !$format;
5789     } elsif ($archive =~ m/.tar.lzo$/) {
5790         $format = 'tar' if !$format;
5791         $comp = 'lzop';
5792     } elsif ($archive =~ m/\.vma$/) {
5793         $format = 'vma' if !$format;
5794     } elsif ($archive =~ m/\.vma\.gz$/) {
5795         $format = 'vma' if !$format;
5796         $comp = 'gzip';
5797     } elsif ($archive =~ m/\.vma\.lzo$/) {
5798         $format = 'vma' if !$format;
5799         $comp = 'lzop';
5800     } else {
5801         $format = 'vma' if !$format; # default
5802     }
5803
5804     # try to detect archive format
5805     if ($format eq 'tar') {
5806         return restore_tar_archive($archive, $vmid, $user, $opts);
5807     } else {
5808         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5809     }
5810 }
5811
5812 sub restore_update_config_line {
5813     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5814
5815     return if $line =~ m/^\#qmdump\#/;
5816     return if $line =~ m/^\#vzdump\#/;
5817     return if $line =~ m/^lock:/;
5818     return if $line =~ m/^unused\d+:/;
5819     return if $line =~ m/^parent:/;
5820     return if $line =~ m/^template:/; # restored VM is never a template
5821
5822     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5823     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5824         # try to convert old 1.X settings
5825         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5826         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5827             my ($model, $macaddr) = split(/\=/, $devconfig);
5828             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5829             my $net = {
5830                 model => $model,
5831                 bridge => "vmbr$ind",
5832                 macaddr => $macaddr,
5833             };
5834             my $netstr = print_net($net);
5835
5836             print $outfd "net$cookie->{netcount}: $netstr\n";
5837             $cookie->{netcount}++;
5838         }
5839     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5840         my ($id, $netstr) = ($1, $2);
5841         my $net = parse_net($netstr);
5842         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5843         $netstr = print_net($net);
5844         print $outfd "$id: $netstr\n";
5845     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5846         my $virtdev = $1;
5847         my $value = $3;
5848         my $di = parse_drive($virtdev, $value);
5849         if (defined($di->{backup}) && !$di->{backup}) {
5850             print $outfd "#$line";
5851         } elsif ($map->{$virtdev}) {
5852             delete $di->{format}; # format can change on restore
5853             $di->{file} = $map->{$virtdev};
5854             $value = print_drive($vmid, $di);
5855             print $outfd "$virtdev: $value\n";
5856         } else {
5857             print $outfd $line;
5858         }
5859     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5860         my $vmgenid = $1;
5861         if ($vmgenid ne '0') {
5862             # always generate a new vmgenid if there was a valid one setup
5863             $vmgenid = generate_uuid();
5864         }
5865         print $outfd "vmgenid: $vmgenid\n";
5866     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5867         my ($uuid, $uuid_str);
5868         UUID::generate($uuid);
5869         UUID::unparse($uuid, $uuid_str);
5870         my $smbios1 = parse_smbios1($2);
5871         $smbios1->{uuid} = $uuid_str;
5872         print $outfd $1.print_smbios1($smbios1)."\n";
5873     } else {
5874         print $outfd $line;
5875     }
5876 }
5877
5878 sub scan_volids {
5879     my ($cfg, $vmid) = @_;
5880
5881     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5882
5883     my $volid_hash = {};
5884     foreach my $storeid (keys %$info) {
5885         foreach my $item (@{$info->{$storeid}}) {
5886             next if !($item->{volid} && $item->{size});
5887             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5888             $volid_hash->{$item->{volid}} = $item;
5889         }
5890     }
5891
5892     return $volid_hash;
5893 }
5894
5895 sub is_volume_in_use {
5896     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5897
5898     my $path = PVE::Storage::path($storecfg, $volid);
5899
5900     my $scan_config = sub {
5901         my ($cref, $snapname) = @_;
5902
5903         foreach my $key (keys %$cref) {
5904             my $value = $cref->{$key};
5905             if (is_valid_drivename($key)) {
5906                 next if $skip_drive && $key eq $skip_drive;
5907                 my $drive = parse_drive($key, $value);
5908                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5909                 return 1 if $volid eq $drive->{file};
5910                 if ($drive->{file} =~ m!^/!) {
5911                     return 1 if $drive->{file} eq $path;
5912                 } else {
5913                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5914                     next if !$storeid;
5915                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5916                     next if !$scfg;
5917                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5918                 }
5919             }
5920         }
5921
5922         return 0;
5923     };
5924
5925     return 1 if &$scan_config($conf);
5926
5927     undef $skip_drive;
5928
5929     foreach my $snapname (keys %{$conf->{snapshots}}) {
5930         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5931     }
5932
5933     return 0;
5934 }
5935
5936 sub update_disksize {
5937     my ($vmid, $conf, $volid_hash) = @_;
5938
5939     my $changes;
5940     my $prefix = "VM $vmid:";
5941
5942     # used and unused disks
5943     my $referenced = {};
5944
5945     # Note: it is allowed to define multiple storages with same path (alias), so
5946     # we need to check both 'volid' and real 'path' (two different volid can point
5947     # to the same path).
5948
5949     my $referencedpath = {};
5950
5951     # update size info
5952     foreach my $opt (keys %$conf) {
5953         if (is_valid_drivename($opt)) {
5954             my $drive = parse_drive($opt, $conf->{$opt});
5955             my $volid = $drive->{file};
5956             next if !$volid;
5957
5958             $referenced->{$volid} = 1;
5959             if ($volid_hash->{$volid} &&
5960                 (my $path = $volid_hash->{$volid}->{path})) {
5961                 $referencedpath->{$path} = 1;
5962             }
5963
5964             next if drive_is_cdrom($drive);
5965             next if !$volid_hash->{$volid};
5966
5967             $drive->{size} = $volid_hash->{$volid}->{size};
5968             my $new = print_drive($vmid, $drive);
5969             if ($new ne $conf->{$opt}) {
5970                 $changes = 1;
5971                 $conf->{$opt} = $new;
5972                 print "$prefix update disk '$opt' information.\n";
5973             }
5974         }
5975     }
5976
5977     # remove 'unusedX' entry if volume is used
5978     foreach my $opt (keys %$conf) {
5979         next if $opt !~ m/^unused\d+$/;
5980         my $volid = $conf->{$opt};
5981         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5982         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5983             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5984             $changes = 1;
5985             delete $conf->{$opt};
5986         }
5987
5988         $referenced->{$volid} = 1;
5989         $referencedpath->{$path} = 1 if $path;
5990     }
5991
5992     foreach my $volid (sort keys %$volid_hash) {
5993         next if $volid =~ m/vm-$vmid-state-/;
5994         next if $referenced->{$volid};
5995         my $path = $volid_hash->{$volid}->{path};
5996         next if !$path; # just to be sure
5997         next if $referencedpath->{$path};
5998         $changes = 1;
5999         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6000         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6001         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6002     }
6003
6004     return $changes;
6005 }
6006
6007 sub rescan {
6008     my ($vmid, $nolock, $dryrun) = @_;
6009
6010     my $cfg = PVE::Storage::config();
6011
6012     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6013     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6014     foreach my $stor (keys %{$cfg->{ids}}) {
6015         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6016     }
6017
6018     print "rescan volumes...\n";
6019     my $volid_hash = scan_volids($cfg, $vmid);
6020
6021     my $updatefn =  sub {
6022         my ($vmid) = @_;
6023
6024         my $conf = PVE::QemuConfig->load_config($vmid);
6025
6026         PVE::QemuConfig->check_lock($conf);
6027
6028         my $vm_volids = {};
6029         foreach my $volid (keys %$volid_hash) {
6030             my $info = $volid_hash->{$volid};
6031             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6032         }
6033
6034         my $changes = update_disksize($vmid, $conf, $vm_volids);
6035
6036         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6037     };
6038
6039     if (defined($vmid)) {
6040         if ($nolock) {
6041             &$updatefn($vmid);
6042         } else {
6043             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6044         }
6045     } else {
6046         my $vmlist = config_list();
6047         foreach my $vmid (keys %$vmlist) {
6048             if ($nolock) {
6049                 &$updatefn($vmid);
6050             } else {
6051                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6052             }
6053         }
6054     }
6055 }
6056
6057 sub restore_vma_archive {
6058     my ($archive, $vmid, $user, $opts, $comp) = @_;
6059
6060     my $readfrom = $archive;
6061
6062     my $cfg = PVE::Storage::config();
6063     my $commands = [];
6064     my $bwlimit = $opts->{bwlimit};
6065
6066     my $dbg_cmdstring = '';
6067     my $add_pipe = sub {
6068         my ($cmd) = @_;
6069         push @$commands, $cmd;
6070         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6071         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6072         $readfrom = '-';
6073     };
6074
6075     my $input = undef;
6076     if ($archive eq '-') {
6077         $input = '<&STDIN';
6078     } else {
6079         # If we use a backup from a PVE defined storage we also consider that
6080         # storage's rate limit:
6081         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6082         if (defined($volid)) {
6083             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6084             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6085             if ($readlimit) {
6086                 print STDERR "applying read rate limit: $readlimit\n";
6087                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6088                 $add_pipe->($cstream);
6089             }
6090         }
6091     }
6092
6093     if ($comp) {
6094         my $cmd;
6095         if ($comp eq 'gzip') {
6096             $cmd = ['zcat', $readfrom];
6097         } elsif ($comp eq 'lzop') {
6098             $cmd = ['lzop', '-d', '-c', $readfrom];
6099         } else {
6100             die "unknown compression method '$comp'\n";
6101         }
6102         $add_pipe->($cmd);
6103     }
6104
6105     my $tmpdir = "/var/tmp/vzdumptmp$$";
6106     rmtree $tmpdir;
6107
6108     # disable interrupts (always do cleanups)
6109     local $SIG{INT} =
6110         local $SIG{TERM} =
6111         local $SIG{QUIT} =
6112         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6113
6114     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6115     POSIX::mkfifo($mapfifo, 0600);
6116     my $fifofh;
6117
6118     my $openfifo = sub {
6119         open($fifofh, '>', $mapfifo) || die $!;
6120     };
6121
6122     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6123
6124     my $oldtimeout;
6125     my $timeout = 5;
6126
6127     my $devinfo = {};
6128
6129     my $rpcenv = PVE::RPCEnvironment::get();
6130
6131     my $conffile = PVE::QemuConfig->config_file($vmid);
6132     my $tmpfn = "$conffile.$$.tmp";
6133
6134     # Note: $oldconf is undef if VM does not exists
6135     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6136     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6137
6138     my %storage_limits;
6139
6140     my $print_devmap = sub {
6141         my $virtdev_hash = {};
6142
6143         my $cfgfn = "$tmpdir/qemu-server.conf";
6144
6145         # we can read the config - that is already extracted
6146         my $fh = IO::File->new($cfgfn, "r") ||
6147             "unable to read qemu-server.conf - $!\n";
6148
6149         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6150         if (-f $fwcfgfn) {
6151             my $pve_firewall_dir = '/etc/pve/firewall';
6152             mkdir $pve_firewall_dir; # make sure the dir exists
6153             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6154         }
6155
6156         while (defined(my $line = <$fh>)) {
6157             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6158                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6159                 die "archive does not contain data for drive '$virtdev'\n"
6160                     if !$devinfo->{$devname};
6161                 if (defined($opts->{storage})) {
6162                     $storeid = $opts->{storage} || 'local';
6163                 } elsif (!$storeid) {
6164                     $storeid = 'local';
6165                 }
6166                 $format = 'raw' if !$format;
6167                 $devinfo->{$devname}->{devname} = $devname;
6168                 $devinfo->{$devname}->{virtdev} = $virtdev;
6169                 $devinfo->{$devname}->{format} = $format;
6170                 $devinfo->{$devname}->{storeid} = $storeid;
6171
6172                 # check permission on storage
6173                 my $pool = $opts->{pool}; # todo: do we need that?
6174                 if ($user ne 'root@pam') {
6175                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6176                 }
6177
6178                 $storage_limits{$storeid} = $bwlimit;
6179
6180                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6181             }
6182         }
6183
6184         foreach my $key (keys %storage_limits) {
6185             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6186             next if !$limit;
6187             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6188             $storage_limits{$key} = $limit * 1024;
6189         }
6190
6191         foreach my $devname (keys %$devinfo) {
6192             die "found no device mapping information for device '$devname'\n"
6193                 if !$devinfo->{$devname}->{virtdev};
6194         }
6195
6196         # create empty/temp config
6197         if ($oldconf) {
6198             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6199             foreach_drive($oldconf, sub {
6200                 my ($ds, $drive) = @_;
6201
6202                 return if drive_is_cdrom($drive);
6203
6204                 my $volid = $drive->{file};
6205
6206                 return if !$volid || $volid =~ m|^/|;
6207
6208                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6209                 return if !$path || !$owner || ($owner != $vmid);
6210
6211                 # Note: only delete disk we want to restore
6212                 # other volumes will become unused
6213                 if ($virtdev_hash->{$ds}) {
6214                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6215                     if (my $err = $@) {
6216                         warn $err;
6217                     }
6218                 }
6219             });
6220
6221             # delete vmstate files
6222             # since after the restore we have no snapshots anymore
6223             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6224                 my $snap = $oldconf->{snapshots}->{$snapname};
6225                 if ($snap->{vmstate}) {
6226                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6227                     if (my $err = $@) {
6228                         warn $err;
6229                     }
6230                 }
6231             }
6232         }
6233
6234         my $map = {};
6235         foreach my $virtdev (sort keys %$virtdev_hash) {
6236             my $d = $virtdev_hash->{$virtdev};
6237             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6238             my $storeid = $d->{storeid};
6239             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6240
6241             my $map_opts = '';
6242             if (my $limit = $storage_limits{$storeid}) {
6243                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6244             }
6245
6246             # test if requested format is supported
6247             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6248             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6249             $d->{format} = $defFormat if !$supported;
6250
6251             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6252                                                   $d->{format}, undef, $alloc_size);
6253             print STDERR "new volume ID is '$volid'\n";
6254             $d->{volid} = $volid;
6255             my $path = PVE::Storage::path($cfg, $volid);
6256
6257             PVE::Storage::activate_volumes($cfg,[$volid]);
6258
6259             my $write_zeros = 1;
6260             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6261                 $write_zeros = 0;
6262             }
6263
6264             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6265
6266             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6267             $map->{$virtdev} = $volid;
6268         }
6269
6270         $fh->seek(0, 0) || die "seek failed - $!\n";
6271
6272         my $outfd = new IO::File ($tmpfn, "w") ||
6273             die "unable to write config for VM $vmid\n";
6274
6275         my $cookie = { netcount => 0 };
6276         while (defined(my $line = <$fh>)) {
6277             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6278         }
6279
6280         $fh->close();
6281         $outfd->close();
6282     };
6283
6284     eval {
6285         # enable interrupts
6286         local $SIG{INT} =
6287             local $SIG{TERM} =
6288             local $SIG{QUIT} =
6289             local $SIG{HUP} =
6290             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6291         local $SIG{ALRM} = sub { die "got timeout\n"; };
6292
6293         $oldtimeout = alarm($timeout);
6294
6295         my $parser = sub {
6296             my $line = shift;
6297
6298             print "$line\n";
6299
6300             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6301                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6302                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6303             } elsif ($line =~ m/^CTIME: /) {
6304                 # we correctly received the vma config, so we can disable
6305                 # the timeout now for disk allocation (set to 10 minutes, so
6306                 # that we always timeout if something goes wrong)
6307                 alarm(600);
6308                 &$print_devmap();
6309                 print $fifofh "done\n";
6310                 my $tmp = $oldtimeout || 0;
6311                 $oldtimeout = undef;
6312                 alarm($tmp);
6313                 close($fifofh);
6314             }
6315         };
6316
6317         print "restore vma archive: $dbg_cmdstring\n";
6318         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6319     };
6320     my $err = $@;
6321
6322     alarm($oldtimeout) if $oldtimeout;
6323
6324     my $vollist = [];
6325     foreach my $devname (keys %$devinfo) {
6326         my $volid = $devinfo->{$devname}->{volid};
6327         push @$vollist, $volid if $volid;
6328     }
6329
6330     PVE::Storage::deactivate_volumes($cfg, $vollist);
6331
6332     unlink $mapfifo;
6333
6334     if ($err) {
6335         rmtree $tmpdir;
6336         unlink $tmpfn;
6337
6338         foreach my $devname (keys %$devinfo) {
6339             my $volid = $devinfo->{$devname}->{volid};
6340             next if !$volid;
6341             eval {
6342                 if ($volid =~ m|^/|) {
6343                     unlink $volid || die 'unlink failed\n';
6344                 } else {
6345                     PVE::Storage::vdisk_free($cfg, $volid);
6346                 }
6347                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6348             };
6349             print STDERR "unable to cleanup '$volid' - $@" if $@;
6350         }
6351         die $err;
6352     }
6353
6354     rmtree $tmpdir;
6355
6356     rename($tmpfn, $conffile) ||
6357         die "unable to commit configuration file '$conffile'\n";
6358
6359     PVE::Cluster::cfs_update(); # make sure we read new file
6360
6361     eval { rescan($vmid, 1); };
6362     warn $@ if $@;
6363 }
6364
6365 sub restore_tar_archive {
6366     my ($archive, $vmid, $user, $opts) = @_;
6367
6368     if ($archive ne '-') {
6369         my $firstfile = tar_archive_read_firstfile($archive);
6370         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6371             if $firstfile ne 'qemu-server.conf';
6372     }
6373
6374     my $storecfg = PVE::Storage::config();
6375
6376     # destroy existing data - keep empty config
6377     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6378     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6379
6380     my $tocmd = "/usr/lib/qemu-server/qmextract";
6381
6382     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6383     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6384     $tocmd .= ' --prealloc' if $opts->{prealloc};
6385     $tocmd .= ' --info' if $opts->{info};
6386
6387     # tar option "xf" does not autodetect compression when read from STDIN,
6388     # so we pipe to zcat
6389     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6390         PVE::Tools::shellquote("--to-command=$tocmd");
6391
6392     my $tmpdir = "/var/tmp/vzdumptmp$$";
6393     mkpath $tmpdir;
6394
6395     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6396     local $ENV{VZDUMP_VMID} = $vmid;
6397     local $ENV{VZDUMP_USER} = $user;
6398
6399     my $conffile = PVE::QemuConfig->config_file($vmid);
6400     my $tmpfn = "$conffile.$$.tmp";
6401
6402     # disable interrupts (always do cleanups)
6403     local $SIG{INT} =
6404         local $SIG{TERM} =
6405         local $SIG{QUIT} =
6406         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6407
6408     eval {
6409         # enable interrupts
6410         local $SIG{INT} =
6411             local $SIG{TERM} =
6412             local $SIG{QUIT} =
6413             local $SIG{HUP} =
6414             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6415
6416         if ($archive eq '-') {
6417             print "extracting archive from STDIN\n";
6418             run_command($cmd, input => "<&STDIN");
6419         } else {
6420             print "extracting archive '$archive'\n";
6421             run_command($cmd);
6422         }
6423
6424         return if $opts->{info};
6425
6426         # read new mapping
6427         my $map = {};
6428         my $statfile = "$tmpdir/qmrestore.stat";
6429         if (my $fd = IO::File->new($statfile, "r")) {
6430             while (defined (my $line = <$fd>)) {
6431                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6432                     $map->{$1} = $2 if $1;
6433                 } else {
6434                     print STDERR "unable to parse line in statfile - $line\n";
6435                 }
6436             }
6437             $fd->close();
6438         }
6439
6440         my $confsrc = "$tmpdir/qemu-server.conf";
6441
6442         my $srcfd = new IO::File($confsrc, "r") ||
6443             die "unable to open file '$confsrc'\n";
6444
6445         my $outfd = new IO::File ($tmpfn, "w") ||
6446             die "unable to write config for VM $vmid\n";
6447
6448         my $cookie = { netcount => 0 };
6449         while (defined (my $line = <$srcfd>)) {
6450             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6451         }
6452
6453         $srcfd->close();
6454         $outfd->close();
6455     };
6456     my $err = $@;
6457
6458     if ($err) {
6459
6460         unlink $tmpfn;
6461
6462         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6463
6464         die $err;
6465     }
6466
6467     rmtree $tmpdir;
6468
6469     rename $tmpfn, $conffile ||
6470         die "unable to commit configuration file '$conffile'\n";
6471
6472     PVE::Cluster::cfs_update(); # make sure we read new file
6473
6474     eval { rescan($vmid, 1); };
6475     warn $@ if $@;
6476 };
6477
6478 sub foreach_storage_used_by_vm {
6479     my ($conf, $func) = @_;
6480
6481     my $sidhash = {};
6482
6483     foreach_drive($conf, sub {
6484         my ($ds, $drive) = @_;
6485         return if drive_is_cdrom($drive);
6486
6487         my $volid = $drive->{file};
6488
6489         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6490         $sidhash->{$sid} = $sid if $sid;
6491     });
6492
6493     foreach my $sid (sort keys %$sidhash) {
6494         &$func($sid);
6495     }
6496 }
6497
6498 sub do_snapshots_with_qemu {
6499     my ($storecfg, $volid) = @_;
6500
6501     my $storage_name = PVE::Storage::parse_volume_id($volid);
6502
6503     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6504         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6505         return 1;
6506     }
6507
6508     if ($volid =~ m/\.(qcow2|qed)$/){
6509         return 1;
6510     }
6511
6512     return undef;
6513 }
6514
6515 sub qga_check_running {
6516     my ($vmid, $nowarn) = @_;
6517
6518     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6519     if ($@) {
6520         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6521         return 0;
6522     }
6523     return 1;
6524 }
6525
6526 sub template_create {
6527     my ($vmid, $conf, $disk) = @_;
6528
6529     my $storecfg = PVE::Storage::config();
6530
6531     foreach_drive($conf, sub {
6532         my ($ds, $drive) = @_;
6533
6534         return if drive_is_cdrom($drive);
6535         return if $disk && $ds ne $disk;
6536
6537         my $volid = $drive->{file};
6538         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6539
6540         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6541         $drive->{file} = $voliddst;
6542         $conf->{$ds} = print_drive($vmid, $drive);
6543         PVE::QemuConfig->write_config($vmid, $conf);
6544     });
6545 }
6546
6547 sub convert_iscsi_path {
6548     my ($path) = @_;
6549
6550     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6551         my $portal = $1;
6552         my $target = $2;
6553         my $lun = $3;
6554
6555         my $initiator_name = get_initiator_name();
6556
6557         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6558                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6559     }
6560
6561     die "cannot convert iscsi path '$path', unkown format\n";
6562 }
6563
6564 sub qemu_img_convert {
6565     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6566
6567     my $storecfg = PVE::Storage::config();
6568     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6569     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6570
6571     if ($src_storeid && $dst_storeid) {
6572
6573         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6574
6575         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6576         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6577
6578         my $src_format = qemu_img_format($src_scfg, $src_volname);
6579         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6580
6581         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6582         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6583
6584         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6585         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6586
6587         my $cmd = [];
6588         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6589         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6590         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6591         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6592
6593         if ($src_is_iscsi) {
6594             push @$cmd, '--image-opts';
6595             $src_path = convert_iscsi_path($src_path);
6596         } else {
6597             push @$cmd, '-f', $src_format;
6598         }
6599
6600         if ($dst_is_iscsi) {
6601             push @$cmd, '--target-image-opts';
6602             $dst_path = convert_iscsi_path($dst_path);
6603         } else {
6604             push @$cmd, '-O', $dst_format;
6605         }
6606
6607         push @$cmd, $src_path;
6608
6609         if (!$dst_is_iscsi && $is_zero_initialized) {
6610             push @$cmd, "zeroinit:$dst_path";
6611         } else {
6612             push @$cmd, $dst_path;
6613         }
6614
6615         my $parser = sub {
6616             my $line = shift;
6617             if($line =~ m/\((\S+)\/100\%\)/){
6618                 my $percent = $1;
6619                 my $transferred = int($size * $percent / 100);
6620                 my $remaining = $size - $transferred;
6621
6622                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6623             }
6624
6625         };
6626
6627         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6628         my $err = $@;
6629         die "copy failed: $err" if $err;
6630     }
6631 }
6632
6633 sub qemu_img_format {
6634     my ($scfg, $volname) = @_;
6635
6636     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6637         return $1;
6638     } else {
6639         return "raw";
6640     }
6641 }
6642
6643 sub qemu_drive_mirror {
6644     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6645
6646     $jobs = {} if !$jobs;
6647
6648     my $qemu_target;
6649     my $format;
6650     $jobs->{"drive-$drive"} = {};
6651
6652     if ($dst_volid =~ /^nbd:/) {
6653         $qemu_target = $dst_volid;
6654         $format = "nbd";
6655     } else {
6656         my $storecfg = PVE::Storage::config();
6657         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6658
6659         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6660
6661         $format = qemu_img_format($dst_scfg, $dst_volname);
6662
6663         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6664
6665         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6666     }
6667
6668     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6669     $opts->{format} = $format if $format;
6670
6671     print "drive mirror is starting for drive-$drive\n";
6672
6673     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6674
6675     if (my $err = $@) {
6676         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6677         die "mirroring error: $err";
6678     }
6679
6680     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6681 }
6682
6683 sub qemu_drive_mirror_monitor {
6684     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6685
6686     eval {
6687         my $err_complete = 0;
6688
6689         while (1) {
6690             die "storage migration timed out\n" if $err_complete > 300;
6691
6692             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6693
6694             my $running_mirror_jobs = {};
6695             foreach my $stat (@$stats) {
6696                 next if $stat->{type} ne 'mirror';
6697                 $running_mirror_jobs->{$stat->{device}} = $stat;
6698             }
6699
6700             my $readycounter = 0;
6701
6702             foreach my $job (keys %$jobs) {
6703
6704                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6705                     print "$job : finished\n";
6706                     delete $jobs->{$job};
6707                     next;
6708                 }
6709
6710                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6711
6712                 my $busy = $running_mirror_jobs->{$job}->{busy};
6713                 my $ready = $running_mirror_jobs->{$job}->{ready};
6714                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6715                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6716                     my $remaining = $total - $transferred;
6717                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6718
6719                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6720                 }
6721
6722                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6723             }
6724
6725             last if scalar(keys %$jobs) == 0;
6726
6727             if ($readycounter == scalar(keys %$jobs)) {
6728                 print "all mirroring jobs are ready \n";
6729                 last if $skipcomplete; #do the complete later
6730
6731                 if ($vmiddst && $vmiddst != $vmid) {
6732                     my $agent_running = $qga && qga_check_running($vmid);
6733                     if ($agent_running) {
6734                         print "freeze filesystem\n";
6735                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6736                     } else {
6737                         print "suspend vm\n";
6738                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6739                     }
6740
6741                     # if we clone a disk for a new target vm, we don't switch the disk
6742                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6743
6744                     if ($agent_running) {
6745                         print "unfreeze filesystem\n";
6746                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6747                     } else {
6748                         print "resume vm\n";
6749                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6750                     }
6751
6752                     last;
6753                 } else {
6754
6755                     foreach my $job (keys %$jobs) {
6756                         # try to switch the disk if source and destination are on the same guest
6757                         print "$job: Completing block job...\n";
6758
6759                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6760                         if ($@ =~ m/cannot be completed/) {
6761                             print "$job: Block job cannot be completed, try again.\n";
6762                             $err_complete++;
6763                         }else {
6764                             print "$job: Completed successfully.\n";
6765                             $jobs->{$job}->{complete} = 1;
6766                         }
6767                     }
6768                 }
6769             }
6770             sleep 1;
6771         }
6772     };
6773     my $err = $@;
6774
6775     if ($err) {
6776         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6777         die "mirroring error: $err";
6778     }
6779
6780 }
6781
6782 sub qemu_blockjobs_cancel {
6783     my ($vmid, $jobs) = @_;
6784
6785     foreach my $job (keys %$jobs) {
6786         print "$job: Cancelling block job\n";
6787         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6788         $jobs->{$job}->{cancel} = 1;
6789     }
6790
6791     while (1) {
6792         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6793
6794         my $running_jobs = {};
6795         foreach my $stat (@$stats) {
6796             $running_jobs->{$stat->{device}} = $stat;
6797         }
6798
6799         foreach my $job (keys %$jobs) {
6800
6801             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6802                 print "$job: Done.\n";
6803                 delete $jobs->{$job};
6804             }
6805         }
6806
6807         last if scalar(keys %$jobs) == 0;
6808
6809         sleep 1;
6810     }
6811 }
6812
6813 sub clone_disk {
6814     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6815         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6816
6817     my $newvolid;
6818
6819     if (!$full) {
6820         print "create linked clone of drive $drivename ($drive->{file})\n";
6821         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6822         push @$newvollist, $newvolid;
6823     } else {
6824
6825         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6826         $storeid = $storage if $storage;
6827
6828         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6829         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6830
6831         print "create full clone of drive $drivename ($drive->{file})\n";
6832         my $name = undef;
6833         if (drive_is_cloudinit($drive)) {
6834             $name = "vm-$newvmid-cloudinit";
6835             $snapname = undef;
6836             # cloudinit only supports raw and qcow2 atm:
6837             if ($dst_format eq 'qcow2') {
6838                 $name .= '.qcow2';
6839             } elsif ($dst_format ne 'raw') {
6840                 die "clone: unhandled format for cloudinit image\n";
6841             }
6842         }
6843         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6844         push @$newvollist, $newvolid;
6845
6846         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6847
6848         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6849         if (!$running || $snapname) {
6850             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6851         } else {
6852
6853             my $kvmver = get_running_qemu_version ($vmid);
6854             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6855                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6856                     if $drive->{iothread};
6857             }
6858
6859             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6860         }
6861     }
6862
6863     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6864
6865     my $disk = $drive;
6866     $disk->{format} = undef;
6867     $disk->{file} = $newvolid;
6868     $disk->{size} = $size;
6869
6870     return $disk;
6871 }
6872
6873 # this only works if VM is running
6874 sub get_current_qemu_machine {
6875     my ($vmid) = @_;
6876
6877     my $cmd = { execute => 'query-machines', arguments => {} };
6878     my $res = vm_qmp_command($vmid, $cmd);
6879
6880     my ($current, $default);
6881     foreach my $e (@$res) {
6882         $default = $e->{name} if $e->{'is-default'};
6883         $current = $e->{name} if $e->{'is-current'};
6884     }
6885
6886     # fallback to the default machine if current is not supported by qemu
6887     return $current || $default || 'pc';
6888 }
6889
6890 sub get_running_qemu_version {
6891     my ($vmid) = @_;
6892     my $cmd = { execute => 'query-version', arguments => {} };
6893     my $res = vm_qmp_command($vmid, $cmd);
6894     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6895 }
6896
6897 sub qemu_machine_feature_enabled {
6898     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6899
6900     my $current_major;
6901     my $current_minor;
6902
6903     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6904
6905         $current_major = $3;
6906         $current_minor = $4;
6907
6908     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6909
6910         $current_major = $1;
6911         $current_minor = $2;
6912     }
6913
6914     return 1 if $current_major > $version_major ||
6915                 ($current_major == $version_major &&
6916                  $current_minor >= $version_minor);
6917 }
6918
6919 sub qemu_machine_pxe {
6920     my ($vmid, $conf, $machine) = @_;
6921
6922     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6923
6924     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6925         $machine .= '.pxe';
6926     }
6927
6928     return $machine;
6929 }
6930
6931 sub qemu_use_old_bios_files {
6932     my ($machine_type) = @_;
6933
6934     return if !$machine_type;
6935
6936     my $use_old_bios_files = undef;
6937
6938     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6939         $machine_type = $1;
6940         $use_old_bios_files = 1;
6941     } else {
6942         my $kvmver = kvm_user_version();
6943         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6944         # load new efi bios files on migration. So this hack is required to allow
6945         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6946         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6947         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6948     }
6949
6950     return ($use_old_bios_files, $machine_type);
6951 }
6952
6953 sub create_efidisk($$$$$) {
6954     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6955
6956     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6957     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6958
6959     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6960     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6961     PVE::Storage::activate_volumes($storecfg, [$volid]);
6962
6963     my $path = PVE::Storage::path($storecfg, $volid);
6964     eval {
6965         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6966     };
6967     die "Copying EFI vars image failed: $@" if $@;
6968
6969     return ($volid, $vars_size);
6970 }
6971
6972 sub vm_iothreads_list {
6973     my ($vmid) = @_;
6974
6975     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6976
6977     my $iothreads = {};
6978     foreach my $iothread (@$res) {
6979         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6980     }
6981
6982     return $iothreads;
6983 }
6984
6985 sub scsihw_infos {
6986     my ($conf, $drive) = @_;
6987
6988     my $maxdev = 0;
6989
6990     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6991         $maxdev = 7;
6992     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6993         $maxdev = 1;
6994     } else {
6995         $maxdev = 256;
6996     }
6997
6998     my $controller = int($drive->{index} / $maxdev);
6999     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7000
7001     return ($maxdev, $controller, $controller_prefix);
7002 }
7003
7004 sub add_hyperv_enlightenments {
7005     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7006
7007     return if $winversion < 6;
7008     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7009
7010     if ($gpu_passthrough || defined($hv_vendor_id)) {
7011         $hv_vendor_id //= 'proxmox';
7012         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7013     }
7014
7015     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7016         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7017         push @$cpuFlags , 'hv_vapic';
7018         push @$cpuFlags , 'hv_time';
7019     } else {
7020         push @$cpuFlags , 'hv_spinlocks=0xffff';
7021     }
7022
7023     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7024         push @$cpuFlags , 'hv_reset';
7025         push @$cpuFlags , 'hv_vpindex';
7026         push @$cpuFlags , 'hv_runtime';
7027     }
7028
7029     if ($winversion >= 7) {
7030         push @$cpuFlags , 'hv_relaxed';
7031
7032         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7033             push @$cpuFlags , 'hv_synic';
7034             push @$cpuFlags , 'hv_stimer';
7035         }
7036     }
7037 }
7038
7039 sub windows_version {
7040     my ($ostype) = @_;
7041
7042     return 0 if !$ostype;
7043
7044     my $winversion = 0;
7045
7046     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7047         $winversion = 5;
7048     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7049         $winversion = 6;
7050     } elsif ($ostype =~ m/^win(\d+)$/) {
7051         $winversion = $1;
7052     }
7053
7054     return $winversion;
7055 }
7056
7057 sub resolve_dst_disk_format {
7058         my ($storecfg, $storeid, $src_volname, $format) = @_;
7059         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7060
7061         if (!$format) {
7062             # if no target format is specified, use the source disk format as hint
7063             if ($src_volname) {
7064                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7065                 $format = qemu_img_format($scfg, $src_volname);
7066             } else {
7067                 return $defFormat;
7068             }
7069         }
7070
7071         # test if requested format is supported - else use default
7072         my $supported = grep { $_ eq $format } @$validFormats;
7073         $format = $defFormat if !$supported;
7074         return $format;
7075 }
7076
7077 sub resolve_first_disk {
7078     my $conf = shift;
7079     my @disks = PVE::QemuServer::valid_drive_names();
7080     my $firstdisk;
7081     foreach my $ds (reverse @disks) {
7082         next if !$conf->{$ds};
7083         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7084         next if PVE::QemuServer::drive_is_cdrom($disk);
7085         $firstdisk = $ds;
7086     }
7087     return $firstdisk;
7088 }
7089
7090 sub generate_uuid {
7091     my ($uuid, $uuid_str);
7092     UUID::generate($uuid);
7093     UUID::unparse($uuid, $uuid_str);
7094     return $uuid_str;
7095 }
7096
7097 sub generate_smbios1_uuid {
7098     return "uuid=".generate_uuid();
7099 }
7100
7101 sub nbd_stop {
7102     my ($vmid) = @_;
7103
7104     vm_mon_cmd($vmid, 'nbd-server-stop');
7105 }
7106
7107 # bash completion helper
7108
7109 sub complete_backup_archives {
7110     my ($cmdname, $pname, $cvalue) = @_;
7111
7112     my $cfg = PVE::Storage::config();
7113
7114     my $storeid;
7115
7116     if ($cvalue =~ m/^([^:]+):/) {
7117         $storeid = $1;
7118     }
7119
7120     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7121
7122     my $res = [];
7123     foreach my $id (keys %$data) {
7124         foreach my $item (@{$data->{$id}}) {
7125             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7126             push @$res, $item->{volid} if defined($item->{volid});
7127         }
7128     }
7129
7130     return $res;
7131 }
7132
7133 my $complete_vmid_full = sub {
7134     my ($running) = @_;
7135
7136     my $idlist = vmstatus();
7137
7138     my $res = [];
7139
7140     foreach my $id (keys %$idlist) {
7141         my $d = $idlist->{$id};
7142         if (defined($running)) {
7143             next if $d->{template};
7144             next if $running && $d->{status} ne 'running';
7145             next if !$running && $d->{status} eq 'running';
7146         }
7147         push @$res, $id;
7148
7149     }
7150     return $res;
7151 };
7152
7153 sub complete_vmid {
7154     return &$complete_vmid_full();
7155 }
7156
7157 sub complete_vmid_stopped {
7158     return &$complete_vmid_full(0);
7159 }
7160
7161 sub complete_vmid_running {
7162     return &$complete_vmid_full(1);
7163 }
7164
7165 sub complete_storage {
7166
7167     my $cfg = PVE::Storage::config();
7168     my $ids = $cfg->{ids};
7169
7170     my $res = [];
7171     foreach my $sid (keys %$ids) {
7172         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7173         next if !$ids->{$sid}->{content}->{images};
7174         push @$res, $sid;
7175     }
7176
7177     return $res;
7178 }
7179
7180 1;