PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_IDE_DISKS = 4;
 735 my $MAX_SCSI_DISKS = 14;
 736 my $MAX_VIRTIO_DISKS = 16;
 737 my $MAX_SATA_DISKS = 6;
 738 my $MAX_USB_DEVICES = 5;
 739 my $MAX_NETS = 32;
 740 my $MAX_UNUSED_DISKS = 256;
 741 my $MAX_HOSTPCI_DEVICES = 4;
 742 my $MAX_SERIAL_PORTS = 4;
 743 my $MAX_PARALLEL_PORTS = 3;
 744 my $MAX_NUMA = 8;
 745
 746 my $numa_fmt = {
 747     cpus => {
 748         type => "string",
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => "CPUs accessing this NUMA node.",
 751         format_description => "id[-id];...",
 752     },
 753     memory => {
 754         type => "number",
 755         description => "Amount of memory this NUMA node provides.",
 756         optional => 1,
 757     },
 758     hostnodes => {
 759         type => "string",
 760         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 761         description => "Host NUMA nodes to use.",
 762         format_description => "id[-id];...",
 763         optional => 1,
 764     },
 765     policy => {
 766         type => 'string',
 767         enum => [qw(preferred bind interleave)],
 768         description => "NUMA allocation policy.",
 769         optional => 1,
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 773 my $numadesc = {
 774     optional => 1,
 775     type => 'string', format => $numa_fmt,
 776     description => "NUMA topology.",
 777 };
 778 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 779
 780 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 781     $confdesc->{"numa$i"} = $numadesc;
 782 }
 783
 784 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 785                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 786                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 787 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 788
 789 my $net_fmt_bridge_descr = <<__EOD__;
 790 Bridge to attach the network device to. The Proxmox VE standard bridge
 791 is called 'vmbr0'.
 792
 793 If you do not specify a bridge, we create a kvm user (NATed) network
 794 device, which provides DHCP and DNS services. The following addresses
 795 are used:
 796
 797  10.0.2.2   Gateway
 798  10.0.2.3   DNS Server
 799  10.0.2.4   SMB Server
 800
 801 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 802 __EOD__
 803
 804 my $net_fmt = {
 805     macaddr => {
 806         type => 'string',
 807         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 808         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 809         format_description => "XX:XX:XX:XX:XX:XX",
 810         optional => 1,
 811     },
 812     model => {
 813         type => 'string',
 814         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 815         enum => $nic_model_list,
 816         default_key => 1,
 817     },
 818     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 819     bridge => {
 820         type => 'string',
 821         description => $net_fmt_bridge_descr,
 822         format_description => 'bridge',
 823         optional => 1,
 824     },
 825     queues => {
 826         type => 'integer',
 827         minimum => 0, maximum => 16,
 828         description => 'Number of packet queues to be used on the device.',
 829         optional => 1,
 830     },
 831     rate => {
 832         type => 'number',
 833         minimum => 0,
 834         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 835         optional => 1,
 836     },
 837     tag => {
 838         type => 'integer',
 839         minimum => 1, maximum => 4094,
 840         description => 'VLAN tag to apply to packets on this interface.',
 841         optional => 1,
 842     },
 843     trunks => {
 844         type => 'string',
 845         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 846         description => 'VLAN trunks to pass through this interface.',
 847         format_description => 'vlanid[;vlanid...]',
 848         optional => 1,
 849     },
 850     firewall => {
 851         type => 'boolean',
 852         description => 'Whether this interface should be protected by the firewall.',
 853         optional => 1,
 854     },
 855     link_down => {
 856         type => 'boolean',
 857         description => 'Whether this interface should be disconnected (like pulling the plug).',
 858         optional => 1,
 859     },
 860 };
 861
 862 my $netdesc = {
 863     optional => 1,
 864     type => 'string', format => $net_fmt,
 865     description => "Specify network devices.",
 866 };
 867
 868 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 869
 870 my $ipconfig_fmt = {
 871     ip => {
 872         type => 'string',
 873         format => 'pve-ipv4-config',
 874         format_description => 'IPv4Format/CIDR',
 875         description => 'IPv4 address in CIDR format.',
 876         optional => 1,
 877         default => 'dhcp',
 878     },
 879     gw => {
 880         type => 'string',
 881         format => 'ipv4',
 882         format_description => 'GatewayIPv4',
 883         description => 'Default gateway for IPv4 traffic.',
 884         optional => 1,
 885         requires => 'ip',
 886     },
 887     ip6 => {
 888         type => 'string',
 889         format => 'pve-ipv6-config',
 890         format_description => 'IPv6Format/CIDR',
 891         description => 'IPv6 address in CIDR format.',
 892         optional => 1,
 893         default => 'dhcp',
 894     },
 895     gw6 => {
 896         type => 'string',
 897         format => 'ipv6',
 898         format_description => 'GatewayIPv6',
 899         description => 'Default gateway for IPv6 traffic.',
 900         optional => 1,
 901         requires => 'ip6',
 902     },
 903 };
 904 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 905 my $ipconfigdesc = {
 906     optional => 1,
 907     type => 'string', format => 'pve-qm-ipconfig',
 908     description => <<'EODESCR',
 909 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 910
 911 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 912
 913 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 914 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 915
 916 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 917 EODESCR
 918 };
 919 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 920
 921 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 922     $confdesc->{"net$i"} = $netdesc;
 923     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 924 }
 925
 926 foreach my $key (keys %$confdesc_cloudinit) {
 927     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 928 }
 929
 930 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 931 sub verify_volume_id_or_qm_path {
 932     my ($volid, $noerr) = @_;
 933
 934     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 935         return $volid;
 936     }
 937
 938     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 939     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 940     if ($@) {
 941         return undef if $noerr;
 942         die $@;
 943     }
 944     return $volid;
 945 }
 946
 947 my $drivename_hash;
 948
 949 my %drivedesc_base = (
 950     volume => { alias => 'file' },
 951     file => {
 952         type => 'string',
 953         format => 'pve-volume-id-or-qm-path',
 954         default_key => 1,
 955         format_description => 'volume',
 956         description => "The drive's backing volume.",
 957     },
 958     media => {
 959         type => 'string',
 960         enum => [qw(cdrom disk)],
 961         description => "The drive's media type.",
 962         default => 'disk',
 963         optional => 1
 964     },
 965     cyls => {
 966         type => 'integer',
 967         description => "Force the drive's physical geometry to have a specific cylinder count.",
 968         optional => 1
 969     },
 970     heads => {
 971         type => 'integer',
 972         description => "Force the drive's physical geometry to have a specific head count.",
 973         optional => 1
 974     },
 975     secs => {
 976         type => 'integer',
 977         description => "Force the drive's physical geometry to have a specific sector count.",
 978         optional => 1
 979     },
 980     trans => {
 981         type => 'string',
 982         enum => [qw(none lba auto)],
 983         description => "Force disk geometry bios translation mode.",
 984         optional => 1,
 985     },
 986     snapshot => {
 987         type => 'boolean',
 988         description => "Controls qemu's snapshot mode feature."
 989             . " If activated, changes made to the disk are temporary and will"
 990             . " be discarded when the VM is shutdown.",
 991         optional => 1,
 992     },
 993     cache => {
 994         type => 'string',
 995         enum => [qw(none writethrough writeback unsafe directsync)],
 996         description => "The drive's cache mode",
 997         optional => 1,
 998     },
 999     format => get_standard_option('pve-qm-image-format'),
1000     size => {
1001         type => 'string',
1002         format => 'disk-size',
1003         format_description => 'DiskSize',
1004         description => "Disk size. This is purely informational and has no effect.",
1005         optional => 1,
1006     },
1007     backup => {
1008         type => 'boolean',
1009         description => "Whether the drive should be included when making backups.",
1010         optional => 1,
1011     },
1012     replicate => {
1013         type => 'boolean',
1014         description => 'Whether the drive should considered for replication jobs.',
1015         optional => 1,
1016         default => 1,
1017     },
1018     rerror => {
1019         type => 'string',
1020         enum => [qw(ignore report stop)],
1021         description => 'Read error action.',
1022         optional => 1,
1023     },
1024     werror => {
1025         type => 'string',
1026         enum => [qw(enospc ignore report stop)],
1027         description => 'Write error action.',
1028         optional => 1,
1029     },
1030     aio => {
1031         type => 'string',
1032         enum => [qw(native threads)],
1033         description => 'AIO type to use.',
1034         optional => 1,
1035     },
1036     discard => {
1037         type => 'string',
1038         enum => [qw(ignore on)],
1039         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1040         optional => 1,
1041     },
1042     detect_zeroes => {
1043         type => 'boolean',
1044         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1045         optional => 1,
1046     },
1047     serial => {
1048         type => 'string',
1049         format => 'urlencoded',
1050         format_description => 'serial',
1051         maxLength => 20*3, # *3 since it's %xx url enoded
1052         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1053         optional => 1,
1054     },
1055     shared => {
1056         type => 'boolean',
1057         description => 'Mark this locally-managed volume as available on all nodes',
1058         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1059         optional => 1,
1060         default => 0,
1061     }
1062 );
1063
1064 my %iothread_fmt = ( iothread => {
1065         type => 'boolean',
1066         description => "Whether to use iothreads for this drive",
1067         optional => 1,
1068 });
1069
1070 my %model_fmt = (
1071     model => {
1072         type => 'string',
1073         format => 'urlencoded',
1074         format_description => 'model',
1075         maxLength => 40*3, # *3 since it's %xx url enoded
1076         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1077         optional => 1,
1078     },
1079 );
1080
1081 my %queues_fmt = (
1082     queues => {
1083         type => 'integer',
1084         description => "Number of queues.",
1085         minimum => 2,
1086         optional => 1
1087     }
1088 );
1089
1090 my %scsiblock_fmt = (
1091     scsiblock => {
1092         type => 'boolean',
1093         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1094         optional => 1,
1095         default => 0,
1096     },
1097 );
1098
1099 my %ssd_fmt = (
1100     ssd => {
1101         type => 'boolean',
1102         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1103         optional => 1,
1104     },
1105 );
1106
1107 my %wwn_fmt = (
1108     wwn => {
1109         type => 'string',
1110         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1111         format_description => 'wwn',
1112         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1113         optional => 1,
1114     },
1115 );
1116
1117 my $add_throttle_desc = sub {
1118     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1119     my $d = {
1120         type => $type,
1121         format_description => $unit,
1122         description => "Maximum $what in $longunit.",
1123         optional => 1,
1124     };
1125     $d->{minimum} = $minimum if defined($minimum);
1126     $drivedesc_base{$key} = $d;
1127 };
1128 # throughput: (leaky bucket)
1129 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1130 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1131 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1132 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1138
1139 # pools: (pool of IO before throttling starts taking effect)
1140 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1141 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1142 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1143 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1144 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1145 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1146
1147 # burst lengths
1148 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1149 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1150 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1151 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1152 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1153 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1154
1155 # legacy support
1156 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1157 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1158 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1159 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1160
1161 my $ide_fmt = {
1162     %drivedesc_base,
1163     %model_fmt,
1164     %ssd_fmt,
1165     %wwn_fmt,
1166 };
1167 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1168
1169 my $idedesc = {
1170     optional => 1,
1171     type => 'string', format => $ide_fmt,
1172     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1173 };
1174 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1175
1176 my $scsi_fmt = {
1177     %drivedesc_base,
1178     %iothread_fmt,
1179     %queues_fmt,
1180     %scsiblock_fmt,
1181     %ssd_fmt,
1182     %wwn_fmt,
1183 };
1184 my $scsidesc = {
1185     optional => 1,
1186     type => 'string', format => $scsi_fmt,
1187     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1188 };
1189 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1190
1191 my $sata_fmt = {
1192     %drivedesc_base,
1193     %ssd_fmt,
1194     %wwn_fmt,
1195 };
1196 my $satadesc = {
1197     optional => 1,
1198     type => 'string', format => $sata_fmt,
1199     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1200 };
1201 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1202
1203 my $virtio_fmt = {
1204     %drivedesc_base,
1205     %iothread_fmt,
1206 };
1207 my $virtiodesc = {
1208     optional => 1,
1209     type => 'string', format => $virtio_fmt,
1210     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1211 };
1212 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1213
1214 my $alldrive_fmt = {
1215     %drivedesc_base,
1216     %iothread_fmt,
1217     %model_fmt,
1218     %queues_fmt,
1219     %scsiblock_fmt,
1220     %ssd_fmt,
1221     %wwn_fmt,
1222 };
1223
1224 my $efidisk_fmt = {
1225     volume => { alias => 'file' },
1226     file => {
1227         type => 'string',
1228         format => 'pve-volume-id-or-qm-path',
1229         default_key => 1,
1230         format_description => 'volume',
1231         description => "The drive's backing volume.",
1232     },
1233     format => get_standard_option('pve-qm-image-format'),
1234     size => {
1235         type => 'string',
1236         format => 'disk-size',
1237         format_description => 'DiskSize',
1238         description => "Disk size. This is purely informational and has no effect.",
1239         optional => 1,
1240     },
1241 };
1242
1243 my $efidisk_desc = {
1244     optional => 1,
1245     type => 'string', format => $efidisk_fmt,
1246     description => "Configure a Disk for storing EFI vars",
1247 };
1248
1249 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1250
1251 my $usb_fmt = {
1252     host => {
1253         default_key => 1,
1254         type => 'string', format => 'pve-qm-usb-device',
1255         format_description => 'HOSTUSBDEVICE|spice',
1256         description => <<EODESCR,
1257 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1258
1259  'bus-port(.port)*' (decimal numbers) or
1260  'vendor_id:product_id' (hexadeciaml numbers) or
1261  'spice'
1262
1263 You can use the 'lsusb -t' command to list existing usb devices.
1264
1265 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1266
1267 The value 'spice' can be used to add a usb redirection devices for spice.
1268 EODESCR
1269     },
1270     usb3 => {
1271         optional => 1,
1272         type => 'boolean',
1273         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1274         default => 0,
1275     },
1276 };
1277
1278 my $usbdesc = {
1279     optional => 1,
1280     type => 'string', format => $usb_fmt,
1281     description => "Configure an USB device (n is 0 to 4).",
1282 };
1283 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1284
1285 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1286 my $hostpci_fmt = {
1287     host => {
1288         default_key => 1,
1289         type => 'string',
1290         pattern => qr/$PCIRE(;$PCIRE)*/,
1291         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1292         description => <<EODESCR,
1293 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1294 of PCI virtual functions of the host. HOSTPCIID syntax is:
1295
1296 'bus:dev.func' (hexadecimal numbers)
1297
1298 You can us the 'lspci' command to list existing PCI devices.
1299 EODESCR
1300     },
1301     rombar => {
1302         type => 'boolean',
1303         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1304         optional => 1,
1305         default => 1,
1306     },
1307     romfile => {
1308         type => 'string',
1309         pattern => '[^,;]+',
1310         format_description => 'string',
1311         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1312         optional => 1,
1313     },
1314     pcie => {
1315         type => 'boolean',
1316         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1317         optional => 1,
1318         default => 0,
1319     },
1320     'x-vga' => {
1321         type => 'boolean',
1322         description =>  "Enable vfio-vga device support.",
1323         optional => 1,
1324         default => 0,
1325     },
1326     'mdev' => {
1327         type => 'string',
1328         format_description => 'string',
1329         pattern => '[^/\.:]+',
1330         optional => 1,
1331         description => <<EODESCR
1332 The type of mediated device to use.
1333 An instance of this type will be created on startup of the VM and
1334 will be cleaned up when the VM stops.
1335 EODESCR
1336     }
1337 };
1338 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1339
1340 my $hostpcidesc = {
1341         optional => 1,
1342         type => 'string', format => 'pve-qm-hostpci',
1343         description => "Map host PCI devices into guest.",
1344         verbose_description =>  <<EODESCR,
1345 Map host PCI devices into guest.
1346
1347 NOTE: This option allows direct access to host hardware. So it is no longer
1348 possible to migrate such machines - use with special care.
1349
1350 CAUTION: Experimental! User reported problems with this option.
1351 EODESCR
1352 };
1353 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1354
1355 my $serialdesc = {
1356         optional => 1,
1357         type => 'string',
1358         pattern => '(/dev/.+|socket)',
1359         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1360         verbose_description =>  <<EODESCR,
1361 Create a serial device inside the VM (n is 0 to 3), and pass through a
1362 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1363 host side (use 'qm terminal' to open a terminal connection).
1364
1365 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1366
1367 CAUTION: Experimental! User reported problems with this option.
1368 EODESCR
1369 };
1370
1371 my $paralleldesc= {
1372         optional => 1,
1373         type => 'string',
1374         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1375         description =>  "Map host parallel devices (n is 0 to 2).",
1376         verbose_description =>  <<EODESCR,
1377 Map host parallel devices (n is 0 to 2).
1378
1379 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1380
1381 CAUTION: Experimental! User reported problems with this option.
1382 EODESCR
1383 };
1384
1385 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1386     $confdesc->{"parallel$i"} = $paralleldesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1390     $confdesc->{"serial$i"} = $serialdesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1394     $confdesc->{"hostpci$i"} = $hostpcidesc;
1395 }
1396
1397 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1398     $drivename_hash->{"ide$i"} = 1;
1399     $confdesc->{"ide$i"} = $idedesc;
1400 }
1401
1402 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1403     $drivename_hash->{"sata$i"} = 1;
1404     $confdesc->{"sata$i"} = $satadesc;
1405 }
1406
1407 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1408     $drivename_hash->{"scsi$i"} = 1;
1409     $confdesc->{"scsi$i"} = $scsidesc ;
1410 }
1411
1412 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1413     $drivename_hash->{"virtio$i"} = 1;
1414     $confdesc->{"virtio$i"} = $virtiodesc;
1415 }
1416
1417 $drivename_hash->{efidisk0} = 1;
1418 $confdesc->{efidisk0} = $efidisk_desc;
1419
1420 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1421     $confdesc->{"usb$i"} = $usbdesc;
1422 }
1423
1424 my $unuseddesc = {
1425     optional => 1,
1426     type => 'string', format => 'pve-volume-id',
1427     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1428 };
1429
1430 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1431     $confdesc->{"unused$i"} = $unuseddesc;
1432 }
1433
1434 my $kvm_api_version = 0;
1435
1436 sub kvm_version {
1437     return $kvm_api_version if $kvm_api_version;
1438
1439     open my $fh, '<', '/dev/kvm'
1440         or return undef;
1441
1442     # 0xae00 => KVM_GET_API_VERSION
1443     $kvm_api_version = ioctl($fh, 0xae00, 0);
1444
1445     return $kvm_api_version;
1446 }
1447
1448 my $kvm_user_version;
1449
1450 sub kvm_user_version {
1451
1452     return $kvm_user_version if $kvm_user_version;
1453
1454     $kvm_user_version = 'unknown';
1455
1456     my $code = sub {
1457         my $line = shift;
1458         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1459             $kvm_user_version = $2;
1460         }
1461     };
1462
1463     eval { run_command("kvm -version", outfunc => $code); };
1464     warn $@ if $@;
1465
1466     return $kvm_user_version;
1467
1468 }
1469
1470 sub kernel_has_vhost_net {
1471     return -c '/dev/vhost-net';
1472 }
1473
1474 sub valid_drive_names {
1475     # order is important - used to autoselect boot disk
1476     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1477             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1478             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1479             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1480             'efidisk0');
1481 }
1482
1483 sub is_valid_drivename {
1484     my $dev = shift;
1485
1486     return defined($drivename_hash->{$dev});
1487 }
1488
1489 sub option_exists {
1490     my $key = shift;
1491     return defined($confdesc->{$key});
1492 }
1493
1494 sub nic_models {
1495     return $nic_model_list;
1496 }
1497
1498 sub os_list_description {
1499
1500     return {
1501         other => 'Other',
1502         wxp => 'Windows XP',
1503         w2k => 'Windows 2000',
1504         w2k3 =>, 'Windows 2003',
1505         w2k8 => 'Windows 2008',
1506         wvista => 'Windows Vista',
1507         win7 => 'Windows 7',
1508         win8 => 'Windows 8/2012',
1509         win10 => 'Windows 10/2016',
1510         l24 => 'Linux 2.4',
1511         l26 => 'Linux 2.6',
1512     };
1513 }
1514
1515 my $cdrom_path;
1516
1517 sub get_cdrom_path {
1518
1519     return  $cdrom_path if $cdrom_path;
1520
1521     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1522     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1523     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1524 }
1525
1526 sub get_iso_path {
1527     my ($storecfg, $vmid, $cdrom) = @_;
1528
1529     if ($cdrom eq 'cdrom') {
1530         return get_cdrom_path();
1531     } elsif ($cdrom eq 'none') {
1532         return '';
1533     } elsif ($cdrom =~ m|^/|) {
1534         return $cdrom;
1535     } else {
1536         return PVE::Storage::path($storecfg, $cdrom);
1537     }
1538 }
1539
1540 # try to convert old style file names to volume IDs
1541 sub filename_to_volume_id {
1542     my ($vmid, $file, $media) = @_;
1543
1544      if (!($file eq 'none' || $file eq 'cdrom' ||
1545           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1546
1547         return undef if $file =~ m|/|;
1548
1549         if ($media && $media eq 'cdrom') {
1550             $file = "local:iso/$file";
1551         } else {
1552             $file = "local:$vmid/$file";
1553         }
1554     }
1555
1556     return $file;
1557 }
1558
1559 sub verify_media_type {
1560     my ($opt, $vtype, $media) = @_;
1561
1562     return if !$media;
1563
1564     my $etype;
1565     if ($media eq 'disk') {
1566         $etype = 'images';
1567     } elsif ($media eq 'cdrom') {
1568         $etype = 'iso';
1569     } else {
1570         die "internal error";
1571     }
1572
1573     return if ($vtype eq $etype);
1574
1575     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1576 }
1577
1578 sub cleanup_drive_path {
1579     my ($opt, $storecfg, $drive) = @_;
1580
1581     # try to convert filesystem paths to volume IDs
1582
1583     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1584         ($drive->{file} !~ m|^/dev/.+|) &&
1585         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1586         ($drive->{file} !~ m/^\d+$/)) {
1587         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1588         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1589         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1590         verify_media_type($opt, $vtype, $drive->{media});
1591         $drive->{file} = $volid;
1592     }
1593
1594     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1595 }
1596
1597 sub parse_hotplug_features {
1598     my ($data) = @_;
1599
1600     my $res = {};
1601
1602     return $res if $data eq '0';
1603
1604     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1605
1606     foreach my $feature (PVE::Tools::split_list($data)) {
1607         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1608             $res->{$1} = 1;
1609         } else {
1610             die "invalid hotplug feature '$feature'\n";
1611         }
1612     }
1613     return $res;
1614 }
1615
1616 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1617 sub pve_verify_hotplug_features {
1618     my ($value, $noerr) = @_;
1619
1620     return $value if parse_hotplug_features($value);
1621
1622     return undef if $noerr;
1623
1624     die "unable to parse hotplug option\n";
1625 }
1626
1627 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1628 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1629 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1630 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1631 #        [,iothread=on][,serial=serial][,model=model]
1632
1633 sub parse_drive {
1634     my ($key, $data) = @_;
1635
1636     my ($interface, $index);
1637
1638     if ($key =~ m/^([^\d]+)(\d+)$/) {
1639         $interface = $1;
1640         $index = $2;
1641     } else {
1642         return undef;
1643     }
1644
1645     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1646                                      : $confdesc->{$key}->{format};
1647     if (!$desc) {
1648         warn "invalid drive key: $key\n";
1649         return undef;
1650     }
1651     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1652     return undef if !$res;
1653     $res->{interface} = $interface;
1654     $res->{index} = $index;
1655
1656     my $error = 0;
1657     foreach my $opt (qw(bps bps_rd bps_wr)) {
1658         if (my $bps = defined(delete $res->{$opt})) {
1659             if (defined($res->{"m$opt"})) {
1660                 warn "both $opt and m$opt specified\n";
1661                 ++$error;
1662                 next;
1663             }
1664             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1665         }
1666     }
1667
1668     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1669     for my $requirement (
1670         [mbps_max => 'mbps'],
1671         [mbps_rd_max => 'mbps_rd'],
1672         [mbps_wr_max => 'mbps_wr'],
1673         [miops_max => 'miops'],
1674         [miops_rd_max => 'miops_rd'],
1675         [miops_wr_max => 'miops_wr'],
1676         [bps_max_length => 'mbps_max'],
1677         [bps_rd_max_length => 'mbps_rd_max'],
1678         [bps_wr_max_length => 'mbps_wr_max'],
1679         [iops_max_length => 'iops_max'],
1680         [iops_rd_max_length => 'iops_rd_max'],
1681         [iops_wr_max_length => 'iops_wr_max']) {
1682         my ($option, $requires) = @$requirement;
1683         if ($res->{$option} && !$res->{$requires}) {
1684             warn "$option requires $requires\n";
1685             ++$error;
1686         }
1687     }
1688
1689     return undef if $error;
1690
1691     return undef if $res->{mbps_rd} && $res->{mbps};
1692     return undef if $res->{mbps_wr} && $res->{mbps};
1693     return undef if $res->{iops_rd} && $res->{iops};
1694     return undef if $res->{iops_wr} && $res->{iops};
1695
1696     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1697         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1698         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1699         return undef if $res->{interface} eq 'virtio';
1700     }
1701
1702     if (my $size = $res->{size}) {
1703         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1704     }
1705
1706     return $res;
1707 }
1708
1709 sub print_drive {
1710     my ($vmid, $drive) = @_;
1711     my $data = { %$drive };
1712     delete $data->{$_} for qw(index interface);
1713     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1714 }
1715
1716 sub scsi_inquiry {
1717     my($fh, $noerr) = @_;
1718
1719     my $SG_IO = 0x2285;
1720     my $SG_GET_VERSION_NUM = 0x2282;
1721
1722     my $versionbuf = "\x00" x 8;
1723     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1724     if (!$ret) {
1725         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1726         return undef;
1727     }
1728     my $version = unpack("I", $versionbuf);
1729     if ($version < 30000) {
1730         die "scsi generic interface too old\n"  if !$noerr;
1731         return undef;
1732     }
1733
1734     my $buf = "\x00" x 36;
1735     my $sensebuf = "\x00" x 8;
1736     my $cmd = pack("C x3 C x1", 0x12, 36);
1737
1738     # see /usr/include/scsi/sg.h
1739     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1740
1741     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1742                       length($sensebuf), 0, length($buf), $buf,
1743                       $cmd, $sensebuf, 6000);
1744
1745     $ret = ioctl($fh, $SG_IO, $packet);
1746     if (!$ret) {
1747         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1748         return undef;
1749     }
1750
1751     my @res = unpack($sg_io_hdr_t, $packet);
1752     if ($res[17] || $res[18]) {
1753         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1754         return undef;
1755     }
1756
1757     my $res = {};
1758     (my $byte0, my $byte1, $res->{vendor},
1759      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1760
1761     $res->{removable} = $byte1 & 128 ? 1 : 0;
1762     $res->{type} = $byte0 & 31;
1763
1764     return $res;
1765 }
1766
1767 sub path_is_scsi {
1768     my ($path) = @_;
1769
1770     my $fh = IO::File->new("+<$path") || return undef;
1771     my $res = scsi_inquiry($fh, 1);
1772     close($fh);
1773
1774     return $res;
1775 }
1776
1777 sub machine_type_is_q35 {
1778     my ($conf) = @_;
1779
1780     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1781 }
1782
1783 sub print_tabletdevice_full {
1784     my ($conf, $arch) = @_;
1785
1786     my $q35 = machine_type_is_q35($conf);
1787
1788     # we use uhci for old VMs because tablet driver was buggy in older qemu
1789     my $usbbus;
1790     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1791         $usbbus = 'ehci';
1792     } else {
1793         $usbbus = 'uhci';
1794     }
1795
1796     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1797 }
1798
1799 sub print_keyboarddevice_full {
1800     my ($conf, $arch, $machine) = @_;
1801
1802     return undef if $arch ne 'aarch64';
1803
1804     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1805 }
1806
1807 sub print_drivedevice_full {
1808     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1809
1810     my $device = '';
1811     my $maxdev = 0;
1812
1813     if ($drive->{interface} eq 'virtio') {
1814         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1815         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1816         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1817     } elsif ($drive->{interface} eq 'scsi') {
1818
1819         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1820         my $unit = $drive->{index} % $maxdev;
1821         my $devicetype = 'hd';
1822         my $path = '';
1823         if (drive_is_cdrom($drive)) {
1824             $devicetype = 'cd';
1825         } else {
1826             if ($drive->{file} =~ m|^/|) {
1827                 $path = $drive->{file};
1828                 if (my $info = path_is_scsi($path)) {
1829                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1830                         $devicetype = 'block';
1831                     } elsif ($info->{type} == 1) { # tape
1832                         $devicetype = 'generic';
1833                     }
1834                 }
1835             } else {
1836                  $path = PVE::Storage::path($storecfg, $drive->{file});
1837             }
1838
1839             if($path =~ m/^iscsi\:\/\//){
1840                 $devicetype = 'generic';
1841             }
1842         }
1843
1844         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1845             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1846         } else {
1847             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1848         }
1849
1850         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1851             $device .= ",rotation_rate=1";
1852         }
1853         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1854
1855     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1856         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1857         my $controller = int($drive->{index} / $maxdev);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1860
1861         $device = "ide-$devicetype";
1862         if ($drive->{interface} eq 'ide') {
1863             $device .= ",bus=ide.$controller,unit=$unit";
1864         } else {
1865             $device .= ",bus=ahci$controller.$unit";
1866         }
1867         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1868
1869         if ($devicetype eq 'hd') {
1870             if (my $model = $drive->{model}) {
1871                 $model = URI::Escape::uri_unescape($model);
1872                 $device .= ",model=$model";
1873             }
1874             if ($drive->{ssd}) {
1875                 $device .= ",rotation_rate=1";
1876             }
1877         }
1878         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1879     } elsif ($drive->{interface} eq 'usb') {
1880         die "implement me";
1881         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1882     } else {
1883         die "unsupported interface type";
1884     }
1885
1886     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1887
1888     if (my $serial = $drive->{serial}) {
1889         $serial = URI::Escape::uri_unescape($serial);
1890         $device .= ",serial=$serial";
1891     }
1892
1893
1894     return $device;
1895 }
1896
1897 sub get_initiator_name {
1898     my $initiator;
1899
1900     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1901     while (defined(my $line = <$fh>)) {
1902         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1903         $initiator = $1;
1904         last;
1905     }
1906     $fh->close();
1907
1908     return $initiator;
1909 }
1910
1911 sub print_drive_full {
1912     my ($storecfg, $vmid, $drive) = @_;
1913
1914     my $path;
1915     my $volid = $drive->{file};
1916     my $format;
1917
1918     if (drive_is_cdrom($drive)) {
1919         $path = get_iso_path($storecfg, $vmid, $volid);
1920     } else {
1921         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1922         if ($storeid) {
1923             $path = PVE::Storage::path($storecfg, $volid);
1924             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1925             $format = qemu_img_format($scfg, $volname);
1926         } else {
1927             $path = $volid;
1928             $format = "raw";
1929         }
1930    }
1931
1932     my $opts = '';
1933     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1934     foreach my $o (@qemu_drive_options) {
1935         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1936     }
1937
1938     # snapshot only accepts on|off
1939     if (defined($drive->{snapshot})) {
1940         my $v = $drive->{snapshot} ? 'on' : 'off';
1941         $opts .= ",snapshot=$v";
1942     }
1943
1944     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1945         my ($dir, $qmpname) = @$type;
1946         if (my $v = $drive->{"mbps$dir"}) {
1947             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1948         }
1949         if (my $v = $drive->{"mbps${dir}_max"}) {
1950             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1951         }
1952         if (my $v = $drive->{"bps${dir}_max_length"}) {
1953             $opts .= ",throttling.bps$qmpname-max-length=$v";
1954         }
1955         if (my $v = $drive->{"iops${dir}"}) {
1956             $opts .= ",throttling.iops$qmpname=$v";
1957         }
1958         if (my $v = $drive->{"iops${dir}_max"}) {
1959             $opts .= ",throttling.iops$qmpname-max=$v";
1960         }
1961         if (my $v = $drive->{"iops${dir}_max_length"}) {
1962             $opts .= ",throttling.iops$qmpname-max-length=$v";
1963         }
1964     }
1965
1966     $opts .= ",format=$format" if $format && !$drive->{format};
1967
1968     my $cache_direct = 0;
1969
1970     if (my $cache = $drive->{cache}) {
1971         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1972     } elsif (!drive_is_cdrom($drive)) {
1973         $opts .= ",cache=none";
1974         $cache_direct = 1;
1975     }
1976
1977     # aio native works only with O_DIRECT
1978     if (!$drive->{aio}) {
1979         if($cache_direct) {
1980             $opts .= ",aio=native";
1981         } else {
1982             $opts .= ",aio=threads";
1983         }
1984     }
1985
1986     if (!drive_is_cdrom($drive)) {
1987         my $detectzeroes;
1988         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1989             $detectzeroes = 'off';
1990         } elsif ($drive->{discard}) {
1991             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1992         } else {
1993             # This used to be our default with discard not being specified:
1994             $detectzeroes = 'on';
1995         }
1996         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1997     }
1998
1999     my $pathinfo = $path ? "file=$path," : '';
2000
2001     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2002 }
2003
2004 sub print_netdevice_full {
2005     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2006
2007     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2008
2009     my $device = $net->{model};
2010     if ($net->{model} eq 'virtio') {
2011          $device = 'virtio-net-pci';
2012      };
2013
2014     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2015     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2016     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2017         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2018         my $vectors = $net->{queues} * 2 + 2;
2019         $tmpstr .= ",vectors=$vectors,mq=on";
2020     }
2021     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2022
2023     if ($use_old_bios_files) {
2024         my $romfile;
2025         if ($device eq 'virtio-net-pci') {
2026             $romfile = 'pxe-virtio.rom';
2027         } elsif ($device eq 'e1000') {
2028             $romfile = 'pxe-e1000.rom';
2029         } elsif ($device eq 'ne2k') {
2030             $romfile = 'pxe-ne2k_pci.rom';
2031         } elsif ($device eq 'pcnet') {
2032             $romfile = 'pxe-pcnet.rom';
2033         } elsif ($device eq 'rtl8139') {
2034             $romfile = 'pxe-rtl8139.rom';
2035         }
2036         $tmpstr .= ",romfile=$romfile" if $romfile;
2037     }
2038
2039     return $tmpstr;
2040 }
2041
2042 sub print_netdev_full {
2043     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2044
2045     my $i = '';
2046     if ($netid =~ m/^net(\d+)$/) {
2047         $i = int($1);
2048     }
2049
2050     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2051
2052     my $ifname = "tap${vmid}i$i";
2053
2054     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2055     die "interface name '$ifname' is too long (max 15 character)\n"
2056         if length($ifname) >= 16;
2057
2058     my $vhostparam = '';
2059     if (is_native($arch)) {
2060         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2061     }
2062
2063     my $vmname = $conf->{name} || "vm$vmid";
2064
2065     my $netdev = "";
2066     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2067
2068     if ($net->{bridge}) {
2069         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2070     } else {
2071         $netdev = "type=user,id=$netid,hostname=$vmname";
2072     }
2073
2074     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2075
2076     return $netdev;
2077 }
2078
2079
2080 sub print_cpu_device {
2081     my ($conf, $id) = @_;
2082
2083     my $kvm = $conf->{kvm} // 1;
2084     my $cpu = $kvm ? "kvm64" : "qemu64";
2085     if (my $cputype = $conf->{cpu}) {
2086         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2087             or die "Cannot parse cpu description: $cputype\n";
2088         $cpu = $cpuconf->{cputype};
2089     }
2090
2091     my $cores = $conf->{cores} || 1;
2092
2093     my $current_core = ($id - 1) % $cores;
2094     my $current_socket = int(($id - 1 - $current_core)/$cores);
2095
2096     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2097 }
2098
2099 my $vga_map = {
2100     'cirrus' => 'cirrus-vga',
2101     'std' => 'VGA',
2102     'vmware' => 'vmware-svga',
2103     'virtio' => 'virtio-vga',
2104 };
2105
2106 sub print_vga_device {
2107     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2108
2109     my $type = $vga_map->{$vga->{type}};
2110     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2111         $type = 'virtio-gpu';
2112     }
2113     my $vgamem_mb = $vga->{memory};
2114     if ($qxlnum) {
2115         $type = $id ? 'qxl' : 'qxl-vga';
2116     }
2117     die "no devicetype for $vga->{type}\n" if !$type;
2118
2119     my $memory = "";
2120     if ($vgamem_mb) {
2121         if ($vga->{type} eq 'virtio') {
2122             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2123             $memory = ",max_hostmem=$bytes";
2124         } elsif ($qxlnum) {
2125             # from https://www.spice-space.org/multiple-monitors.html
2126             $memory = ",vgamem_mb=$vga->{memory}";
2127             my $ram = $vgamem_mb * 4;
2128             my $vram = $vgamem_mb * 2;
2129             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2130         } else {
2131             $memory = ",vgamem_mb=$vga->{memory}";
2132         }
2133     } elsif ($qxlnum && $id) {
2134         $memory = ",ram_size=67108864,vram_size=33554432";
2135     }
2136
2137     my $q35 = machine_type_is_q35($conf);
2138     my $vgaid = "vga" . ($id // '');
2139     my $pciaddr;
2140
2141     if ($q35 && $vgaid eq 'vga') {
2142         # the first display uses pcie.0 bus on q35 machines
2143         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2144     } else {
2145         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2146     }
2147
2148     return "$type,id=${vgaid}${memory}${pciaddr}";
2149 }
2150
2151 sub drive_is_cloudinit {
2152     my ($drive) = @_;
2153     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2154 }
2155
2156 sub drive_is_cdrom {
2157     my ($drive, $exclude_cloudinit) = @_;
2158
2159     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2160
2161     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2162
2163 }
2164
2165 sub parse_number_sets {
2166     my ($set) = @_;
2167     my $res = [];
2168     foreach my $part (split(/;/, $set)) {
2169         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2170             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2171             push @$res, [ $1, $2 ];
2172         } else {
2173             die "invalid range: $part\n";
2174         }
2175     }
2176     return $res;
2177 }
2178
2179 sub parse_numa {
2180     my ($data) = @_;
2181
2182     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2183     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2184     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2185     return $res;
2186 }
2187
2188 sub parse_hostpci {
2189     my ($value) = @_;
2190
2191     return undef if !$value;
2192
2193     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2194
2195     my @idlist = split(/;/, $res->{host});
2196     delete $res->{host};
2197     foreach my $id (@idlist) {
2198         if ($id =~ m/\./) { # full id 00:00.1
2199             push @{$res->{pciid}}, {
2200                 id => $id,
2201             };
2202         } else { # partial id 00:00
2203             $res->{pciid} = PVE::SysFSTools::lspci($id);
2204         }
2205     }
2206     return $res;
2207 }
2208
2209 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2210 sub parse_net {
2211     my ($data) = @_;
2212
2213     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2214     if ($@) {
2215         warn $@;
2216         return undef;
2217     }
2218     if (!defined($res->{macaddr})) {
2219         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2220         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2221     }
2222     return $res;
2223 }
2224
2225 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2226 sub parse_ipconfig {
2227     my ($data) = @_;
2228
2229     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2230     if ($@) {
2231         warn $@;
2232         return undef;
2233     }
2234
2235     if ($res->{gw} && !$res->{ip}) {
2236         warn 'gateway specified without specifying an IP address';
2237         return undef;
2238     }
2239     if ($res->{gw6} && !$res->{ip6}) {
2240         warn 'IPv6 gateway specified without specifying an IPv6 address';
2241         return undef;
2242     }
2243     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2244         warn 'gateway specified together with DHCP';
2245         return undef;
2246     }
2247     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2248         # gw6 + auto/dhcp
2249         warn "IPv6 gateway specified together with $res->{ip6} address";
2250         return undef;
2251     }
2252
2253     if (!$res->{ip} && !$res->{ip6}) {
2254         return { ip => 'dhcp', ip6 => 'dhcp' };
2255     }
2256
2257     return $res;
2258 }
2259
2260 sub print_net {
2261     my $net = shift;
2262
2263     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2264 }
2265
2266 sub add_random_macs {
2267     my ($settings) = @_;
2268
2269     foreach my $opt (keys %$settings) {
2270         next if $opt !~ m/^net(\d+)$/;
2271         my $net = parse_net($settings->{$opt});
2272         next if !$net;
2273         $settings->{$opt} = print_net($net);
2274     }
2275 }
2276
2277 sub vm_is_volid_owner {
2278     my ($storecfg, $vmid, $volid) = @_;
2279
2280     if ($volid !~  m|^/|) {
2281         my ($path, $owner);
2282         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2283         if ($owner && ($owner == $vmid)) {
2284             return 1;
2285         }
2286     }
2287
2288     return undef;
2289 }
2290
2291 sub split_flagged_list {
2292     my $text = shift || '';
2293     $text =~ s/[,;]/ /g;
2294     $text =~ s/^\s+//;
2295     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2296 }
2297
2298 sub join_flagged_list {
2299     my ($how, $lst) = @_;
2300     join $how, map { $lst->{$_} . $_ } keys %$lst;
2301 }
2302
2303 sub vmconfig_delete_pending_option {
2304     my ($conf, $key, $force) = @_;
2305
2306     delete $conf->{pending}->{$key};
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     $pending_delete_hash->{$key} = $force ? '!' : '';
2309     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2310 }
2311
2312 sub vmconfig_undelete_pending_option {
2313     my ($conf, $key) = @_;
2314
2315     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2316     delete $pending_delete_hash->{$key};
2317
2318     if (%$pending_delete_hash) {
2319         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2320     } else {
2321         delete $conf->{pending}->{delete};
2322     }
2323 }
2324
2325 sub vmconfig_register_unused_drive {
2326     my ($storecfg, $vmid, $conf, $drive) = @_;
2327
2328     if (drive_is_cloudinit($drive)) {
2329         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2330         warn $@ if $@;
2331     } elsif (!drive_is_cdrom($drive)) {
2332         my $volid = $drive->{file};
2333         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2334             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2335         }
2336     }
2337 }
2338
2339 sub vmconfig_cleanup_pending {
2340     my ($conf) = @_;
2341
2342     # remove pending changes when nothing changed
2343     my $changes;
2344     foreach my $opt (keys %{$conf->{pending}}) {
2345         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2346             $changes = 1;
2347             delete $conf->{pending}->{$opt};
2348         }
2349     }
2350
2351     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2352     my $pending_delete_hash = {};
2353     while (my ($opt, $force) = each %$current_delete_hash) {
2354         if (defined($conf->{$opt})) {
2355             $pending_delete_hash->{$opt} = $force;
2356         } else {
2357             $changes = 1;
2358         }
2359     }
2360
2361     if (%$pending_delete_hash) {
2362         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2363     } else {
2364         delete $conf->{pending}->{delete};
2365     }
2366
2367     return $changes;
2368 }
2369
2370 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2371 my $smbios1_fmt = {
2372     uuid => {
2373         type => 'string',
2374         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2375         format_description => 'UUID',
2376         description => "Set SMBIOS1 UUID.",
2377         optional => 1,
2378     },
2379     version => {
2380         type => 'string',
2381         pattern => '\S+',
2382         format_description => 'string',
2383         description => "Set SMBIOS1 version.",
2384         optional => 1,
2385     },
2386     serial => {
2387         type => 'string',
2388         pattern => '\S+',
2389         format_description => 'string',
2390         description => "Set SMBIOS1 serial number.",
2391         optional => 1,
2392     },
2393     manufacturer => {
2394         type => 'string',
2395         pattern => '\S+',
2396         format_description => 'string',
2397         description => "Set SMBIOS1 manufacturer.",
2398         optional => 1,
2399     },
2400     product => {
2401         type => 'string',
2402         pattern => '\S+',
2403         format_description => 'string',
2404         description => "Set SMBIOS1 product ID.",
2405         optional => 1,
2406     },
2407     sku => {
2408         type => 'string',
2409         pattern => '\S+',
2410         format_description => 'string',
2411         description => "Set SMBIOS1 SKU string.",
2412         optional => 1,
2413     },
2414     family => {
2415         type => 'string',
2416         pattern => '\S+',
2417         format_description => 'string',
2418         description => "Set SMBIOS1 family string.",
2419         optional => 1,
2420     },
2421 };
2422
2423 sub parse_smbios1 {
2424     my ($data) = @_;
2425
2426     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2427     warn $@ if $@;
2428     return $res;
2429 }
2430
2431 sub print_smbios1 {
2432     my ($smbios1) = @_;
2433     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2434 }
2435
2436 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2437
2438 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2439 sub verify_bootdisk {
2440     my ($value, $noerr) = @_;
2441
2442     return $value if is_valid_drivename($value);
2443
2444     return undef if $noerr;
2445
2446     die "invalid boot disk '$value'\n";
2447 }
2448
2449 sub parse_watchdog {
2450     my ($value) = @_;
2451
2452     return undef if !$value;
2453
2454     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2455     warn $@ if $@;
2456     return $res;
2457 }
2458
2459 sub parse_guest_agent {
2460     my ($value) = @_;
2461
2462     return {} if !defined($value->{agent});
2463
2464     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2465     warn $@ if $@;
2466
2467     # if the agent is disabled ignore the other potentially set properties
2468     return {} if !$res->{enabled};
2469     return $res;
2470 }
2471
2472 sub parse_vga {
2473     my ($value) = @_;
2474
2475     return {} if !$value;
2476     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2477     warn $@ if $@;
2478     return $res;
2479 }
2480
2481 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2482 sub verify_usb_device {
2483     my ($value, $noerr) = @_;
2484
2485     return $value if parse_usb_device($value);
2486
2487     return undef if $noerr;
2488
2489     die "unable to parse usb device\n";
2490 }
2491
2492 # add JSON properties for create and set function
2493 sub json_config_properties {
2494     my $prop = shift;
2495
2496     foreach my $opt (keys %$confdesc) {
2497         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2498         $prop->{$opt} = $confdesc->{$opt};
2499     }
2500
2501     return $prop;
2502 }
2503
2504 # return copy of $confdesc_cloudinit to generate documentation
2505 sub cloudinit_config_properties {
2506
2507     return dclone($confdesc_cloudinit);
2508 }
2509
2510 sub check_type {
2511     my ($key, $value) = @_;
2512
2513     die "unknown setting '$key'\n" if !$confdesc->{$key};
2514
2515     my $type = $confdesc->{$key}->{type};
2516
2517     if (!defined($value)) {
2518         die "got undefined value\n";
2519     }
2520
2521     if ($value =~ m/[\n\r]/) {
2522         die "property contains a line feed\n";
2523     }
2524
2525     if ($type eq 'boolean') {
2526         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2527         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2528         die "type check ('boolean') failed - got '$value'\n";
2529     } elsif ($type eq 'integer') {
2530         return int($1) if $value =~ m/^(\d+)$/;
2531         die "type check ('integer') failed - got '$value'\n";
2532     } elsif ($type eq 'number') {
2533         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2534         die "type check ('number') failed - got '$value'\n";
2535     } elsif ($type eq 'string') {
2536         if (my $fmt = $confdesc->{$key}->{format}) {
2537             PVE::JSONSchema::check_format($fmt, $value);
2538             return $value;
2539         }
2540         $value =~ s/^\"(.*)\"$/$1/;
2541         return $value;
2542     } else {
2543         die "internal error"
2544     }
2545 }
2546
2547 sub touch_config {
2548     my ($vmid) = @_;
2549
2550     my $conf = PVE::QemuConfig->config_file($vmid);
2551     utime undef, undef, $conf;
2552 }
2553
2554 sub destroy_vm {
2555     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2556
2557     my $conffile = PVE::QemuConfig->config_file($vmid);
2558
2559     my $conf = PVE::QemuConfig->load_config($vmid);
2560
2561     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2562
2563     if ($conf->{template}) {
2564         # check if any base image is still used by a linked clone
2565         foreach_drive($conf, sub {
2566                 my ($ds, $drive) = @_;
2567
2568                 return if drive_is_cdrom($drive);
2569
2570                 my $volid = $drive->{file};
2571
2572                 return if !$volid || $volid =~ m|^/|;
2573
2574                 die "base volume '$volid' is still in use by linked cloned\n"
2575                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2576
2577         });
2578     }
2579
2580     # only remove disks owned by this VM
2581     foreach_drive($conf, sub {
2582         my ($ds, $drive) = @_;
2583
2584         return if drive_is_cdrom($drive, 1);
2585
2586         my $volid = $drive->{file};
2587
2588         return if !$volid || $volid =~ m|^/|;
2589
2590         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2591         return if !$path || !$owner || ($owner != $vmid);
2592
2593         eval {
2594             PVE::Storage::vdisk_free($storecfg, $volid);
2595         };
2596         warn "Could not remove disk '$volid', check manually: $@" if $@;
2597
2598     });
2599
2600     if ($keep_empty_config) {
2601         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2602     } else {
2603         unlink $conffile;
2604     }
2605
2606     # also remove unused disk
2607     eval {
2608         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2609
2610         eval {
2611             PVE::Storage::foreach_volid($dl, sub {
2612                 my ($volid, $sid, $volname, $d) = @_;
2613                 PVE::Storage::vdisk_free($storecfg, $volid);
2614             });
2615         };
2616         warn $@ if $@;
2617
2618     };
2619     warn $@ if $@;
2620 }
2621
2622 sub parse_vm_config {
2623     my ($filename, $raw) = @_;
2624
2625     return undef if !defined($raw);
2626
2627     my $res = {
2628         digest => Digest::SHA::sha1_hex($raw),
2629         snapshots => {},
2630         pending => {},
2631     };
2632
2633     $filename =~ m|/qemu-server/(\d+)\.conf$|
2634         || die "got strange filename '$filename'";
2635
2636     my $vmid = $1;
2637
2638     my $conf = $res;
2639     my $descr;
2640     my $section = '';
2641
2642     my @lines = split(/\n/, $raw);
2643     foreach my $line (@lines) {
2644         next if $line =~ m/^\s*$/;
2645
2646         if ($line =~ m/^\[PENDING\]\s*$/i) {
2647             $section = 'pending';
2648             if (defined($descr)) {
2649                 $descr =~ s/\s+$//;
2650                 $conf->{description} = $descr;
2651             }
2652             $descr = undef;
2653             $conf = $res->{$section} = {};
2654             next;
2655
2656         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2657             $section = $1;
2658             if (defined($descr)) {
2659                 $descr =~ s/\s+$//;
2660                 $conf->{description} = $descr;
2661             }
2662             $descr = undef;
2663             $conf = $res->{snapshots}->{$section} = {};
2664             next;
2665         }
2666
2667         if ($line =~ m/^\#(.*)\s*$/) {
2668             $descr = '' if !defined($descr);
2669             $descr .= PVE::Tools::decode_text($1) . "\n";
2670             next;
2671         }
2672
2673         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2674             $descr = '' if !defined($descr);
2675             $descr .= PVE::Tools::decode_text($2);
2676         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2677             $conf->{snapstate} = $1;
2678         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2679             my $key = $1;
2680             my $value = $2;
2681             $conf->{$key} = $value;
2682         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2683             my $value = $1;
2684             if ($section eq 'pending') {
2685                 $conf->{delete} = $value; # we parse this later
2686             } else {
2687                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2688             }
2689         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2690             my $key = $1;
2691             my $value = $2;
2692             eval { $value = check_type($key, $value); };
2693             if ($@) {
2694                 warn "vm $vmid - unable to parse value of '$key' - $@";
2695             } else {
2696                 $key = 'ide2' if $key eq 'cdrom';
2697                 my $fmt = $confdesc->{$key}->{format};
2698                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2699                     my $v = parse_drive($key, $value);
2700                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2701                         $v->{file} = $volid;
2702                         $value = print_drive($vmid, $v);
2703                     } else {
2704                         warn "vm $vmid - unable to parse value of '$key'\n";
2705                         next;
2706                     }
2707                 }
2708
2709                 $conf->{$key} = $value;
2710             }
2711         }
2712     }
2713
2714     if (defined($descr)) {
2715         $descr =~ s/\s+$//;
2716         $conf->{description} = $descr;
2717     }
2718     delete $res->{snapstate}; # just to be sure
2719
2720     return $res;
2721 }
2722
2723 sub write_vm_config {
2724     my ($filename, $conf) = @_;
2725
2726     delete $conf->{snapstate}; # just to be sure
2727
2728     if ($conf->{cdrom}) {
2729         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2730         $conf->{ide2} = $conf->{cdrom};
2731         delete $conf->{cdrom};
2732     }
2733
2734     # we do not use 'smp' any longer
2735     if ($conf->{sockets}) {
2736         delete $conf->{smp};
2737     } elsif ($conf->{smp}) {
2738         $conf->{sockets} = $conf->{smp};
2739         delete $conf->{cores};
2740         delete $conf->{smp};
2741     }
2742
2743     my $used_volids = {};
2744
2745     my $cleanup_config = sub {
2746         my ($cref, $pending, $snapname) = @_;
2747
2748         foreach my $key (keys %$cref) {
2749             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2750                 $key eq 'snapstate' || $key eq 'pending';
2751             my $value = $cref->{$key};
2752             if ($key eq 'delete') {
2753                 die "propertry 'delete' is only allowed in [PENDING]\n"
2754                     if !$pending;
2755                 # fixme: check syntax?
2756                 next;
2757             }
2758             eval { $value = check_type($key, $value); };
2759             die "unable to parse value of '$key' - $@" if $@;
2760
2761             $cref->{$key} = $value;
2762
2763             if (!$snapname && is_valid_drivename($key)) {
2764                 my $drive = parse_drive($key, $value);
2765                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2766             }
2767         }
2768     };
2769
2770     &$cleanup_config($conf);
2771
2772     &$cleanup_config($conf->{pending}, 1);
2773
2774     foreach my $snapname (keys %{$conf->{snapshots}}) {
2775         die "internal error" if $snapname eq 'pending';
2776         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2777     }
2778
2779     # remove 'unusedX' settings if we re-add a volume
2780     foreach my $key (keys %$conf) {
2781         my $value = $conf->{$key};
2782         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2783             delete $conf->{$key};
2784         }
2785     }
2786
2787     my $generate_raw_config = sub {
2788         my ($conf, $pending) = @_;
2789
2790         my $raw = '';
2791
2792         # add description as comment to top of file
2793         if (defined(my $descr = $conf->{description})) {
2794             if ($descr) {
2795                 foreach my $cl (split(/\n/, $descr)) {
2796                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2797                 }
2798             } else {
2799                 $raw .= "#\n" if $pending;
2800             }
2801         }
2802
2803         foreach my $key (sort keys %$conf) {
2804             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2805             $raw .= "$key: $conf->{$key}\n";
2806         }
2807         return $raw;
2808     };
2809
2810     my $raw = &$generate_raw_config($conf);
2811
2812     if (scalar(keys %{$conf->{pending}})){
2813         $raw .= "\n[PENDING]\n";
2814         $raw .= &$generate_raw_config($conf->{pending}, 1);
2815     }
2816
2817     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2818         $raw .= "\n[$snapname]\n";
2819         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2820     }
2821
2822     return $raw;
2823 }
2824
2825 sub load_defaults {
2826
2827     my $res = {};
2828
2829     # we use static defaults from our JSON schema configuration
2830     foreach my $key (keys %$confdesc) {
2831         if (defined(my $default = $confdesc->{$key}->{default})) {
2832             $res->{$key} = $default;
2833         }
2834     }
2835
2836     return $res;
2837 }
2838
2839 sub config_list {
2840     my $vmlist = PVE::Cluster::get_vmlist();
2841     my $res = {};
2842     return $res if !$vmlist || !$vmlist->{ids};
2843     my $ids = $vmlist->{ids};
2844
2845     foreach my $vmid (keys %$ids) {
2846         my $d = $ids->{$vmid};
2847         next if !$d->{node} || $d->{node} ne $nodename;
2848         next if !$d->{type} || $d->{type} ne 'qemu';
2849         $res->{$vmid}->{exists} = 1;
2850     }
2851     return $res;
2852 }
2853
2854 # test if VM uses local resources (to prevent migration)
2855 sub check_local_resources {
2856     my ($conf, $noerr) = @_;
2857
2858     my $loc_res = 0;
2859
2860     $loc_res = 1 if $conf->{hostusb}; # old syntax
2861     $loc_res = 1 if $conf->{hostpci}; # old syntax
2862
2863     $loc_res = 1 if $conf->{ivshmem};
2864
2865     foreach my $k (keys %$conf) {
2866         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2867         # sockets are safe: they will recreated be on the target side post-migrate
2868         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2869         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2870     }
2871
2872     die "VM uses local resources\n" if $loc_res && !$noerr;
2873
2874     return $loc_res;
2875 }
2876
2877 # check if used storages are available on all nodes (use by migrate)
2878 sub check_storage_availability {
2879     my ($storecfg, $conf, $node) = @_;
2880
2881     foreach_drive($conf, sub {
2882         my ($ds, $drive) = @_;
2883
2884         my $volid = $drive->{file};
2885         return if !$volid;
2886
2887         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2888         return if !$sid;
2889
2890         # check if storage is available on both nodes
2891         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2892         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2893    });
2894 }
2895
2896 # list nodes where all VM images are available (used by has_feature API)
2897 sub shared_nodes {
2898     my ($conf, $storecfg) = @_;
2899
2900     my $nodelist = PVE::Cluster::get_nodelist();
2901     my $nodehash = { map { $_ => 1 } @$nodelist };
2902     my $nodename = PVE::INotify::nodename();
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913             if ($scfg->{disable}) {
2914                 $nodehash = {};
2915             } elsif (my $avail = $scfg->{nodes}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if !$avail->{$node};
2918                 }
2919             } elsif (!$scfg->{shared}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     delete $nodehash->{$node} if $node ne $nodename
2922                 }
2923             }
2924         }
2925     });
2926
2927     return $nodehash
2928 }
2929
2930 sub check_cmdline {
2931     my ($pidfile, $pid) = @_;
2932
2933     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2934     if (defined($fh)) {
2935         my $line = <$fh>;
2936         $fh->close;
2937         return undef if !$line;
2938         my @param = split(/\0/, $line);
2939
2940         my $cmd = $param[0];
2941         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2942
2943         for (my $i = 0; $i < scalar (@param); $i++) {
2944             my $p = $param[$i];
2945             next if !$p;
2946             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2947                 my $p = $param[$i+1];
2948                 return 1 if $p && ($p eq $pidfile);
2949                 return undef;
2950             }
2951         }
2952     }
2953     return undef;
2954 }
2955
2956 sub check_running {
2957     my ($vmid, $nocheck, $node) = @_;
2958
2959     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2960
2961     die "unable to find configuration file for VM $vmid - no such machine\n"
2962         if !$nocheck && ! -f $filename;
2963
2964     my $pidfile = pidfile_name($vmid);
2965
2966     if (my $fd = IO::File->new("<$pidfile")) {
2967         my $st = stat($fd);
2968         my $line = <$fd>;
2969         close($fd);
2970
2971         my $mtime = $st->mtime;
2972         if ($mtime > time()) {
2973             warn "file '$filename' modified in future\n";
2974         }
2975
2976         if ($line =~ m/^(\d+)$/) {
2977             my $pid = $1;
2978             if (check_cmdline($pidfile, $pid)) {
2979                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2980                     return $pid;
2981                 }
2982             }
2983         }
2984     }
2985
2986     return undef;
2987 }
2988
2989 sub vzlist {
2990
2991     my $vzlist = config_list();
2992
2993     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2994
2995     while (defined(my $de = $fd->read)) {
2996         next if $de !~ m/^(\d+)\.pid$/;
2997         my $vmid = $1;
2998         next if !defined($vzlist->{$vmid});
2999         if (my $pid = check_running($vmid)) {
3000             $vzlist->{$vmid}->{pid} = $pid;
3001         }
3002     }
3003
3004     return $vzlist;
3005 }
3006
3007 sub disksize {
3008     my ($storecfg, $conf) = @_;
3009
3010     my $bootdisk = $conf->{bootdisk};
3011     return undef if !$bootdisk;
3012     return undef if !is_valid_drivename($bootdisk);
3013
3014     return undef if !$conf->{$bootdisk};
3015
3016     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3017     return undef if !defined($drive);
3018
3019     return undef if drive_is_cdrom($drive);
3020
3021     my $volid = $drive->{file};
3022     return undef if !$volid;
3023
3024     return $drive->{size};
3025 }
3026
3027 our $vmstatus_return_properties = {
3028     vmid => get_standard_option('pve-vmid'),
3029     status => {
3030         description => "Qemu process status.",
3031         type => 'string',
3032         enum => ['stopped', 'running'],
3033     },
3034     maxmem => {
3035         description => "Maximum memory in bytes.",
3036         type => 'integer',
3037         optional => 1,
3038         renderer => 'bytes',
3039     },
3040     maxdisk => {
3041         description => "Root disk size in bytes.",
3042         type => 'integer',
3043         optional => 1,
3044         renderer => 'bytes',
3045     },
3046     name => {
3047         description => "VM name.",
3048         type => 'string',
3049         optional => 1,
3050     },
3051     qmpstatus => {
3052         description => "Qemu QMP agent status.",
3053         type => 'string',
3054         optional => 1,
3055     },
3056     pid => {
3057         description => "PID of running qemu process.",
3058         type => 'integer',
3059         optional => 1,
3060     },
3061     uptime => {
3062         description => "Uptime.",
3063         type => 'integer',
3064         optional => 1,
3065         renderer => 'duration',
3066     },
3067     cpus => {
3068         description => "Maximum usable CPUs.",
3069         type => 'number',
3070         optional => 1,
3071     },
3072 };
3073
3074 my $last_proc_pid_stat;
3075
3076 # get VM status information
3077 # This must be fast and should not block ($full == false)
3078 # We only query KVM using QMP if $full == true (this can be slow)
3079 sub vmstatus {
3080     my ($opt_vmid, $full) = @_;
3081
3082     my $res = {};
3083
3084     my $storecfg = PVE::Storage::config();
3085
3086     my $list = vzlist();
3087     my $defaults = load_defaults();
3088
3089     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3090
3091     my $cpucount = $cpuinfo->{cpus} || 1;
3092
3093     foreach my $vmid (keys %$list) {
3094         next if $opt_vmid && ($vmid ne $opt_vmid);
3095
3096         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3097         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3098
3099         my $d = { vmid => $vmid };
3100         $d->{pid} = $list->{$vmid}->{pid};
3101
3102         # fixme: better status?
3103         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3104
3105         my $size = disksize($storecfg, $conf);
3106         if (defined($size)) {
3107             $d->{disk} = 0; # no info available
3108             $d->{maxdisk} = $size;
3109         } else {
3110             $d->{disk} = 0;
3111             $d->{maxdisk} = 0;
3112         }
3113
3114         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3115             * ($conf->{cores} || $defaults->{cores});
3116         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3117         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3118
3119         $d->{name} = $conf->{name} || "VM $vmid";
3120         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3121             : $defaults->{memory}*(1024*1024);
3122
3123         if ($conf->{balloon}) {
3124             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3125             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3126                 : $defaults->{shares};
3127         }
3128
3129         $d->{uptime} = 0;
3130         $d->{cpu} = 0;
3131         $d->{mem} = 0;
3132
3133         $d->{netout} = 0;
3134         $d->{netin} = 0;
3135
3136         $d->{diskread} = 0;
3137         $d->{diskwrite} = 0;
3138
3139         $d->{template} = PVE::QemuConfig->is_template($conf);
3140
3141         $d->{serial} = 1 if conf_has_serial($conf);
3142
3143         $res->{$vmid} = $d;
3144     }
3145
3146     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3147     foreach my $dev (keys %$netdev) {
3148         next if $dev !~ m/^tap([1-9]\d*)i/;
3149         my $vmid = $1;
3150         my $d = $res->{$vmid};
3151         next if !$d;
3152
3153         $d->{netout} += $netdev->{$dev}->{receive};
3154         $d->{netin} += $netdev->{$dev}->{transmit};
3155
3156         if ($full) {
3157             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3158             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3159         }
3160
3161     }
3162
3163     my $ctime = gettimeofday;
3164
3165     foreach my $vmid (keys %$list) {
3166
3167         my $d = $res->{$vmid};
3168         my $pid = $d->{pid};
3169         next if !$pid;
3170
3171         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3172         next if !$pstat; # not running
3173
3174         my $used = $pstat->{utime} + $pstat->{stime};
3175
3176         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3177
3178         if ($pstat->{vsize}) {
3179             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3180         }
3181
3182         my $old = $last_proc_pid_stat->{$pid};
3183         if (!$old) {
3184             $last_proc_pid_stat->{$pid} = {
3185                 time => $ctime,
3186                 used => $used,
3187                 cpu => 0,
3188             };
3189             next;
3190         }
3191
3192         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3193
3194         if ($dtime > 1000) {
3195             my $dutime = $used -  $old->{used};
3196
3197             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3198             $last_proc_pid_stat->{$pid} = {
3199                 time => $ctime,
3200                 used => $used,
3201                 cpu => $d->{cpu},
3202             };
3203         } else {
3204             $d->{cpu} = $old->{cpu};
3205         }
3206     }
3207
3208     return $res if !$full;
3209
3210     my $qmpclient = PVE::QMPClient->new();
3211
3212     my $ballooncb = sub {
3213         my ($vmid, $resp) = @_;
3214
3215         my $info = $resp->{'return'};
3216         return if !$info->{max_mem};
3217
3218         my $d = $res->{$vmid};
3219
3220         # use memory assigned to VM
3221         $d->{maxmem} = $info->{max_mem};
3222         $d->{balloon} = $info->{actual};
3223
3224         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3225             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3226             $d->{freemem} = $info->{free_mem};
3227         }
3228
3229         $d->{ballooninfo} = $info;
3230     };
3231
3232     my $blockstatscb = sub {
3233         my ($vmid, $resp) = @_;
3234         my $data = $resp->{'return'} || [];
3235         my $totalrdbytes = 0;
3236         my $totalwrbytes = 0;
3237
3238         for my $blockstat (@$data) {
3239             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3240             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3241
3242             $blockstat->{device} =~ s/drive-//;
3243             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3244         }
3245         $res->{$vmid}->{diskread} = $totalrdbytes;
3246         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3247     };
3248
3249     my $statuscb = sub {
3250         my ($vmid, $resp) = @_;
3251
3252         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3253         # this fails if ballon driver is not loaded, so this must be
3254         # the last commnand (following command are aborted if this fails).
3255         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3256
3257         my $status = 'unknown';
3258         if (!defined($status = $resp->{'return'}->{status})) {
3259             warn "unable to get VM status\n";
3260             return;
3261         }
3262
3263         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3264     };
3265
3266     foreach my $vmid (keys %$list) {
3267         next if $opt_vmid && ($vmid ne $opt_vmid);
3268         next if !$res->{$vmid}->{pid}; # not running
3269         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3270     }
3271
3272     $qmpclient->queue_execute(undef, 2);
3273
3274     foreach my $vmid (keys %$list) {
3275         next if $opt_vmid && ($vmid ne $opt_vmid);
3276         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3277     }
3278
3279     return $res;
3280 }
3281
3282 sub foreach_drive {
3283     my ($conf, $func, @param) = @_;
3284
3285     foreach my $ds (valid_drive_names()) {
3286         next if !defined($conf->{$ds});
3287
3288         my $drive = parse_drive($ds, $conf->{$ds});
3289         next if !$drive;
3290
3291         &$func($ds, $drive, @param);
3292     }
3293 }
3294
3295 sub foreach_volid {
3296     my ($conf, $func, @param) = @_;
3297
3298     my $volhash = {};
3299
3300     my $test_volid = sub {
3301         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3302
3303         return if !$volid;
3304
3305         $volhash->{$volid}->{cdrom} //= 1;
3306         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3307
3308         $volhash->{$volid}->{replicate} //= 0;
3309         $volhash->{$volid}->{replicate} = 1 if $replicate;
3310
3311         $volhash->{$volid}->{shared} //= 0;
3312         $volhash->{$volid}->{shared} = 1 if $shared;
3313
3314         $volhash->{$volid}->{referenced_in_config} //= 0;
3315         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3316
3317         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3318             if defined($snapname);
3319     };
3320
3321     foreach_drive($conf, sub {
3322         my ($ds, $drive) = @_;
3323         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3324     });
3325
3326     foreach my $snapname (keys %{$conf->{snapshots}}) {
3327         my $snap = $conf->{snapshots}->{$snapname};
3328         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3329         foreach_drive($snap, sub {
3330             my ($ds, $drive) = @_;
3331             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3332         });
3333     }
3334
3335     foreach my $volid (keys %$volhash) {
3336         &$func($volid, $volhash->{$volid}, @param);
3337     }
3338 }
3339
3340 sub conf_has_serial {
3341     my ($conf) = @_;
3342
3343     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3344         if ($conf->{"serial$i"}) {
3345             return 1;
3346         }
3347     }
3348
3349     return 0;
3350 }
3351
3352 sub vga_conf_has_spice {
3353     my ($vga) = @_;
3354
3355     my $vgaconf = parse_vga($vga);
3356     my $vgatype = $vgaconf->{type};
3357     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3358
3359     return $1 || 1;
3360 }
3361
3362 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3363 sub get_host_arch() {
3364     $host_arch = (POSIX::uname())[4] if !$host_arch;
3365     return $host_arch;
3366 }
3367
3368 sub is_native($) {
3369     my ($arch) = @_;
3370     return get_host_arch() eq $arch;
3371 }
3372
3373 my $default_machines = {
3374     x86_64 => 'pc',
3375     aarch64 => 'virt',
3376 };
3377
3378 sub get_basic_machine_info {
3379     my ($conf, $forcemachine) = @_;
3380
3381     my $arch = $conf->{arch} // get_host_arch();
3382     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3383     return ($arch, $machine);
3384 }
3385
3386 sub get_ovmf_files($) {
3387     my ($arch) = @_;
3388
3389     my $ovmf = $OVMF->{$arch}
3390         or die "no OVMF images known for architecture '$arch'\n";
3391
3392     return @$ovmf;
3393 }
3394
3395 my $Arch2Qemu = {
3396     aarch64 => '/usr/bin/qemu-system-aarch64',
3397     x86_64 => '/usr/bin/qemu-system-x86_64',
3398 };
3399 sub get_command_for_arch($) {
3400     my ($arch) = @_;
3401     return '/usr/bin/kvm' if is_native($arch);
3402
3403     my $cmd = $Arch2Qemu->{$arch}
3404         or die "don't know how to emulate architecture '$arch'\n";
3405     return $cmd;
3406 }
3407
3408 sub get_cpu_options {
3409     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3410
3411     my $cpuFlags = [];
3412     my $ostype = $conf->{ostype};
3413
3414     my $cpu = $kvm ? "kvm64" : "qemu64";
3415     if ($arch eq 'aarch64') {
3416         $cpu = 'cortex-a57';
3417     }
3418     my $hv_vendor_id;
3419     if (my $cputype = $conf->{cpu}) {
3420         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3421             or die "Cannot parse cpu description: $cputype\n";
3422         $cpu = $cpuconf->{cputype};
3423         $kvm_off = 1 if $cpuconf->{hidden};
3424         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3425
3426         if (defined(my $flags = $cpuconf->{flags})) {
3427             push @$cpuFlags, split(";", $flags);
3428         }
3429     }
3430
3431     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3432
3433     push @$cpuFlags , '-x2apic'
3434         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3435
3436     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3437
3438     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3439
3440     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3441
3442         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3443         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3444     }
3445
3446     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3447
3448     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3449
3450     push @$cpuFlags, 'kvm=off' if $kvm_off;
3451
3452     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3453         push @$cpuFlags, "vendor=${cpu_vendor}"
3454             if $cpu_vendor ne 'default';
3455     } elsif ($arch ne 'aarch64') {
3456         die "internal error"; # should not happen
3457     }
3458
3459     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3460
3461     return ('-cpu', $cpu);
3462 }
3463
3464 sub config_to_command {
3465     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3466
3467     my $cmd = [];
3468     my $globalFlags = [];
3469     my $machineFlags = [];
3470     my $rtcFlags = [];
3471     my $devices = [];
3472     my $pciaddr = '';
3473     my $bridges = {};
3474     my $kvmver = kvm_user_version();
3475     my $vernum = 0; # unknown
3476     my $ostype = $conf->{ostype};
3477     my $winversion = windows_version($ostype);
3478     my $kvm = $conf->{kvm};
3479
3480     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3481     $kvm //= 1 if is_native($arch);
3482
3483     if ($kvm) {
3484         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3485             if !defined kvm_version();
3486     }
3487
3488     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3489         $vernum = $1*1000000+$2*1000;
3490     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3491         $vernum = $1*1000000+$2*1000+$3;
3492     }
3493
3494     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3495
3496     my $have_ovz = -f '/proc/vz/vestat';
3497
3498     my $q35 = machine_type_is_q35($conf);
3499     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3500     my $use_old_bios_files = undef;
3501     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3502
3503     my $cpuunits = defined($conf->{cpuunits}) ?
3504             $conf->{cpuunits} : $defaults->{cpuunits};
3505
3506     push @$cmd, get_command_for_arch($arch);
3507
3508     push @$cmd, '-id', $vmid;
3509
3510     my $vmname = $conf->{name} || "vm$vmid";
3511
3512     push @$cmd, '-name', $vmname;
3513
3514     my $use_virtio = 0;
3515
3516     my $qmpsocket = qmp_socket($vmid);
3517     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3518     push @$cmd, '-mon', "chardev=qmp,mode=control";
3519
3520     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3521         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3522         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3523     }
3524
3525     push @$cmd, '-pidfile' , pidfile_name($vmid);
3526
3527     push @$cmd, '-daemonize';
3528
3529     if ($conf->{smbios1}) {
3530         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3531     }
3532
3533     if ($conf->{vmgenid}) {
3534         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3535     }
3536
3537     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3538     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3539         die "uefi base image not found\n" if ! -f $ovmf_code;
3540
3541         my $path;
3542         my $format;
3543         if (my $efidisk = $conf->{efidisk0}) {
3544             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3545             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3546             $format = $d->{format};
3547             if ($storeid) {
3548                 $path = PVE::Storage::path($storecfg, $d->{file});
3549                 if (!defined($format)) {
3550                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3551                     $format = qemu_img_format($scfg, $volname);
3552                 }
3553             } else {
3554                 $path = $d->{file};
3555                 die "efidisk format must be specified\n"
3556                     if !defined($format);
3557             }
3558         } else {
3559             warn "no efidisk configured! Using temporary efivars disk.\n";
3560             $path = "/tmp/$vmid-ovmf.fd";
3561             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3562             $format = 'raw';
3563         }
3564
3565         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3566         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3567     }
3568
3569
3570     # add usb controllers
3571     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3572     push @$devices, @usbcontrollers if @usbcontrollers;
3573     my $vga = parse_vga($conf->{vga});
3574
3575     my $qxlnum = vga_conf_has_spice($conf->{vga});
3576     $vga->{type} = 'qxl' if $qxlnum;
3577
3578     if (!$vga->{type}) {
3579         if ($arch eq 'aarch64') {
3580             $vga->{type} = 'virtio';
3581         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3582             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3583         } else {
3584             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3585         }
3586     }
3587
3588     # enable absolute mouse coordinates (needed by vnc)
3589     my $tablet;
3590     if (defined($conf->{tablet})) {
3591         $tablet = $conf->{tablet};
3592     } else {
3593         $tablet = $defaults->{tablet};
3594         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3595         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3596     }
3597
3598     if ($tablet) {
3599         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3600         my $kbd = print_keyboarddevice_full($conf, $arch);
3601         push @$devices, '-device', $kbd if defined($kbd);
3602     }
3603
3604     my $kvm_off = 0;
3605     my $gpu_passthrough;
3606
3607     # host pci devices
3608     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3609         my $d = parse_hostpci($conf->{"hostpci$i"});
3610         next if !$d;
3611
3612         my $pcie = $d->{pcie};
3613         if ($pcie) {
3614             die "q35 machine model is not enabled" if !$q35;
3615             # win7 wants to have the pcie devices directly on the pcie bus
3616             # instead of in the root port
3617             if ($winversion == 7) {
3618                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3619             } else {
3620                 $pciaddr = print_pcie_addr("hostpci$i");
3621             }
3622         } else {
3623             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3624         }
3625
3626         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3627         my $romfile = $d->{romfile};
3628
3629         my $xvga = '';
3630         if ($d->{'x-vga'}) {
3631             $xvga = ',x-vga=on';
3632             $kvm_off = 1;
3633             $vga->{type} = 'none' if !defined($conf->{vga});
3634             $gpu_passthrough = 1;
3635
3636             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3637                 $xvga = "";
3638             }
3639         }
3640         my $pcidevices = $d->{pciid};
3641         my $multifunction = 1 if @$pcidevices > 1;
3642         my $sysfspath;
3643         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3644             my $id = $pcidevices->[0]->{id};
3645             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3646             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3647         } elsif ($d->{mdev}) {
3648             warn "ignoring mediated device with multifunction device\n";
3649         }
3650
3651         my $j=0;
3652         foreach my $pcidevice (@$pcidevices) {
3653
3654             my $id = "hostpci$i";
3655             $id .= ".$j" if $multifunction;
3656             my $addr = $pciaddr;
3657             $addr .= ".$j" if $multifunction;
3658             my $devicestr = "vfio-pci";
3659             if ($sysfspath) {
3660                 $devicestr .= ",sysfsdev=$sysfspath";
3661             } else {
3662                 $devicestr .= ",host=$pcidevice->{id}";
3663             }
3664             $devicestr .= ",id=$id$addr";
3665
3666             if($j == 0){
3667                 $devicestr .= "$rombar$xvga";
3668                 $devicestr .= ",multifunction=on" if $multifunction;
3669                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3670             }
3671
3672             push @$devices, '-device', $devicestr;
3673             $j++;
3674         }
3675     }
3676
3677     # usb devices
3678     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3679     push @$devices, @usbdevices if @usbdevices;
3680     # serial devices
3681     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3682         if (my $path = $conf->{"serial$i"}) {
3683             if ($path eq 'socket') {
3684                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3685                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3686                 # On aarch64, serial0 is the UART device. Qemu only allows
3687                 # connecting UART devices via the '-serial' command line, as
3688                 # the device has a fixed slot on the hardware...
3689                 if ($arch eq 'aarch64' && $i == 0) {
3690                     push @$devices, '-serial', "chardev:serial$i";
3691                 } else {
3692                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3693                 }
3694             } else {
3695                 die "no such serial device\n" if ! -c $path;
3696                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3697                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3698             }
3699         }
3700     }
3701
3702     # parallel devices
3703     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3704         if (my $path = $conf->{"parallel$i"}) {
3705             die "no such parallel device\n" if ! -c $path;
3706             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3707             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3708             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3709         }
3710     }
3711
3712
3713     my $sockets = 1;
3714     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3715     $sockets = $conf->{sockets} if  $conf->{sockets};
3716
3717     my $cores = $conf->{cores} || 1;
3718
3719     my $maxcpus = $sockets * $cores;
3720
3721     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3722
3723     my $allowed_vcpus = $cpuinfo->{cpus};
3724
3725     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3726         if ($allowed_vcpus < $maxcpus);
3727
3728     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3729
3730         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3731         for (my $i = 2; $i <= $vcpus; $i++)  {
3732             my $cpustr = print_cpu_device($conf,$i);
3733             push @$cmd, '-device', $cpustr;
3734         }
3735
3736     } else {
3737
3738         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3739     }
3740     push @$cmd, '-nodefaults';
3741
3742     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3743
3744     my $bootindex_hash = {};
3745     my $i = 1;
3746     foreach my $o (split(//, $bootorder)) {
3747         $bootindex_hash->{$o} = $i*100;
3748         $i++;
3749     }
3750
3751     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3752
3753     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3754
3755     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3756
3757     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3758         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3759         my $socket = vnc_socket($vmid);
3760         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3761     } else {
3762         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3763         push @$cmd, '-nographic';
3764     }
3765
3766     # time drift fix
3767     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3768
3769     my $useLocaltime = $conf->{localtime};
3770
3771     if ($winversion >= 5) { # windows
3772         $useLocaltime = 1 if !defined($conf->{localtime});
3773
3774         # use time drift fix when acpi is enabled
3775         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3776             $tdf = 1 if !defined($conf->{tdf});
3777         }
3778     }
3779
3780     if ($winversion >= 6) {
3781         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3782         push @$cmd, '-no-hpet';
3783     }
3784
3785     push @$rtcFlags, 'driftfix=slew' if $tdf;
3786
3787     if (!$kvm) {
3788         push @$machineFlags, 'accel=tcg';
3789     }
3790
3791     if ($machine_type) {
3792         push @$machineFlags, "type=${machine_type}";
3793     }
3794
3795     if ($conf->{startdate}) {
3796         push @$rtcFlags, "base=$conf->{startdate}";
3797     } elsif ($useLocaltime) {
3798         push @$rtcFlags, 'base=localtime';
3799     }
3800
3801     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3802
3803     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3804
3805     push @$cmd, '-S' if $conf->{freeze};
3806
3807     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3808
3809     # enable sound
3810     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3811     #push @$cmd, '-soundhw', 'es1370';
3812     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3813
3814     if (parse_guest_agent($conf)->{enabled}) {
3815         my $qgasocket = qmp_socket($vmid, 1);
3816         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3817         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3818         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3819         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3820     }
3821
3822     my $spice_port;
3823
3824     if ($qxlnum) {
3825         if ($qxlnum > 1) {
3826             if ($winversion){
3827                 for(my $i = 1; $i < $qxlnum; $i++){
3828                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3829                 }
3830             } else {
3831                 # assume other OS works like Linux
3832                 my ($ram, $vram) = ("134217728", "67108864");
3833                 if ($vga->{memory}) {
3834                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3835                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3836                 }
3837                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3838                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3839             }
3840         }
3841
3842         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3843
3844         my $nodename = PVE::INotify::nodename();
3845         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3846         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3847         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3848         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3849         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3850
3851         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3852
3853         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3854         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3855         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3856     }
3857
3858     # enable balloon by default, unless explicitly disabled
3859     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3860         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3861         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3862     }
3863
3864     if ($conf->{watchdog}) {
3865         my $wdopts = parse_watchdog($conf->{watchdog});
3866         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3867         my $watchdog = $wdopts->{model} || 'i6300esb';
3868         push @$devices, '-device', "$watchdog$pciaddr";
3869         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3870     }
3871
3872     my $vollist = [];
3873     my $scsicontroller = {};
3874     my $ahcicontroller = {};
3875     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3876
3877     # Add iscsi initiator name if available
3878     if (my $initiator = get_initiator_name()) {
3879         push @$devices, '-iscsi', "initiator-name=$initiator";
3880     }
3881
3882     foreach_drive($conf, sub {
3883         my ($ds, $drive) = @_;
3884
3885         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3886             push @$vollist, $drive->{file};
3887         }
3888
3889         # ignore efidisk here, already added in bios/fw handling code above
3890         return if $drive->{interface} eq 'efidisk';
3891
3892         $use_virtio = 1 if $ds =~ m/^virtio/;
3893
3894         if (drive_is_cdrom ($drive)) {
3895             if ($bootindex_hash->{d}) {
3896                 $drive->{bootindex} = $bootindex_hash->{d};
3897                 $bootindex_hash->{d} += 1;
3898             }
3899         } else {
3900             if ($bootindex_hash->{c}) {
3901                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3902                 $bootindex_hash->{c} += 1;
3903             }
3904         }
3905
3906         if($drive->{interface} eq 'virtio'){
3907            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3908         }
3909
3910         if ($drive->{interface} eq 'scsi') {
3911
3912             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3913
3914             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3915             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3916
3917             my $iothread = '';
3918             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3919                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3920                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3921             } elsif ($drive->{iothread}) {
3922                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3923             }
3924
3925             my $queues = '';
3926             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3927                 $queues = ",num_queues=$drive->{queues}";
3928             }
3929
3930             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3931             $scsicontroller->{$controller}=1;
3932         }
3933
3934         if ($drive->{interface} eq 'sata') {
3935            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3936            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3937            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3938            $ahcicontroller->{$controller}=1;
3939         }
3940
3941         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3942         push @$devices, '-drive',$drive_cmd;
3943         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3944     });
3945
3946     for (my $i = 0; $i < $MAX_NETS; $i++) {
3947          next if !$conf->{"net$i"};
3948          my $d = parse_net($conf->{"net$i"});
3949          next if !$d;
3950
3951          $use_virtio = 1 if $d->{model} eq 'virtio';
3952
3953          if ($bootindex_hash->{n}) {
3954             $d->{bootindex} = $bootindex_hash->{n};
3955             $bootindex_hash->{n} += 1;
3956          }
3957
3958          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3959          push @$devices, '-netdev', $netdevfull;
3960
3961          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3962          push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3980     }
3981
3982     if (!$q35) {
3983         # add pci bridges
3984         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3985            $bridges->{1} = 1;
3986            $bridges->{2} = 1;
3987         }
3988
3989         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3990
3991         while (my ($k, $v) = each %$bridges) {
3992             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3993             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3994         }
3995     }
3996
3997     push @$cmd, @$devices;
3998     push @$cmd, '-rtc', join(',', @$rtcFlags)
3999         if scalar(@$rtcFlags);
4000     push @$cmd, '-machine', join(',', @$machineFlags)
4001         if scalar(@$machineFlags);
4002     push @$cmd, '-global', join(',', @$globalFlags)
4003         if scalar(@$globalFlags);
4004
4005     # add custom args
4006     if ($conf->{args}) {
4007         my $aa = PVE::Tools::split_args($conf->{args});
4008         push @$cmd, @$aa;
4009     }
4010
4011     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4012 }
4013
4014 sub vnc_socket {
4015     my ($vmid) = @_;
4016     return "${var_run_tmpdir}/$vmid.vnc";
4017 }
4018
4019 sub spice_port {
4020     my ($vmid) = @_;
4021
4022     my $res = vm_mon_cmd($vmid, 'query-spice');
4023
4024     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4025 }
4026
4027 sub qmp_socket {
4028     my ($vmid, $qga, $name) = @_;
4029     my $sockettype = $qga ? 'qga' : 'qmp';
4030     my $ext = $name ? '-'.$name : '';
4031     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4032 }
4033
4034 sub pidfile_name {
4035     my ($vmid) = @_;
4036     return "${var_run_tmpdir}/$vmid.pid";
4037 }
4038
4039 sub vm_devices_list {
4040     my ($vmid) = @_;
4041
4042     my $res = vm_mon_cmd($vmid, 'query-pci');
4043     my $devices_to_check = [];
4044     my $devices = {};
4045     foreach my $pcibus (@$res) {
4046         push @$devices_to_check, @{$pcibus->{devices}},
4047     }
4048
4049     while (@$devices_to_check) {
4050         my $to_check = [];
4051         for my $d (@$devices_to_check) {
4052             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4053             next if !$d->{'pci_bridge'};
4054
4055             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4056             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4057         }
4058         $devices_to_check = $to_check;
4059     }
4060
4061     my $resblock = vm_mon_cmd($vmid, 'query-block');
4062     foreach my $block (@$resblock) {
4063         if($block->{device} =~ m/^drive-(\S+)/){
4064                 $devices->{$1} = 1;
4065         }
4066     }
4067
4068     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4069     foreach my $mice (@$resmice) {
4070         if ($mice->{name} eq 'QEMU HID Tablet') {
4071             $devices->{tablet} = 1;
4072             last;
4073         }
4074     }
4075
4076     # for usb devices there is no query-usb
4077     # but we can iterate over the entries in
4078     # qom-list path=/machine/peripheral
4079     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4080     foreach my $per (@$resperipheral) {
4081         if ($per->{name} =~ m/^usb\d+$/) {
4082             $devices->{$per->{name}} = 1;
4083         }
4084     }
4085
4086     return $devices;
4087 }
4088
4089 sub vm_deviceplug {
4090     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4091
4092     my $q35 = machine_type_is_q35($conf);
4093
4094     my $devices_list = vm_devices_list($vmid);
4095     return 1 if defined($devices_list->{$deviceid});
4096
4097     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4098
4099     if ($deviceid eq 'tablet') {
4100
4101         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4102
4103     } elsif ($deviceid eq 'keyboard') {
4104
4105         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4106
4107     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4108
4109         die "usb hotplug currently not reliable\n";
4110         # since we can't reliably hot unplug all added usb devices
4111         # and usb passthrough disables live migration
4112         # we disable usb hotplugging for now
4113         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4114
4115     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4116
4117         qemu_iothread_add($vmid, $deviceid, $device);
4118
4119         qemu_driveadd($storecfg, $vmid, $device);
4120         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4121
4122         qemu_deviceadd($vmid, $devicefull);
4123         eval { qemu_deviceaddverify($vmid, $deviceid); };
4124         if (my $err = $@) {
4125             eval { qemu_drivedel($vmid, $deviceid); };
4126             warn $@ if $@;
4127             die $err;
4128         }
4129
4130     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4131
4132
4133         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4134         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4135         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4136
4137         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4138
4139         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4140             qemu_iothread_add($vmid, $deviceid, $device);
4141             $devicefull .= ",iothread=iothread-$deviceid";
4142         }
4143
4144         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4145             $devicefull .= ",num_queues=$device->{queues}";
4146         }
4147
4148         qemu_deviceadd($vmid, $devicefull);
4149         qemu_deviceaddverify($vmid, $deviceid);
4150
4151     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4152
4153         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4154         qemu_driveadd($storecfg, $vmid, $device);
4155
4156         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4157         eval { qemu_deviceadd($vmid, $devicefull); };
4158         if (my $err = $@) {
4159             eval { qemu_drivedel($vmid, $deviceid); };
4160             warn $@ if $@;
4161             die $err;
4162         }
4163
4164     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4165
4166         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4167
4168         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4169         my $use_old_bios_files = undef;
4170         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4171
4172         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4173         qemu_deviceadd($vmid, $netdevicefull);
4174         eval {
4175             qemu_deviceaddverify($vmid, $deviceid);
4176             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4177         };
4178         if (my $err = $@) {
4179             eval { qemu_netdevdel($vmid, $deviceid); };
4180             warn $@ if $@;
4181             die $err;
4182         }
4183
4184     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4185
4186         my $bridgeid = $2;
4187         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4188         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4189
4190         qemu_deviceadd($vmid, $devicefull);
4191         qemu_deviceaddverify($vmid, $deviceid);
4192
4193     } else {
4194         die "can't hotplug device '$deviceid'\n";
4195     }
4196
4197     return 1;
4198 }
4199
4200 # fixme: this should raise exceptions on error!
4201 sub vm_deviceunplug {
4202     my ($vmid, $conf, $deviceid) = @_;
4203
4204     my $devices_list = vm_devices_list($vmid);
4205     return 1 if !defined($devices_list->{$deviceid});
4206
4207     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4208
4209     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4210
4211         qemu_devicedel($vmid, $deviceid);
4212
4213     } elsif ($deviceid =~ m/^usb\d+$/) {
4214
4215         die "usb hotplug currently not reliable\n";
4216         # when unplugging usb devices this way,
4217         # there may be remaining usb controllers/hubs
4218         # so we disable it for now
4219         qemu_devicedel($vmid, $deviceid);
4220         qemu_devicedelverify($vmid, $deviceid);
4221
4222     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4223
4224         qemu_devicedel($vmid, $deviceid);
4225         qemu_devicedelverify($vmid, $deviceid);
4226         qemu_drivedel($vmid, $deviceid);
4227         qemu_iothread_del($conf, $vmid, $deviceid);
4228
4229     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4230
4231         qemu_devicedel($vmid, $deviceid);
4232         qemu_devicedelverify($vmid, $deviceid);
4233         qemu_iothread_del($conf, $vmid, $deviceid);
4234
4235     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4236
4237         qemu_devicedel($vmid, $deviceid);
4238         qemu_drivedel($vmid, $deviceid);
4239         qemu_deletescsihw($conf, $vmid, $deviceid);
4240
4241     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4242
4243         qemu_devicedel($vmid, $deviceid);
4244         qemu_devicedelverify($vmid, $deviceid);
4245         qemu_netdevdel($vmid, $deviceid);
4246
4247     } else {
4248         die "can't unplug device '$deviceid'\n";
4249     }
4250
4251     return 1;
4252 }
4253
4254 sub qemu_deviceadd {
4255     my ($vmid, $devicefull) = @_;
4256
4257     $devicefull = "driver=".$devicefull;
4258     my %options =  split(/[=,]/, $devicefull);
4259
4260     vm_mon_cmd($vmid, "device_add" , %options);
4261 }
4262
4263 sub qemu_devicedel {
4264     my ($vmid, $deviceid) = @_;
4265
4266     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4267 }
4268
4269 sub qemu_iothread_add {
4270     my($vmid, $deviceid, $device) = @_;
4271
4272     if ($device->{iothread}) {
4273         my $iothreads = vm_iothreads_list($vmid);
4274         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4275     }
4276 }
4277
4278 sub qemu_iothread_del {
4279     my($conf, $vmid, $deviceid) = @_;
4280
4281     my $device = parse_drive($deviceid, $conf->{$deviceid});
4282     if ($device->{iothread}) {
4283         my $iothreads = vm_iothreads_list($vmid);
4284         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4285     }
4286 }
4287
4288 sub qemu_objectadd {
4289     my($vmid, $objectid, $qomtype) = @_;
4290
4291     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4292
4293     return 1;
4294 }
4295
4296 sub qemu_objectdel {
4297     my($vmid, $objectid) = @_;
4298
4299     vm_mon_cmd($vmid, "object-del", id => $objectid);
4300
4301     return 1;
4302 }
4303
4304 sub qemu_driveadd {
4305     my ($storecfg, $vmid, $device) = @_;
4306
4307     my $drive = print_drive_full($storecfg, $vmid, $device);
4308     $drive =~ s/\\/\\\\/g;
4309     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4310
4311     # If the command succeeds qemu prints: "OK"
4312     return 1 if $ret =~ m/OK/s;
4313
4314     die "adding drive failed: $ret\n";
4315 }
4316
4317 sub qemu_drivedel {
4318     my($vmid, $deviceid) = @_;
4319
4320     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4321     $ret =~ s/^\s+//;
4322
4323     return 1 if $ret eq "";
4324
4325     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4326     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4327
4328     die "deleting drive $deviceid failed : $ret\n";
4329 }
4330
4331 sub qemu_deviceaddverify {
4332     my ($vmid, $deviceid) = @_;
4333
4334     for (my $i = 0; $i <= 5; $i++) {
4335          my $devices_list = vm_devices_list($vmid);
4336          return 1 if defined($devices_list->{$deviceid});
4337          sleep 1;
4338     }
4339
4340     die "error on hotplug device '$deviceid'\n";
4341 }
4342
4343
4344 sub qemu_devicedelverify {
4345     my ($vmid, $deviceid) = @_;
4346
4347     # need to verify that the device is correctly removed as device_del
4348     # is async and empty return is not reliable
4349
4350     for (my $i = 0; $i <= 5; $i++) {
4351          my $devices_list = vm_devices_list($vmid);
4352          return 1 if !defined($devices_list->{$deviceid});
4353          sleep 1;
4354     }
4355
4356     die "error on hot-unplugging device '$deviceid'\n";
4357 }
4358
4359 sub qemu_findorcreatescsihw {
4360     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4361
4362     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4363
4364     my $scsihwid="$controller_prefix$controller";
4365     my $devices_list = vm_devices_list($vmid);
4366
4367     if(!defined($devices_list->{$scsihwid})) {
4368         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4369     }
4370
4371     return 1;
4372 }
4373
4374 sub qemu_deletescsihw {
4375     my ($conf, $vmid, $opt) = @_;
4376
4377     my $device = parse_drive($opt, $conf->{$opt});
4378
4379     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4380         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4381         return 1;
4382     }
4383
4384     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4385
4386     my $devices_list = vm_devices_list($vmid);
4387     foreach my $opt (keys %{$devices_list}) {
4388         if (PVE::QemuServer::is_valid_drivename($opt)) {
4389             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4390             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4391                 return 1;
4392             }
4393         }
4394     }
4395
4396     my $scsihwid="scsihw$controller";
4397
4398     vm_deviceunplug($vmid, $conf, $scsihwid);
4399
4400     return 1;
4401 }
4402
4403 sub qemu_add_pci_bridge {
4404     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4405
4406     my $bridges = {};
4407
4408     my $bridgeid;
4409
4410     print_pci_addr($device, $bridges, $arch, $machine_type);
4411
4412     while (my ($k, $v) = each %$bridges) {
4413         $bridgeid = $k;
4414     }
4415     return 1 if !defined($bridgeid) || $bridgeid < 1;
4416
4417     my $bridge = "pci.$bridgeid";
4418     my $devices_list = vm_devices_list($vmid);
4419
4420     if (!defined($devices_list->{$bridge})) {
4421         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4422     }
4423
4424     return 1;
4425 }
4426
4427 sub qemu_set_link_status {
4428     my ($vmid, $device, $up) = @_;
4429
4430     vm_mon_cmd($vmid, "set_link", name => $device,
4431                up => $up ? JSON::true : JSON::false);
4432 }
4433
4434 sub qemu_netdevadd {
4435     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4436
4437     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4438     my %options =  split(/[=,]/, $netdev);
4439
4440     vm_mon_cmd($vmid, "netdev_add",  %options);
4441     return 1;
4442 }
4443
4444 sub qemu_netdevdel {
4445     my ($vmid, $deviceid) = @_;
4446
4447     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4448 }
4449
4450 sub qemu_usb_hotplug {
4451     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4452
4453     return if !$device;
4454
4455     # remove the old one first
4456     vm_deviceunplug($vmid, $conf, $deviceid);
4457
4458     # check if xhci controller is necessary and available
4459     if ($device->{usb3}) {
4460
4461         my $devicelist = vm_devices_list($vmid);
4462
4463         if (!$devicelist->{xhci}) {
4464             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4465             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4466         }
4467     }
4468     my $d = parse_usb_device($device->{host});
4469     $d->{usb3} = $device->{usb3};
4470
4471     # add the new one
4472     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4473 }
4474
4475 sub qemu_cpu_hotplug {
4476     my ($vmid, $conf, $vcpus) = @_;
4477
4478     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4479
4480     my $sockets = 1;
4481     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4482     $sockets = $conf->{sockets} if  $conf->{sockets};
4483     my $cores = $conf->{cores} || 1;
4484     my $maxcpus = $sockets * $cores;
4485
4486     $vcpus = $maxcpus if !$vcpus;
4487
4488     die "you can't add more vcpus than maxcpus\n"
4489         if $vcpus > $maxcpus;
4490
4491     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4492
4493     if ($vcpus < $currentvcpus) {
4494
4495         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4496
4497             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4498                 qemu_devicedel($vmid, "cpu$i");
4499                 my $retry = 0;
4500                 my $currentrunningvcpus = undef;
4501                 while (1) {
4502                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4503                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4504                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4505                     $retry++;
4506                     sleep 1;
4507                 }
4508                 #update conf after each succesfull cpu unplug
4509                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4510                 PVE::QemuConfig->write_config($vmid, $conf);
4511             }
4512         } else {
4513             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4514         }
4515
4516         return;
4517     }
4518
4519     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4520     die "vcpus in running vm does not match its configuration\n"
4521         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4522
4523     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4524
4525         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4526             my $cpustr = print_cpu_device($conf, $i);
4527             qemu_deviceadd($vmid, $cpustr);
4528
4529             my $retry = 0;
4530             my $currentrunningvcpus = undef;
4531             while (1) {
4532                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4533                 last if scalar(@{$currentrunningvcpus}) == $i;
4534                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4535                 sleep 1;
4536                 $retry++;
4537             }
4538             #update conf after each succesfull cpu hotplug
4539             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4540             PVE::QemuConfig->write_config($vmid, $conf);
4541         }
4542     } else {
4543
4544         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4545             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4546         }
4547     }
4548 }
4549
4550 sub qemu_block_set_io_throttle {
4551     my ($vmid, $deviceid,
4552         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4553         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4554         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4555         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4556
4557     return if !check_running($vmid) ;
4558
4559     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4560         bps => int($bps),
4561         bps_rd => int($bps_rd),
4562         bps_wr => int($bps_wr),
4563         iops => int($iops),
4564         iops_rd => int($iops_rd),
4565         iops_wr => int($iops_wr),
4566         bps_max => int($bps_max),
4567         bps_rd_max => int($bps_rd_max),
4568         bps_wr_max => int($bps_wr_max),
4569         iops_max => int($iops_max),
4570         iops_rd_max => int($iops_rd_max),
4571         iops_wr_max => int($iops_wr_max),
4572         bps_max_length => int($bps_max_length),
4573         bps_rd_max_length => int($bps_rd_max_length),
4574         bps_wr_max_length => int($bps_wr_max_length),
4575         iops_max_length => int($iops_max_length),
4576         iops_rd_max_length => int($iops_rd_max_length),
4577         iops_wr_max_length => int($iops_wr_max_length),
4578     );
4579
4580 }
4581
4582 # old code, only used to shutdown old VM after update
4583 sub __read_avail {
4584     my ($fh, $timeout) = @_;
4585
4586     my $sel = new IO::Select;
4587     $sel->add($fh);
4588
4589     my $res = '';
4590     my $buf;
4591
4592     my @ready;
4593     while (scalar (@ready = $sel->can_read($timeout))) {
4594         my $count;
4595         if ($count = $fh->sysread($buf, 8192)) {
4596             if ($buf =~ /^(.*)\(qemu\) $/s) {
4597                 $res .= $1;
4598                 last;
4599             } else {
4600                 $res .= $buf;
4601             }
4602         } else {
4603             if (!defined($count)) {
4604                 die "$!\n";
4605             }
4606             last;
4607         }
4608     }
4609
4610     die "monitor read timeout\n" if !scalar(@ready);
4611
4612     return $res;
4613 }
4614
4615 sub qemu_block_resize {
4616     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4617
4618     my $running = check_running($vmid);
4619
4620     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4621
4622     return if !$running;
4623
4624     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4625
4626 }
4627
4628 sub qemu_volume_snapshot {
4629     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4630
4631     my $running = check_running($vmid);
4632
4633     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4634         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4635     } else {
4636         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4637     }
4638 }
4639
4640 sub qemu_volume_snapshot_delete {
4641     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4642
4643     my $running = check_running($vmid);
4644
4645     if($running) {
4646
4647         $running = undef;
4648         my $conf = PVE::QemuConfig->load_config($vmid);
4649         foreach_drive($conf, sub {
4650             my ($ds, $drive) = @_;
4651             $running = 1 if $drive->{file} eq $volid;
4652         });
4653     }
4654
4655     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4656         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4657     } else {
4658         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4659     }
4660 }
4661
4662 sub set_migration_caps {
4663     my ($vmid) = @_;
4664
4665     my $cap_ref = [];
4666
4667     my $enabled_cap = {
4668         "auto-converge" => 1,
4669         "xbzrle" => 1,
4670         "x-rdma-pin-all" => 0,
4671         "zero-blocks" => 0,
4672         "compress" => 0
4673     };
4674
4675     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4676
4677     for my $supported_capability (@$supported_capabilities) {
4678         push @$cap_ref, {
4679             capability => $supported_capability->{capability},
4680             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4681         };
4682     }
4683
4684     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4685 }
4686
4687 my $fast_plug_option = {
4688     'lock' => 1,
4689     'name' => 1,
4690     'onboot' => 1,
4691     'shares' => 1,
4692     'startup' => 1,
4693     'description' => 1,
4694     'protection' => 1,
4695     'vmstatestorage' => 1,
4696     'hookscript' => 1,
4697 };
4698
4699 # hotplug changes in [PENDING]
4700 # $selection hash can be used to only apply specified options, for
4701 # example: { cores => 1 } (only apply changed 'cores')
4702 # $errors ref is used to return error messages
4703 sub vmconfig_hotplug_pending {
4704     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4705
4706     my $defaults = load_defaults();
4707     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4708
4709     # commit values which do not have any impact on running VM first
4710     # Note: those option cannot raise errors, we we do not care about
4711     # $selection and always apply them.
4712
4713     my $add_error = sub {
4714         my ($opt, $msg) = @_;
4715         $errors->{$opt} = "hotplug problem - $msg";
4716     };
4717
4718     my $changes = 0;
4719     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4720         if ($fast_plug_option->{$opt}) {
4721             $conf->{$opt} = $conf->{pending}->{$opt};
4722             delete $conf->{pending}->{$opt};
4723             $changes = 1;
4724         }
4725     }
4726
4727     if ($changes) {
4728         PVE::QemuConfig->write_config($vmid, $conf);
4729         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4730     }
4731
4732     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4733
4734     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4735     while (my ($opt, $force) = each %$pending_delete_hash) {
4736         next if $selection && !$selection->{$opt};
4737         eval {
4738             if ($opt eq 'hotplug') {
4739                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4740             } elsif ($opt eq 'tablet') {
4741                 die "skip\n" if !$hotplug_features->{usb};
4742                 if ($defaults->{tablet}) {
4743                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4744                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4745                         if $arch eq 'aarch64';
4746                 } else {
4747                     vm_deviceunplug($vmid, $conf, 'tablet');
4748                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4749                 }
4750             } elsif ($opt =~ m/^usb\d+/) {
4751                 die "skip\n";
4752                 # since we cannot reliably hot unplug usb devices
4753                 # we are disabling it
4754                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4755                 vm_deviceunplug($vmid, $conf, $opt);
4756             } elsif ($opt eq 'vcpus') {
4757                 die "skip\n" if !$hotplug_features->{cpu};
4758                 qemu_cpu_hotplug($vmid, $conf, undef);
4759             } elsif ($opt eq 'balloon') {
4760                 # enable balloon device is not hotpluggable
4761                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4762                 # here we reset the ballooning value to memory
4763                 my $balloon = $conf->{memory} || $defaults->{memory};
4764                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4765             } elsif ($fast_plug_option->{$opt}) {
4766                 # do nothing
4767             } elsif ($opt =~ m/^net(\d+)$/) {
4768                 die "skip\n" if !$hotplug_features->{network};
4769                 vm_deviceunplug($vmid, $conf, $opt);
4770             } elsif (is_valid_drivename($opt)) {
4771                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4772                 vm_deviceunplug($vmid, $conf, $opt);
4773                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4774             } elsif ($opt =~ m/^memory$/) {
4775                 die "skip\n" if !$hotplug_features->{memory};
4776                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4777             } elsif ($opt eq 'cpuunits') {
4778                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4779             } elsif ($opt eq 'cpulimit') {
4780                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4781             } else {
4782                 die "skip\n";
4783             }
4784         };
4785         if (my $err = $@) {
4786             &$add_error($opt, $err) if $err ne "skip\n";
4787         } else {
4788             # save new config if hotplug was successful
4789             delete $conf->{$opt};
4790             vmconfig_undelete_pending_option($conf, $opt);
4791             PVE::QemuConfig->write_config($vmid, $conf);
4792             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4793         }
4794     }
4795
4796     my $apply_pending_cloudinit;
4797     $apply_pending_cloudinit = sub {
4798         my ($key, $value) = @_;
4799         $apply_pending_cloudinit = sub {}; # once is enough
4800
4801         my @cloudinit_opts = keys %$confdesc_cloudinit;
4802         foreach my $opt (keys %{$conf->{pending}}) {
4803             next if !grep { $_ eq $opt } @cloudinit_opts;
4804             $conf->{$opt} = delete $conf->{pending}->{$opt};
4805         }
4806
4807         my $new_conf = { %$conf };
4808         $new_conf->{$key} = $value;
4809         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4810     };
4811
4812     foreach my $opt (keys %{$conf->{pending}}) {
4813         next if $selection && !$selection->{$opt};
4814         my $value = $conf->{pending}->{$opt};
4815         eval {
4816             if ($opt eq 'hotplug') {
4817                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4818             } elsif ($opt eq 'tablet') {
4819                 die "skip\n" if !$hotplug_features->{usb};
4820                 if ($value == 1) {
4821                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4822                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4823                         if $arch eq 'aarch64';
4824                 } elsif ($value == 0) {
4825                     vm_deviceunplug($vmid, $conf, 'tablet');
4826                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4827                 }
4828             } elsif ($opt =~ m/^usb\d+$/) {
4829                 die "skip\n";
4830                 # since we cannot reliably hot unplug usb devices
4831                 # we are disabling it
4832                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4833                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4834                 die "skip\n" if !$d;
4835                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4836             } elsif ($opt eq 'vcpus') {
4837                 die "skip\n" if !$hotplug_features->{cpu};
4838                 qemu_cpu_hotplug($vmid, $conf, $value);
4839             } elsif ($opt eq 'balloon') {
4840                 # enable/disable balloning device is not hotpluggable
4841                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4842                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4843                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4844
4845                 # allow manual ballooning if shares is set to zero
4846                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4847                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4848                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4849                 }
4850             } elsif ($opt =~ m/^net(\d+)$/) {
4851                 # some changes can be done without hotplug
4852                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4853                                     $vmid, $opt, $value, $arch, $machine_type);
4854             } elsif (is_valid_drivename($opt)) {
4855                 # some changes can be done without hotplug
4856                 my $drive = parse_drive($opt, $value);
4857                 if (drive_is_cloudinit($drive)) {
4858                     &$apply_pending_cloudinit($opt, $value);
4859                 }
4860                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4861                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4862             } elsif ($opt =~ m/^memory$/) { #dimms
4863                 die "skip\n" if !$hotplug_features->{memory};
4864                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4865             } elsif ($opt eq 'cpuunits') {
4866                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4867             } elsif ($opt eq 'cpulimit') {
4868                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4869                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4870             } else {
4871                 die "skip\n";  # skip non-hot-pluggable options
4872             }
4873         };
4874         if (my $err = $@) {
4875             &$add_error($opt, $err) if $err ne "skip\n";
4876         } else {
4877             # save new config if hotplug was successful
4878             $conf->{$opt} = $value;
4879             delete $conf->{pending}->{$opt};
4880             PVE::QemuConfig->write_config($vmid, $conf);
4881             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4882         }
4883     }
4884 }
4885
4886 sub try_deallocate_drive {
4887     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4888
4889     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4890         my $volid = $drive->{file};
4891         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4892             my $sid = PVE::Storage::parse_volume_id($volid);
4893             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4894
4895             # check if the disk is really unused
4896             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4897                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4898             PVE::Storage::vdisk_free($storecfg, $volid);
4899             return 1;
4900         } else {
4901             # If vm is not owner of this disk remove from config
4902             return 1;
4903         }
4904     }
4905
4906     return undef;
4907 }
4908
4909 sub vmconfig_delete_or_detach_drive {
4910     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4911
4912     my $drive = parse_drive($opt, $conf->{$opt});
4913
4914     my $rpcenv = PVE::RPCEnvironment::get();
4915     my $authuser = $rpcenv->get_user();
4916
4917     if ($force) {
4918         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4919         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4920     } else {
4921         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4922     }
4923 }
4924
4925 sub vmconfig_apply_pending {
4926     my ($vmid, $conf, $storecfg) = @_;
4927
4928     # cold plug
4929
4930     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4931     while (my ($opt, $force) = each %$pending_delete_hash) {
4932         die "internal error" if $opt =~ m/^unused/;
4933         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4934         if (!defined($conf->{$opt})) {
4935             vmconfig_undelete_pending_option($conf, $opt);
4936             PVE::QemuConfig->write_config($vmid, $conf);
4937         } elsif (is_valid_drivename($opt)) {
4938             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4939             vmconfig_undelete_pending_option($conf, $opt);
4940             delete $conf->{$opt};
4941             PVE::QemuConfig->write_config($vmid, $conf);
4942         } else {
4943             vmconfig_undelete_pending_option($conf, $opt);
4944             delete $conf->{$opt};
4945             PVE::QemuConfig->write_config($vmid, $conf);
4946         }
4947     }
4948
4949     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4950
4951     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4952         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4953
4954         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4955             # skip if nothing changed
4956         } elsif (is_valid_drivename($opt)) {
4957             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4958                 if defined($conf->{$opt});
4959             $conf->{$opt} = $conf->{pending}->{$opt};
4960         } else {
4961             $conf->{$opt} = $conf->{pending}->{$opt};
4962         }
4963
4964         delete $conf->{pending}->{$opt};
4965         PVE::QemuConfig->write_config($vmid, $conf);
4966     }
4967 }
4968
4969 my $safe_num_ne = sub {
4970     my ($a, $b) = @_;
4971
4972     return 0 if !defined($a) && !defined($b);
4973     return 1 if !defined($a);
4974     return 1 if !defined($b);
4975
4976     return $a != $b;
4977 };
4978
4979 my $safe_string_ne = sub {
4980     my ($a, $b) = @_;
4981
4982     return 0 if !defined($a) && !defined($b);
4983     return 1 if !defined($a);
4984     return 1 if !defined($b);
4985
4986     return $a ne $b;
4987 };
4988
4989 sub vmconfig_update_net {
4990     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4991
4992     my $newnet = parse_net($value);
4993
4994     if ($conf->{$opt}) {
4995         my $oldnet = parse_net($conf->{$opt});
4996
4997         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4998             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4999             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5000             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5001
5002             # for non online change, we try to hot-unplug
5003             die "skip\n" if !$hotplug;
5004             vm_deviceunplug($vmid, $conf, $opt);
5005         } else {
5006
5007             die "internal error" if $opt !~ m/net(\d+)/;
5008             my $iface = "tap${vmid}i$1";
5009
5010             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5011                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5012                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5013                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5014                 PVE::Network::tap_unplug($iface);
5015                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5016             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5017                 # Rate can be applied on its own but any change above needs to
5018                 # include the rate in tap_plug since OVS resets everything.
5019                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5020             }
5021
5022             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5023                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5024             }
5025
5026             return 1;
5027         }
5028     }
5029
5030     if ($hotplug) {
5031         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5032     } else {
5033         die "skip\n";
5034     }
5035 }
5036
5037 sub vmconfig_update_disk {
5038     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5039
5040     # fixme: do we need force?
5041
5042     my $drive = parse_drive($opt, $value);
5043
5044     if ($conf->{$opt}) {
5045
5046         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5047
5048             my $media = $drive->{media} || 'disk';
5049             my $oldmedia = $old_drive->{media} || 'disk';
5050             die "unable to change media type\n" if $media ne $oldmedia;
5051
5052             if (!drive_is_cdrom($old_drive)) {
5053
5054                 if ($drive->{file} ne $old_drive->{file}) {
5055
5056                     die "skip\n" if !$hotplug;
5057
5058                     # unplug and register as unused
5059                     vm_deviceunplug($vmid, $conf, $opt);
5060                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5061
5062                 } else {
5063                     # update existing disk
5064
5065                     # skip non hotpluggable value
5066                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5067                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5068                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5069                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5070                         die "skip\n";
5071                     }
5072
5073                     # apply throttle
5074                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5075                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5076                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5077                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5078                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5079                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5080                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5081                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5082                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5083                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5084                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5085                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5086                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5087                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5088                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5089                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5090                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5091                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5092
5093                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5094                                                    ($drive->{mbps} || 0)*1024*1024,
5095                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5096                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5097                                                    $drive->{iops} || 0,
5098                                                    $drive->{iops_rd} || 0,
5099                                                    $drive->{iops_wr} || 0,
5100                                                    ($drive->{mbps_max} || 0)*1024*1024,
5101                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5102                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5103                                                    $drive->{iops_max} || 0,
5104                                                    $drive->{iops_rd_max} || 0,
5105                                                    $drive->{iops_wr_max} || 0,
5106                                                    $drive->{bps_max_length} || 1,
5107                                                    $drive->{bps_rd_max_length} || 1,
5108                                                    $drive->{bps_wr_max_length} || 1,
5109                                                    $drive->{iops_max_length} || 1,
5110                                                    $drive->{iops_rd_max_length} || 1,
5111                                                    $drive->{iops_wr_max_length} || 1);
5112
5113                     }
5114
5115                     return 1;
5116                 }
5117
5118             } else { # cdrom
5119
5120                 if ($drive->{file} eq 'none') {
5121                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5122                     if (drive_is_cloudinit($old_drive)) {
5123                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5124                     }
5125                 } else {
5126                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5127                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5128                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5129                 }
5130
5131                 return 1;
5132             }
5133         }
5134     }
5135
5136     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5137     # hotplug new disks
5138     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5139     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5140 }
5141
5142 sub vm_start {
5143     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5144         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5145
5146     PVE::QemuConfig->lock_config($vmid, sub {
5147         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5148
5149         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5150
5151         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5152
5153         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5154
5155         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5156             vmconfig_apply_pending($vmid, $conf, $storecfg);
5157             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5158         }
5159
5160         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5161
5162         my $defaults = load_defaults();
5163
5164         # set environment variable useful inside network script
5165         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5166
5167         my $local_volumes = {};
5168
5169         if ($targetstorage) {
5170             foreach_drive($conf, sub {
5171                 my ($ds, $drive) = @_;
5172
5173                 return if drive_is_cdrom($drive);
5174
5175                 my $volid = $drive->{file};
5176
5177                 return if !$volid;
5178
5179                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5180
5181                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5182                 return if $scfg->{shared};
5183                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5184             });
5185
5186             my $format = undef;
5187
5188             foreach my $opt (sort keys %$local_volumes) {
5189
5190                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5191                 my $drive = parse_drive($opt, $conf->{$opt});
5192
5193                 #if remote storage is specified, use default format
5194                 if ($targetstorage && $targetstorage ne "1") {
5195                     $storeid = $targetstorage;
5196                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5197                     $format = $defFormat;
5198                 } else {
5199                     #else we use same format than original
5200                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5201                     $format = qemu_img_format($scfg, $volid);
5202                 }
5203
5204                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5205                 my $newdrive = $drive;
5206                 $newdrive->{format} = $format;
5207                 $newdrive->{file} = $newvolid;
5208                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5209                 $local_volumes->{$opt} = $drivestr;
5210                 #pass drive to conf for command line
5211                 $conf->{$opt} = $drivestr;
5212             }
5213         }
5214
5215         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5216
5217         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5218
5219         my $migrate_port = 0;
5220         my $migrate_uri;
5221         if ($statefile) {
5222             if ($statefile eq 'tcp') {
5223                 my $localip = "localhost";
5224                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5225                 my $nodename = PVE::INotify::nodename();
5226
5227                 if (!defined($migration_type)) {
5228                     if (defined($datacenterconf->{migration}->{type})) {
5229                         $migration_type = $datacenterconf->{migration}->{type};
5230                     } else {
5231                         $migration_type = 'secure';
5232                     }
5233                 }
5234
5235                 if ($migration_type eq 'insecure') {
5236                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5237                     if ($migrate_network_addr) {
5238                         $localip = $migrate_network_addr;
5239                     } else {
5240                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5241                     }
5242
5243                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5244                 }
5245
5246                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5247                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5248                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5249                 push @$cmd, '-incoming', $migrate_uri;
5250                 push @$cmd, '-S';
5251
5252             } elsif ($statefile eq 'unix') {
5253                 # should be default for secure migrations as a ssh TCP forward
5254                 # tunnel is not deterministic reliable ready and fails regurarly
5255                 # to set up in time, so use UNIX socket forwards
5256                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5257                 unlink $socket_addr;
5258
5259                 $migrate_uri = "unix:$socket_addr";
5260
5261                 push @$cmd, '-incoming', $migrate_uri;
5262                 push @$cmd, '-S';
5263
5264             } else {
5265                 push @$cmd, '-loadstate', $statefile;
5266             }
5267         } elsif ($paused) {
5268             push @$cmd, '-S';
5269         }
5270
5271         # host pci devices
5272         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5273           my $d = parse_hostpci($conf->{"hostpci$i"});
5274           next if !$d;
5275           my $pcidevices = $d->{pciid};
5276           foreach my $pcidevice (@$pcidevices) {
5277                 my $pciid = $pcidevice->{id};
5278
5279                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5280                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5281                 die "no pci device info for device '$pciid'\n" if !$info;
5282
5283                 if ($d->{mdev}) {
5284                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5285                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5286                 } else {
5287                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5288                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5289                     die "can't reset pci device '$pciid'\n"
5290                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5291                 }
5292           }
5293         }
5294
5295         PVE::Storage::activate_volumes($storecfg, $vollist);
5296
5297         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5298             eval {
5299                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5300                     outfunc => sub {}, errfunc => sub {});
5301             };
5302         }
5303
5304         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5305                                                   : $defaults->{cpuunits};
5306
5307         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5308         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5309
5310         my %properties = (
5311             Slice => 'qemu.slice',
5312             KillMode => 'none',
5313             CPUShares => $cpuunits
5314         );
5315
5316         if (my $cpulimit = $conf->{cpulimit}) {
5317             $properties{CPUQuota} = int($cpulimit * 100);
5318         }
5319         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5320
5321         my $run_qemu = sub {
5322             PVE::Tools::run_fork sub {
5323                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5324                 run_command($cmd, %run_params);
5325             };
5326         };
5327
5328         if ($conf->{hugepages}) {
5329
5330             my $code = sub {
5331                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5332                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5333
5334                 PVE::QemuServer::Memory::hugepages_mount();
5335                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5336
5337                 eval { $run_qemu->() };
5338                 if (my $err = $@) {
5339                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5340                     die $err;
5341                 }
5342
5343                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5344             };
5345             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5346
5347         } else {
5348             eval { $run_qemu->() };
5349         }
5350
5351         if (my $err = $@) {
5352             # deactivate volumes if start fails
5353             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5354             die "start failed: $err";
5355         }
5356
5357         print "migration listens on $migrate_uri\n" if $migrate_uri;
5358
5359         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5360             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5361             warn $@ if $@;
5362         }
5363
5364         #start nbd server for storage migration
5365         if ($targetstorage) {
5366             my $nodename = PVE::INotify::nodename();
5367             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5368             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5369             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5370             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5371
5372             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5373
5374             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5375
5376             foreach my $opt (sort keys %$local_volumes) {
5377                 my $volid = $local_volumes->{$opt};
5378                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5379                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5380                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5381             }
5382         }
5383
5384         if ($migratedfrom) {
5385             eval {
5386                 set_migration_caps($vmid);
5387             };
5388             warn $@ if $@;
5389
5390             if ($spice_port) {
5391                 print "spice listens on port $spice_port\n";
5392                 if ($spice_ticket) {
5393                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5394                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5395                 }
5396             }
5397
5398         } else {
5399             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5400                 if !$statefile && $conf->{balloon};
5401
5402             foreach my $opt (keys %$conf) {
5403                 next if $opt !~  m/^net\d+$/;
5404                 my $nicconf = parse_net($conf->{$opt});
5405                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5406             }
5407         }
5408
5409         vm_mon_cmd_nocheck($vmid, 'qom-set',
5410                     path => "machine/peripheral/balloon0",
5411                     property => "guest-stats-polling-interval",
5412                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5413
5414         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5415     });
5416 }
5417
5418 sub vm_mon_cmd {
5419     my ($vmid, $execute, %params) = @_;
5420
5421     my $cmd = { execute => $execute, arguments => \%params };
5422     vm_qmp_command($vmid, $cmd);
5423 }
5424
5425 sub vm_mon_cmd_nocheck {
5426     my ($vmid, $execute, %params) = @_;
5427
5428     my $cmd = { execute => $execute, arguments => \%params };
5429     vm_qmp_command($vmid, $cmd, 1);
5430 }
5431
5432 sub vm_qmp_command {
5433     my ($vmid, $cmd, $nocheck) = @_;
5434
5435     my $res;
5436
5437     my $timeout;
5438     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5439         $timeout = $cmd->{arguments}->{timeout};
5440         delete $cmd->{arguments}->{timeout};
5441     }
5442
5443     eval {
5444         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5445         my $sname = qmp_socket($vmid);
5446         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5447             my $qmpclient = PVE::QMPClient->new();
5448
5449             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5450         } else {
5451             die "unable to open monitor socket\n";
5452         }
5453     };
5454     if (my $err = $@) {
5455         syslog("err", "VM $vmid qmp command failed - $err");
5456         die $err;
5457     }
5458
5459     return $res;
5460 }
5461
5462 sub vm_human_monitor_command {
5463     my ($vmid, $cmdline) = @_;
5464
5465     my $res;
5466
5467     my $cmd = {
5468         execute => 'human-monitor-command',
5469         arguments => { 'command-line' => $cmdline},
5470     };
5471
5472     return vm_qmp_command($vmid, $cmd);
5473 }
5474
5475 sub vm_commandline {
5476     my ($storecfg, $vmid, $snapname) = @_;
5477
5478     my $conf = PVE::QemuConfig->load_config($vmid);
5479
5480     if ($snapname) {
5481         my $snapshot = $conf->{snapshots}->{$snapname};
5482         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5483
5484         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5485
5486         $conf = $snapshot;
5487     }
5488
5489     my $defaults = load_defaults();
5490
5491     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5492
5493     return PVE::Tools::cmd2string($cmd);
5494 }
5495
5496 sub vm_reset {
5497     my ($vmid, $skiplock) = @_;
5498
5499     PVE::QemuConfig->lock_config($vmid, sub {
5500
5501         my $conf = PVE::QemuConfig->load_config($vmid);
5502
5503         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5504
5505         vm_mon_cmd($vmid, "system_reset");
5506     });
5507 }
5508
5509 sub get_vm_volumes {
5510     my ($conf) = @_;
5511
5512     my $vollist = [];
5513     foreach_volid($conf, sub {
5514         my ($volid, $attr) = @_;
5515
5516         return if $volid =~ m|^/|;
5517
5518         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5519         return if !$sid;
5520
5521         push @$vollist, $volid;
5522     });
5523
5524     return $vollist;
5525 }
5526
5527 sub vm_stop_cleanup {
5528     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5529
5530     eval {
5531
5532         if (!$keepActive) {
5533             my $vollist = get_vm_volumes($conf);
5534             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5535         }
5536
5537         foreach my $ext (qw(mon qmp pid vnc qga)) {
5538             unlink "/var/run/qemu-server/${vmid}.$ext";
5539         }
5540
5541         if ($conf->{ivshmem}) {
5542             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5543             # just delete it for now, VMs which have this already open do not
5544             # are affected, but new VMs will get a separated one. If this
5545             # becomes an issue we either add some sort of ref-counting or just
5546             # add a "don't delete on stop" flag to the ivshmem format.
5547             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5548         }
5549
5550         foreach my $key (keys %$conf) {
5551             next if $key !~ m/^hostpci(\d+)$/;
5552             my $hostpciindex = $1;
5553             my $d = parse_hostpci($conf->{$key});
5554             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5555
5556             foreach my $pci (@{$d->{pciid}}) {
5557                 my $pciid = $pci->{id};
5558                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5559             }
5560         }
5561
5562         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5563     };
5564     warn $@ if $@; # avoid errors - just warn
5565 }
5566
5567 # Note: use $nockeck to skip tests if VM configuration file exists.
5568 # We need that when migration VMs to other nodes (files already moved)
5569 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5570 sub vm_stop {
5571     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5572
5573     $force = 1 if !defined($force) && !$shutdown;
5574
5575     if ($migratedfrom){
5576         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5577         kill 15, $pid if $pid;
5578         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5579         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5580         return;
5581     }
5582
5583     PVE::QemuConfig->lock_config($vmid, sub {
5584
5585         my $pid = check_running($vmid, $nocheck);
5586         return if !$pid;
5587
5588         my $conf;
5589         if (!$nocheck) {
5590             $conf = PVE::QemuConfig->load_config($vmid);
5591             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5592             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5593                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5594                 $timeout = $opts->{down} if $opts->{down};
5595             }
5596             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5597         }
5598
5599         $timeout = 60 if !defined($timeout);
5600
5601         eval {
5602             if ($shutdown) {
5603                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5604                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5605                 } else {
5606                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5607                 }
5608             } else {
5609                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5610             }
5611         };
5612         my $err = $@;
5613
5614         if (!$err) {
5615             my $count = 0;
5616             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5617                 $count++;
5618                 sleep 1;
5619             }
5620
5621             if ($count >= $timeout) {
5622                 if ($force) {
5623                     warn "VM still running - terminating now with SIGTERM\n";
5624                     kill 15, $pid;
5625                 } else {
5626                     die "VM quit/powerdown failed - got timeout\n";
5627                 }
5628             } else {
5629                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5630                 return;
5631             }
5632         } else {
5633             if ($force) {
5634                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5635                 kill 15, $pid;
5636             } else {
5637                 die "VM quit/powerdown failed\n";
5638             }
5639         }
5640
5641         # wait again
5642         $timeout = 10;
5643
5644         my $count = 0;
5645         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5646             $count++;
5647             sleep 1;
5648         }
5649
5650         if ($count >= $timeout) {
5651             warn "VM still running - terminating now with SIGKILL\n";
5652             kill 9, $pid;
5653             sleep 1;
5654         }
5655
5656         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5657    });
5658 }
5659
5660 sub vm_suspend {
5661     my ($vmid, $skiplock) = @_;
5662
5663     PVE::QemuConfig->lock_config($vmid, sub {
5664
5665         my $conf = PVE::QemuConfig->load_config($vmid);
5666
5667         PVE::QemuConfig->check_lock($conf)
5668             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5669
5670         vm_mon_cmd($vmid, "stop");
5671     });
5672 }
5673
5674 sub vm_resume {
5675     my ($vmid, $skiplock, $nocheck) = @_;
5676
5677     PVE::QemuConfig->lock_config($vmid, sub {
5678
5679         my $res = vm_mon_cmd($vmid, 'query-status');
5680         my $resume_cmd = 'cont';
5681
5682         if ($res->{status} && $res->{status} eq 'suspended') {
5683             $resume_cmd = 'system_wakeup';
5684         }
5685
5686         if (!$nocheck) {
5687
5688             my $conf = PVE::QemuConfig->load_config($vmid);
5689
5690             PVE::QemuConfig->check_lock($conf)
5691                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5692
5693             vm_mon_cmd($vmid, $resume_cmd);
5694
5695         } else {
5696             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5697         }
5698     });
5699 }
5700
5701 sub vm_sendkey {
5702     my ($vmid, $skiplock, $key) = @_;
5703
5704     PVE::QemuConfig->lock_config($vmid, sub {
5705
5706         my $conf = PVE::QemuConfig->load_config($vmid);
5707
5708         # there is no qmp command, so we use the human monitor command
5709         vm_human_monitor_command($vmid, "sendkey $key");
5710     });
5711 }
5712
5713 sub vm_destroy {
5714     my ($storecfg, $vmid, $skiplock) = @_;
5715
5716     PVE::QemuConfig->lock_config($vmid, sub {
5717
5718         my $conf = PVE::QemuConfig->load_config($vmid);
5719
5720         if (!check_running($vmid)) {
5721             destroy_vm($storecfg, $vmid, undef, $skiplock);
5722         } else {
5723             die "VM $vmid is running - destroy failed\n";
5724         }
5725     });
5726 }
5727
5728 # vzdump restore implementaion
5729
5730 sub tar_archive_read_firstfile {
5731     my $archive = shift;
5732
5733     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5734
5735     # try to detect archive type first
5736     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5737         die "unable to open file '$archive'\n";
5738     my $firstfile = <$fh>;
5739     kill 15, $pid;
5740     close $fh;
5741
5742     die "ERROR: archive contaions no data\n" if !$firstfile;
5743     chomp $firstfile;
5744
5745     return $firstfile;
5746 }
5747
5748 sub tar_restore_cleanup {
5749     my ($storecfg, $statfile) = @_;
5750
5751     print STDERR "starting cleanup\n";
5752
5753     if (my $fd = IO::File->new($statfile, "r")) {
5754         while (defined(my $line = <$fd>)) {
5755             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5756                 my $volid = $2;
5757                 eval {
5758                     if ($volid =~ m|^/|) {
5759                         unlink $volid || die 'unlink failed\n';
5760                     } else {
5761                         PVE::Storage::vdisk_free($storecfg, $volid);
5762                     }
5763                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5764                 };
5765                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5766             } else {
5767                 print STDERR "unable to parse line in statfile - $line";
5768             }
5769         }
5770         $fd->close();
5771     }
5772 }
5773
5774 sub restore_archive {
5775     my ($archive, $vmid, $user, $opts) = @_;
5776
5777     my $format = $opts->{format};
5778     my $comp;
5779
5780     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5781         $format = 'tar' if !$format;
5782         $comp = 'gzip';
5783     } elsif ($archive =~ m/\.tar$/) {
5784         $format = 'tar' if !$format;
5785     } elsif ($archive =~ m/.tar.lzo$/) {
5786         $format = 'tar' if !$format;
5787         $comp = 'lzop';
5788     } elsif ($archive =~ m/\.vma$/) {
5789         $format = 'vma' if !$format;
5790     } elsif ($archive =~ m/\.vma\.gz$/) {
5791         $format = 'vma' if !$format;
5792         $comp = 'gzip';
5793     } elsif ($archive =~ m/\.vma\.lzo$/) {
5794         $format = 'vma' if !$format;
5795         $comp = 'lzop';
5796     } else {
5797         $format = 'vma' if !$format; # default
5798     }
5799
5800     # try to detect archive format
5801     if ($format eq 'tar') {
5802         return restore_tar_archive($archive, $vmid, $user, $opts);
5803     } else {
5804         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5805     }
5806 }
5807
5808 sub restore_update_config_line {
5809     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5810
5811     return if $line =~ m/^\#qmdump\#/;
5812     return if $line =~ m/^\#vzdump\#/;
5813     return if $line =~ m/^lock:/;
5814     return if $line =~ m/^unused\d+:/;
5815     return if $line =~ m/^parent:/;
5816     return if $line =~ m/^template:/; # restored VM is never a template
5817
5818     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5819     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5820         # try to convert old 1.X settings
5821         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5822         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5823             my ($model, $macaddr) = split(/\=/, $devconfig);
5824             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5825             my $net = {
5826                 model => $model,
5827                 bridge => "vmbr$ind",
5828                 macaddr => $macaddr,
5829             };
5830             my $netstr = print_net($net);
5831
5832             print $outfd "net$cookie->{netcount}: $netstr\n";
5833             $cookie->{netcount}++;
5834         }
5835     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5836         my ($id, $netstr) = ($1, $2);
5837         my $net = parse_net($netstr);
5838         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5839         $netstr = print_net($net);
5840         print $outfd "$id: $netstr\n";
5841     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5842         my $virtdev = $1;
5843         my $value = $3;
5844         my $di = parse_drive($virtdev, $value);
5845         if (defined($di->{backup}) && !$di->{backup}) {
5846             print $outfd "#$line";
5847         } elsif ($map->{$virtdev}) {
5848             delete $di->{format}; # format can change on restore
5849             $di->{file} = $map->{$virtdev};
5850             $value = print_drive($vmid, $di);
5851             print $outfd "$virtdev: $value\n";
5852         } else {
5853             print $outfd $line;
5854         }
5855     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5856         my $vmgenid = $1;
5857         if ($vmgenid ne '0') {
5858             # always generate a new vmgenid if there was a valid one setup
5859             $vmgenid = generate_uuid();
5860         }
5861         print $outfd "vmgenid: $vmgenid\n";
5862     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5863         my ($uuid, $uuid_str);
5864         UUID::generate($uuid);
5865         UUID::unparse($uuid, $uuid_str);
5866         my $smbios1 = parse_smbios1($2);
5867         $smbios1->{uuid} = $uuid_str;
5868         print $outfd $1.print_smbios1($smbios1)."\n";
5869     } else {
5870         print $outfd $line;
5871     }
5872 }
5873
5874 sub scan_volids {
5875     my ($cfg, $vmid) = @_;
5876
5877     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5878
5879     my $volid_hash = {};
5880     foreach my $storeid (keys %$info) {
5881         foreach my $item (@{$info->{$storeid}}) {
5882             next if !($item->{volid} && $item->{size});
5883             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5884             $volid_hash->{$item->{volid}} = $item;
5885         }
5886     }
5887
5888     return $volid_hash;
5889 }
5890
5891 sub is_volume_in_use {
5892     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5893
5894     my $path = PVE::Storage::path($storecfg, $volid);
5895
5896     my $scan_config = sub {
5897         my ($cref, $snapname) = @_;
5898
5899         foreach my $key (keys %$cref) {
5900             my $value = $cref->{$key};
5901             if (is_valid_drivename($key)) {
5902                 next if $skip_drive && $key eq $skip_drive;
5903                 my $drive = parse_drive($key, $value);
5904                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5905                 return 1 if $volid eq $drive->{file};
5906                 if ($drive->{file} =~ m!^/!) {
5907                     return 1 if $drive->{file} eq $path;
5908                 } else {
5909                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5910                     next if !$storeid;
5911                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5912                     next if !$scfg;
5913                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5914                 }
5915             }
5916         }
5917
5918         return 0;
5919     };
5920
5921     return 1 if &$scan_config($conf);
5922
5923     undef $skip_drive;
5924
5925     foreach my $snapname (keys %{$conf->{snapshots}}) {
5926         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5927     }
5928
5929     return 0;
5930 }
5931
5932 sub update_disksize {
5933     my ($vmid, $conf, $volid_hash) = @_;
5934
5935     my $changes;
5936     my $prefix = "VM $vmid:";
5937
5938     # used and unused disks
5939     my $referenced = {};
5940
5941     # Note: it is allowed to define multiple storages with same path (alias), so
5942     # we need to check both 'volid' and real 'path' (two different volid can point
5943     # to the same path).
5944
5945     my $referencedpath = {};
5946
5947     # update size info
5948     foreach my $opt (keys %$conf) {
5949         if (is_valid_drivename($opt)) {
5950             my $drive = parse_drive($opt, $conf->{$opt});
5951             my $volid = $drive->{file};
5952             next if !$volid;
5953
5954             $referenced->{$volid} = 1;
5955             if ($volid_hash->{$volid} &&
5956                 (my $path = $volid_hash->{$volid}->{path})) {
5957                 $referencedpath->{$path} = 1;
5958             }
5959
5960             next if drive_is_cdrom($drive);
5961             next if !$volid_hash->{$volid};
5962
5963             $drive->{size} = $volid_hash->{$volid}->{size};
5964             my $new = print_drive($vmid, $drive);
5965             if ($new ne $conf->{$opt}) {
5966                 $changes = 1;
5967                 $conf->{$opt} = $new;
5968                 print "$prefix update disk '$opt' information.\n";
5969             }
5970         }
5971     }
5972
5973     # remove 'unusedX' entry if volume is used
5974     foreach my $opt (keys %$conf) {
5975         next if $opt !~ m/^unused\d+$/;
5976         my $volid = $conf->{$opt};
5977         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5978         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5979             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5980             $changes = 1;
5981             delete $conf->{$opt};
5982         }
5983
5984         $referenced->{$volid} = 1;
5985         $referencedpath->{$path} = 1 if $path;
5986     }
5987
5988     foreach my $volid (sort keys %$volid_hash) {
5989         next if $volid =~ m/vm-$vmid-state-/;
5990         next if $referenced->{$volid};
5991         my $path = $volid_hash->{$volid}->{path};
5992         next if !$path; # just to be sure
5993         next if $referencedpath->{$path};
5994         $changes = 1;
5995         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5996         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5997         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5998     }
5999
6000     return $changes;
6001 }
6002
6003 sub rescan {
6004     my ($vmid, $nolock, $dryrun) = @_;
6005
6006     my $cfg = PVE::Storage::config();
6007
6008     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6009     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6010     foreach my $stor (keys %{$cfg->{ids}}) {
6011         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6012     }
6013
6014     print "rescan volumes...\n";
6015     my $volid_hash = scan_volids($cfg, $vmid);
6016
6017     my $updatefn =  sub {
6018         my ($vmid) = @_;
6019
6020         my $conf = PVE::QemuConfig->load_config($vmid);
6021
6022         PVE::QemuConfig->check_lock($conf);
6023
6024         my $vm_volids = {};
6025         foreach my $volid (keys %$volid_hash) {
6026             my $info = $volid_hash->{$volid};
6027             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6028         }
6029
6030         my $changes = update_disksize($vmid, $conf, $vm_volids);
6031
6032         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6033     };
6034
6035     if (defined($vmid)) {
6036         if ($nolock) {
6037             &$updatefn($vmid);
6038         } else {
6039             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6040         }
6041     } else {
6042         my $vmlist = config_list();
6043         foreach my $vmid (keys %$vmlist) {
6044             if ($nolock) {
6045                 &$updatefn($vmid);
6046             } else {
6047                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6048             }
6049         }
6050     }
6051 }
6052
6053 sub restore_vma_archive {
6054     my ($archive, $vmid, $user, $opts, $comp) = @_;
6055
6056     my $readfrom = $archive;
6057
6058     my $cfg = PVE::Storage::config();
6059     my $commands = [];
6060     my $bwlimit = $opts->{bwlimit};
6061
6062     my $dbg_cmdstring = '';
6063     my $add_pipe = sub {
6064         my ($cmd) = @_;
6065         push @$commands, $cmd;
6066         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6067         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6068         $readfrom = '-';
6069     };
6070
6071     my $input = undef;
6072     if ($archive eq '-') {
6073         $input = '<&STDIN';
6074     } else {
6075         # If we use a backup from a PVE defined storage we also consider that
6076         # storage's rate limit:
6077         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6078         if (defined($volid)) {
6079             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6080             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6081             if ($readlimit) {
6082                 print STDERR "applying read rate limit: $readlimit\n";
6083                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6084                 $add_pipe->($cstream);
6085             }
6086         }
6087     }
6088
6089     if ($comp) {
6090         my $cmd;
6091         if ($comp eq 'gzip') {
6092             $cmd = ['zcat', $readfrom];
6093         } elsif ($comp eq 'lzop') {
6094             $cmd = ['lzop', '-d', '-c', $readfrom];
6095         } else {
6096             die "unknown compression method '$comp'\n";
6097         }
6098         $add_pipe->($cmd);
6099     }
6100
6101     my $tmpdir = "/var/tmp/vzdumptmp$$";
6102     rmtree $tmpdir;
6103
6104     # disable interrupts (always do cleanups)
6105     local $SIG{INT} =
6106         local $SIG{TERM} =
6107         local $SIG{QUIT} =
6108         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6109
6110     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6111     POSIX::mkfifo($mapfifo, 0600);
6112     my $fifofh;
6113
6114     my $openfifo = sub {
6115         open($fifofh, '>', $mapfifo) || die $!;
6116     };
6117
6118     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6119
6120     my $oldtimeout;
6121     my $timeout = 5;
6122
6123     my $devinfo = {};
6124
6125     my $rpcenv = PVE::RPCEnvironment::get();
6126
6127     my $conffile = PVE::QemuConfig->config_file($vmid);
6128     my $tmpfn = "$conffile.$$.tmp";
6129
6130     # Note: $oldconf is undef if VM does not exists
6131     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6132     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6133
6134     my %storage_limits;
6135
6136     my $print_devmap = sub {
6137         my $virtdev_hash = {};
6138
6139         my $cfgfn = "$tmpdir/qemu-server.conf";
6140
6141         # we can read the config - that is already extracted
6142         my $fh = IO::File->new($cfgfn, "r") ||
6143             "unable to read qemu-server.conf - $!\n";
6144
6145         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6146         if (-f $fwcfgfn) {
6147             my $pve_firewall_dir = '/etc/pve/firewall';
6148             mkdir $pve_firewall_dir; # make sure the dir exists
6149             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6150         }
6151
6152         while (defined(my $line = <$fh>)) {
6153             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6154                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6155                 die "archive does not contain data for drive '$virtdev'\n"
6156                     if !$devinfo->{$devname};
6157                 if (defined($opts->{storage})) {
6158                     $storeid = $opts->{storage} || 'local';
6159                 } elsif (!$storeid) {
6160                     $storeid = 'local';
6161                 }
6162                 $format = 'raw' if !$format;
6163                 $devinfo->{$devname}->{devname} = $devname;
6164                 $devinfo->{$devname}->{virtdev} = $virtdev;
6165                 $devinfo->{$devname}->{format} = $format;
6166                 $devinfo->{$devname}->{storeid} = $storeid;
6167
6168                 # check permission on storage
6169                 my $pool = $opts->{pool}; # todo: do we need that?
6170                 if ($user ne 'root@pam') {
6171                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6172                 }
6173
6174                 $storage_limits{$storeid} = $bwlimit;
6175
6176                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6177             }
6178         }
6179
6180         foreach my $key (keys %storage_limits) {
6181             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6182             next if !$limit;
6183             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6184             $storage_limits{$key} = $limit * 1024;
6185         }
6186
6187         foreach my $devname (keys %$devinfo) {
6188             die "found no device mapping information for device '$devname'\n"
6189                 if !$devinfo->{$devname}->{virtdev};
6190         }
6191
6192         # create empty/temp config
6193         if ($oldconf) {
6194             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6195             foreach_drive($oldconf, sub {
6196                 my ($ds, $drive) = @_;
6197
6198                 return if drive_is_cdrom($drive);
6199
6200                 my $volid = $drive->{file};
6201
6202                 return if !$volid || $volid =~ m|^/|;
6203
6204                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6205                 return if !$path || !$owner || ($owner != $vmid);
6206
6207                 # Note: only delete disk we want to restore
6208                 # other volumes will become unused
6209                 if ($virtdev_hash->{$ds}) {
6210                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6211                     if (my $err = $@) {
6212                         warn $err;
6213                     }
6214                 }
6215             });
6216
6217             # delete vmstate files
6218             # since after the restore we have no snapshots anymore
6219             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6220                 my $snap = $oldconf->{snapshots}->{$snapname};
6221                 if ($snap->{vmstate}) {
6222                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6223                     if (my $err = $@) {
6224                         warn $err;
6225                     }
6226                 }
6227             }
6228         }
6229
6230         my $map = {};
6231         foreach my $virtdev (sort keys %$virtdev_hash) {
6232             my $d = $virtdev_hash->{$virtdev};
6233             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6234             my $storeid = $d->{storeid};
6235             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6236
6237             my $map_opts = '';
6238             if (my $limit = $storage_limits{$storeid}) {
6239                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6240             }
6241
6242             # test if requested format is supported
6243             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6244             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6245             $d->{format} = $defFormat if !$supported;
6246
6247             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6248                                                   $d->{format}, undef, $alloc_size);
6249             print STDERR "new volume ID is '$volid'\n";
6250             $d->{volid} = $volid;
6251             my $path = PVE::Storage::path($cfg, $volid);
6252
6253             PVE::Storage::activate_volumes($cfg,[$volid]);
6254
6255             my $write_zeros = 1;
6256             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6257                 $write_zeros = 0;
6258             }
6259
6260             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6261
6262             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6263             $map->{$virtdev} = $volid;
6264         }
6265
6266         $fh->seek(0, 0) || die "seek failed - $!\n";
6267
6268         my $outfd = new IO::File ($tmpfn, "w") ||
6269             die "unable to write config for VM $vmid\n";
6270
6271         my $cookie = { netcount => 0 };
6272         while (defined(my $line = <$fh>)) {
6273             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6274         }
6275
6276         $fh->close();
6277         $outfd->close();
6278     };
6279
6280     eval {
6281         # enable interrupts
6282         local $SIG{INT} =
6283             local $SIG{TERM} =
6284             local $SIG{QUIT} =
6285             local $SIG{HUP} =
6286             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6287         local $SIG{ALRM} = sub { die "got timeout\n"; };
6288
6289         $oldtimeout = alarm($timeout);
6290
6291         my $parser = sub {
6292             my $line = shift;
6293
6294             print "$line\n";
6295
6296             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6297                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6298                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6299             } elsif ($line =~ m/^CTIME: /) {
6300                 # we correctly received the vma config, so we can disable
6301                 # the timeout now for disk allocation (set to 10 minutes, so
6302                 # that we always timeout if something goes wrong)
6303                 alarm(600);
6304                 &$print_devmap();
6305                 print $fifofh "done\n";
6306                 my $tmp = $oldtimeout || 0;
6307                 $oldtimeout = undef;
6308                 alarm($tmp);
6309                 close($fifofh);
6310             }
6311         };
6312
6313         print "restore vma archive: $dbg_cmdstring\n";
6314         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6315     };
6316     my $err = $@;
6317
6318     alarm($oldtimeout) if $oldtimeout;
6319
6320     my $vollist = [];
6321     foreach my $devname (keys %$devinfo) {
6322         my $volid = $devinfo->{$devname}->{volid};
6323         push @$vollist, $volid if $volid;
6324     }
6325
6326     PVE::Storage::deactivate_volumes($cfg, $vollist);
6327
6328     unlink $mapfifo;
6329
6330     if ($err) {
6331         rmtree $tmpdir;
6332         unlink $tmpfn;
6333
6334         foreach my $devname (keys %$devinfo) {
6335             my $volid = $devinfo->{$devname}->{volid};
6336             next if !$volid;
6337             eval {
6338                 if ($volid =~ m|^/|) {
6339                     unlink $volid || die 'unlink failed\n';
6340                 } else {
6341                     PVE::Storage::vdisk_free($cfg, $volid);
6342                 }
6343                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6344             };
6345             print STDERR "unable to cleanup '$volid' - $@" if $@;
6346         }
6347         die $err;
6348     }
6349
6350     rmtree $tmpdir;
6351
6352     rename($tmpfn, $conffile) ||
6353         die "unable to commit configuration file '$conffile'\n";
6354
6355     PVE::Cluster::cfs_update(); # make sure we read new file
6356
6357     eval { rescan($vmid, 1); };
6358     warn $@ if $@;
6359 }
6360
6361 sub restore_tar_archive {
6362     my ($archive, $vmid, $user, $opts) = @_;
6363
6364     if ($archive ne '-') {
6365         my $firstfile = tar_archive_read_firstfile($archive);
6366         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6367             if $firstfile ne 'qemu-server.conf';
6368     }
6369
6370     my $storecfg = PVE::Storage::config();
6371
6372     # destroy existing data - keep empty config
6373     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6374     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6375
6376     my $tocmd = "/usr/lib/qemu-server/qmextract";
6377
6378     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6379     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6380     $tocmd .= ' --prealloc' if $opts->{prealloc};
6381     $tocmd .= ' --info' if $opts->{info};
6382
6383     # tar option "xf" does not autodetect compression when read from STDIN,
6384     # so we pipe to zcat
6385     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6386         PVE::Tools::shellquote("--to-command=$tocmd");
6387
6388     my $tmpdir = "/var/tmp/vzdumptmp$$";
6389     mkpath $tmpdir;
6390
6391     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6392     local $ENV{VZDUMP_VMID} = $vmid;
6393     local $ENV{VZDUMP_USER} = $user;
6394
6395     my $conffile = PVE::QemuConfig->config_file($vmid);
6396     my $tmpfn = "$conffile.$$.tmp";
6397
6398     # disable interrupts (always do cleanups)
6399     local $SIG{INT} =
6400         local $SIG{TERM} =
6401         local $SIG{QUIT} =
6402         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6403
6404     eval {
6405         # enable interrupts
6406         local $SIG{INT} =
6407             local $SIG{TERM} =
6408             local $SIG{QUIT} =
6409             local $SIG{HUP} =
6410             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6411
6412         if ($archive eq '-') {
6413             print "extracting archive from STDIN\n";
6414             run_command($cmd, input => "<&STDIN");
6415         } else {
6416             print "extracting archive '$archive'\n";
6417             run_command($cmd);
6418         }
6419
6420         return if $opts->{info};
6421
6422         # read new mapping
6423         my $map = {};
6424         my $statfile = "$tmpdir/qmrestore.stat";
6425         if (my $fd = IO::File->new($statfile, "r")) {
6426             while (defined (my $line = <$fd>)) {
6427                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6428                     $map->{$1} = $2 if $1;
6429                 } else {
6430                     print STDERR "unable to parse line in statfile - $line\n";
6431                 }
6432             }
6433             $fd->close();
6434         }
6435
6436         my $confsrc = "$tmpdir/qemu-server.conf";
6437
6438         my $srcfd = new IO::File($confsrc, "r") ||
6439             die "unable to open file '$confsrc'\n";
6440
6441         my $outfd = new IO::File ($tmpfn, "w") ||
6442             die "unable to write config for VM $vmid\n";
6443
6444         my $cookie = { netcount => 0 };
6445         while (defined (my $line = <$srcfd>)) {
6446             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6447         }
6448
6449         $srcfd->close();
6450         $outfd->close();
6451     };
6452     my $err = $@;
6453
6454     if ($err) {
6455
6456         unlink $tmpfn;
6457
6458         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6459
6460         die $err;
6461     }
6462
6463     rmtree $tmpdir;
6464
6465     rename $tmpfn, $conffile ||
6466         die "unable to commit configuration file '$conffile'\n";
6467
6468     PVE::Cluster::cfs_update(); # make sure we read new file
6469
6470     eval { rescan($vmid, 1); };
6471     warn $@ if $@;
6472 };
6473
6474 sub foreach_storage_used_by_vm {
6475     my ($conf, $func) = @_;
6476
6477     my $sidhash = {};
6478
6479     foreach_drive($conf, sub {
6480         my ($ds, $drive) = @_;
6481         return if drive_is_cdrom($drive);
6482
6483         my $volid = $drive->{file};
6484
6485         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6486         $sidhash->{$sid} = $sid if $sid;
6487     });
6488
6489     foreach my $sid (sort keys %$sidhash) {
6490         &$func($sid);
6491     }
6492 }
6493
6494 sub do_snapshots_with_qemu {
6495     my ($storecfg, $volid) = @_;
6496
6497     my $storage_name = PVE::Storage::parse_volume_id($volid);
6498
6499     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6500         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6501         return 1;
6502     }
6503
6504     if ($volid =~ m/\.(qcow2|qed)$/){
6505         return 1;
6506     }
6507
6508     return undef;
6509 }
6510
6511 sub qga_check_running {
6512     my ($vmid, $nowarn) = @_;
6513
6514     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6515     if ($@) {
6516         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6517         return 0;
6518     }
6519     return 1;
6520 }
6521
6522 sub template_create {
6523     my ($vmid, $conf, $disk) = @_;
6524
6525     my $storecfg = PVE::Storage::config();
6526
6527     foreach_drive($conf, sub {
6528         my ($ds, $drive) = @_;
6529
6530         return if drive_is_cdrom($drive);
6531         return if $disk && $ds ne $disk;
6532
6533         my $volid = $drive->{file};
6534         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6535
6536         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6537         $drive->{file} = $voliddst;
6538         $conf->{$ds} = print_drive($vmid, $drive);
6539         PVE::QemuConfig->write_config($vmid, $conf);
6540     });
6541 }
6542
6543 sub convert_iscsi_path {
6544     my ($path) = @_;
6545
6546     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6547         my $portal = $1;
6548         my $target = $2;
6549         my $lun = $3;
6550
6551         my $initiator_name = get_initiator_name();
6552
6553         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6554                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6555     }
6556
6557     die "cannot convert iscsi path '$path', unkown format\n";
6558 }
6559
6560 sub qemu_img_convert {
6561     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6562
6563     my $storecfg = PVE::Storage::config();
6564     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6565     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6566
6567     if ($src_storeid && $dst_storeid) {
6568
6569         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6570
6571         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6572         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6573
6574         my $src_format = qemu_img_format($src_scfg, $src_volname);
6575         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6576
6577         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6578         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6579
6580         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6581         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6582
6583         my $cmd = [];
6584         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6585         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6586         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6587         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6588
6589         if ($src_is_iscsi) {
6590             push @$cmd, '--image-opts';
6591             $src_path = convert_iscsi_path($src_path);
6592         } else {
6593             push @$cmd, '-f', $src_format;
6594         }
6595
6596         if ($dst_is_iscsi) {
6597             push @$cmd, '--target-image-opts';
6598             $dst_path = convert_iscsi_path($dst_path);
6599         } else {
6600             push @$cmd, '-O', $dst_format;
6601         }
6602
6603         push @$cmd, $src_path;
6604
6605         if (!$dst_is_iscsi && $is_zero_initialized) {
6606             push @$cmd, "zeroinit:$dst_path";
6607         } else {
6608             push @$cmd, $dst_path;
6609         }
6610
6611         my $parser = sub {
6612             my $line = shift;
6613             if($line =~ m/\((\S+)\/100\%\)/){
6614                 my $percent = $1;
6615                 my $transferred = int($size * $percent / 100);
6616                 my $remaining = $size - $transferred;
6617
6618                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6619             }
6620
6621         };
6622
6623         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6624         my $err = $@;
6625         die "copy failed: $err" if $err;
6626     }
6627 }
6628
6629 sub qemu_img_format {
6630     my ($scfg, $volname) = @_;
6631
6632     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6633         return $1;
6634     } else {
6635         return "raw";
6636     }
6637 }
6638
6639 sub qemu_drive_mirror {
6640     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6641
6642     $jobs = {} if !$jobs;
6643
6644     my $qemu_target;
6645     my $format;
6646     $jobs->{"drive-$drive"} = {};
6647
6648     if ($dst_volid =~ /^nbd:/) {
6649         $qemu_target = $dst_volid;
6650         $format = "nbd";
6651     } else {
6652         my $storecfg = PVE::Storage::config();
6653         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6654
6655         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6656
6657         $format = qemu_img_format($dst_scfg, $dst_volname);
6658
6659         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6660
6661         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6662     }
6663
6664     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6665     $opts->{format} = $format if $format;
6666
6667     print "drive mirror is starting for drive-$drive\n";
6668
6669     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6670
6671     if (my $err = $@) {
6672         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6673         die "mirroring error: $err";
6674     }
6675
6676     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6677 }
6678
6679 sub qemu_drive_mirror_monitor {
6680     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6681
6682     eval {
6683         my $err_complete = 0;
6684
6685         while (1) {
6686             die "storage migration timed out\n" if $err_complete > 300;
6687
6688             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6689
6690             my $running_mirror_jobs = {};
6691             foreach my $stat (@$stats) {
6692                 next if $stat->{type} ne 'mirror';
6693                 $running_mirror_jobs->{$stat->{device}} = $stat;
6694             }
6695
6696             my $readycounter = 0;
6697
6698             foreach my $job (keys %$jobs) {
6699
6700                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6701                     print "$job : finished\n";
6702                     delete $jobs->{$job};
6703                     next;
6704                 }
6705
6706                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6707
6708                 my $busy = $running_mirror_jobs->{$job}->{busy};
6709                 my $ready = $running_mirror_jobs->{$job}->{ready};
6710                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6711                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6712                     my $remaining = $total - $transferred;
6713                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6714
6715                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6716                 }
6717
6718                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6719             }
6720
6721             last if scalar(keys %$jobs) == 0;
6722
6723             if ($readycounter == scalar(keys %$jobs)) {
6724                 print "all mirroring jobs are ready \n";
6725                 last if $skipcomplete; #do the complete later
6726
6727                 if ($vmiddst && $vmiddst != $vmid) {
6728                     my $agent_running = $qga && qga_check_running($vmid);
6729                     if ($agent_running) {
6730                         print "freeze filesystem\n";
6731                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6732                     } else {
6733                         print "suspend vm\n";
6734                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6735                     }
6736
6737                     # if we clone a disk for a new target vm, we don't switch the disk
6738                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6739
6740                     if ($agent_running) {
6741                         print "unfreeze filesystem\n";
6742                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6743                     } else {
6744                         print "resume vm\n";
6745                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6746                     }
6747
6748                     last;
6749                 } else {
6750
6751                     foreach my $job (keys %$jobs) {
6752                         # try to switch the disk if source and destination are on the same guest
6753                         print "$job: Completing block job...\n";
6754
6755                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6756                         if ($@ =~ m/cannot be completed/) {
6757                             print "$job: Block job cannot be completed, try again.\n";
6758                             $err_complete++;
6759                         }else {
6760                             print "$job: Completed successfully.\n";
6761                             $jobs->{$job}->{complete} = 1;
6762                         }
6763                     }
6764                 }
6765             }
6766             sleep 1;
6767         }
6768     };
6769     my $err = $@;
6770
6771     if ($err) {
6772         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6773         die "mirroring error: $err";
6774     }
6775
6776 }
6777
6778 sub qemu_blockjobs_cancel {
6779     my ($vmid, $jobs) = @_;
6780
6781     foreach my $job (keys %$jobs) {
6782         print "$job: Cancelling block job\n";
6783         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6784         $jobs->{$job}->{cancel} = 1;
6785     }
6786
6787     while (1) {
6788         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6789
6790         my $running_jobs = {};
6791         foreach my $stat (@$stats) {
6792             $running_jobs->{$stat->{device}} = $stat;
6793         }
6794
6795         foreach my $job (keys %$jobs) {
6796
6797             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6798                 print "$job: Done.\n";
6799                 delete $jobs->{$job};
6800             }
6801         }
6802
6803         last if scalar(keys %$jobs) == 0;
6804
6805         sleep 1;
6806     }
6807 }
6808
6809 sub clone_disk {
6810     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6811         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6812
6813     my $newvolid;
6814
6815     if (!$full) {
6816         print "create linked clone of drive $drivename ($drive->{file})\n";
6817         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6818         push @$newvollist, $newvolid;
6819     } else {
6820
6821         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6822         $storeid = $storage if $storage;
6823
6824         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6825         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6826
6827         print "create full clone of drive $drivename ($drive->{file})\n";
6828         my $name = undef;
6829         if (drive_is_cloudinit($drive)) {
6830             $name = "vm-$newvmid-cloudinit";
6831             $snapname = undef;
6832             # cloudinit only supports raw and qcow2 atm:
6833             if ($dst_format eq 'qcow2') {
6834                 $name .= '.qcow2';
6835             } elsif ($dst_format ne 'raw') {
6836                 die "clone: unhandled format for cloudinit image\n";
6837             }
6838         }
6839         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6840         push @$newvollist, $newvolid;
6841
6842         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6843
6844         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6845         if (!$running || $snapname) {
6846             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6847         } else {
6848
6849             my $kvmver = get_running_qemu_version ($vmid);
6850             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6851                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6852                     if $drive->{iothread};
6853             }
6854
6855             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6856         }
6857     }
6858
6859     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6860
6861     my $disk = $drive;
6862     $disk->{format} = undef;
6863     $disk->{file} = $newvolid;
6864     $disk->{size} = $size;
6865
6866     return $disk;
6867 }
6868
6869 # this only works if VM is running
6870 sub get_current_qemu_machine {
6871     my ($vmid) = @_;
6872
6873     my $cmd = { execute => 'query-machines', arguments => {} };
6874     my $res = vm_qmp_command($vmid, $cmd);
6875
6876     my ($current, $default);
6877     foreach my $e (@$res) {
6878         $default = $e->{name} if $e->{'is-default'};
6879         $current = $e->{name} if $e->{'is-current'};
6880     }
6881
6882     # fallback to the default machine if current is not supported by qemu
6883     return $current || $default || 'pc';
6884 }
6885
6886 sub get_running_qemu_version {
6887     my ($vmid) = @_;
6888     my $cmd = { execute => 'query-version', arguments => {} };
6889     my $res = vm_qmp_command($vmid, $cmd);
6890     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6891 }
6892
6893 sub qemu_machine_feature_enabled {
6894     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6895
6896     my $current_major;
6897     my $current_minor;
6898
6899     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6900
6901         $current_major = $3;
6902         $current_minor = $4;
6903
6904     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6905
6906         $current_major = $1;
6907         $current_minor = $2;
6908     }
6909
6910     return 1 if $current_major > $version_major ||
6911                 ($current_major == $version_major &&
6912                  $current_minor >= $version_minor);
6913 }
6914
6915 sub qemu_machine_pxe {
6916     my ($vmid, $conf, $machine) = @_;
6917
6918     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6919
6920     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6921         $machine .= '.pxe';
6922     }
6923
6924     return $machine;
6925 }
6926
6927 sub qemu_use_old_bios_files {
6928     my ($machine_type) = @_;
6929
6930     return if !$machine_type;
6931
6932     my $use_old_bios_files = undef;
6933
6934     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6935         $machine_type = $1;
6936         $use_old_bios_files = 1;
6937     } else {
6938         my $kvmver = kvm_user_version();
6939         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6940         # load new efi bios files on migration. So this hack is required to allow
6941         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6942         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6943         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6944     }
6945
6946     return ($use_old_bios_files, $machine_type);
6947 }
6948
6949 sub create_efidisk($$$$$) {
6950     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6951
6952     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6953     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6954
6955     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6956     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6957     PVE::Storage::activate_volumes($storecfg, [$volid]);
6958
6959     my $path = PVE::Storage::path($storecfg, $volid);
6960     eval {
6961         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6962     };
6963     die "Copying EFI vars image failed: $@" if $@;
6964
6965     return ($volid, $vars_size);
6966 }
6967
6968 sub vm_iothreads_list {
6969     my ($vmid) = @_;
6970
6971     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6972
6973     my $iothreads = {};
6974     foreach my $iothread (@$res) {
6975         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6976     }
6977
6978     return $iothreads;
6979 }
6980
6981 sub scsihw_infos {
6982     my ($conf, $drive) = @_;
6983
6984     my $maxdev = 0;
6985
6986     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6987         $maxdev = 7;
6988     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6989         $maxdev = 1;
6990     } else {
6991         $maxdev = 256;
6992     }
6993
6994     my $controller = int($drive->{index} / $maxdev);
6995     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6996
6997     return ($maxdev, $controller, $controller_prefix);
6998 }
6999
7000 sub add_hyperv_enlightenments {
7001     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7002
7003     return if $winversion < 6;
7004     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7005
7006     if ($gpu_passthrough || defined($hv_vendor_id)) {
7007         $hv_vendor_id //= 'proxmox';
7008         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7009     }
7010
7011     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7012         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7013         push @$cpuFlags , 'hv_vapic';
7014         push @$cpuFlags , 'hv_time';
7015     } else {
7016         push @$cpuFlags , 'hv_spinlocks=0xffff';
7017     }
7018
7019     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7020         push @$cpuFlags , 'hv_reset';
7021         push @$cpuFlags , 'hv_vpindex';
7022         push @$cpuFlags , 'hv_runtime';
7023     }
7024
7025     if ($winversion >= 7) {
7026         push @$cpuFlags , 'hv_relaxed';
7027
7028         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7029             push @$cpuFlags , 'hv_synic';
7030             push @$cpuFlags , 'hv_stimer';
7031         }
7032     }
7033 }
7034
7035 sub windows_version {
7036     my ($ostype) = @_;
7037
7038     return 0 if !$ostype;
7039
7040     my $winversion = 0;
7041
7042     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7043         $winversion = 5;
7044     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7045         $winversion = 6;
7046     } elsif ($ostype =~ m/^win(\d+)$/) {
7047         $winversion = $1;
7048     }
7049
7050     return $winversion;
7051 }
7052
7053 sub resolve_dst_disk_format {
7054         my ($storecfg, $storeid, $src_volname, $format) = @_;
7055         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7056
7057         if (!$format) {
7058             # if no target format is specified, use the source disk format as hint
7059             if ($src_volname) {
7060                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7061                 $format = qemu_img_format($scfg, $src_volname);
7062             } else {
7063                 return $defFormat;
7064             }
7065         }
7066
7067         # test if requested format is supported - else use default
7068         my $supported = grep { $_ eq $format } @$validFormats;
7069         $format = $defFormat if !$supported;
7070         return $format;
7071 }
7072
7073 sub resolve_first_disk {
7074     my $conf = shift;
7075     my @disks = PVE::QemuServer::valid_drive_names();
7076     my $firstdisk;
7077     foreach my $ds (reverse @disks) {
7078         next if !$conf->{$ds};
7079         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7080         next if PVE::QemuServer::drive_is_cdrom($disk);
7081         $firstdisk = $ds;
7082     }
7083     return $firstdisk;
7084 }
7085
7086 sub generate_uuid {
7087     my ($uuid, $uuid_str);
7088     UUID::generate($uuid);
7089     UUID::unparse($uuid, $uuid_str);
7090     return $uuid_str;
7091 }
7092
7093 sub generate_smbios1_uuid {
7094     return "uuid=".generate_uuid();
7095 }
7096
7097 sub nbd_stop {
7098     my ($vmid) = @_;
7099
7100     vm_mon_cmd($vmid, 'nbd-server-stop');
7101 }
7102
7103 # bash completion helper
7104
7105 sub complete_backup_archives {
7106     my ($cmdname, $pname, $cvalue) = @_;
7107
7108     my $cfg = PVE::Storage::config();
7109
7110     my $storeid;
7111
7112     if ($cvalue =~ m/^([^:]+):/) {
7113         $storeid = $1;
7114     }
7115
7116     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7117
7118     my $res = [];
7119     foreach my $id (keys %$data) {
7120         foreach my $item (@{$data->{$id}}) {
7121             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7122             push @$res, $item->{volid} if defined($item->{volid});
7123         }
7124     }
7125
7126     return $res;
7127 }
7128
7129 my $complete_vmid_full = sub {
7130     my ($running) = @_;
7131
7132     my $idlist = vmstatus();
7133
7134     my $res = [];
7135
7136     foreach my $id (keys %$idlist) {
7137         my $d = $idlist->{$id};
7138         if (defined($running)) {
7139             next if $d->{template};
7140             next if $running && $d->{status} ne 'running';
7141             next if !$running && $d->{status} eq 'running';
7142         }
7143         push @$res, $id;
7144
7145     }
7146     return $res;
7147 };
7148
7149 sub complete_vmid {
7150     return &$complete_vmid_full();
7151 }
7152
7153 sub complete_vmid_stopped {
7154     return &$complete_vmid_full(0);
7155 }
7156
7157 sub complete_vmid_running {
7158     return &$complete_vmid_full(1);
7159 }
7160
7161 sub complete_storage {
7162
7163     my $cfg = PVE::Storage::config();
7164     my $ids = $cfg->{ids};
7165
7166     my $res = [];
7167     foreach my $sid (keys %$ids) {
7168         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7169         next if !$ids->{$sid}->{content}->{images};
7170         push @$res, $sid;
7171     }
7172
7173     return $res;
7174 }
7175
7176 1;