PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename_cache;
 111 sub nodename {
 112     $nodename_cache //= PVE::INotify::nodename();
 113     return $nodename_cache;
 114 }
 115
 116 my $watchdog_fmt = {
 117     model => {
 118         default_key => 1,
 119         type => 'string',
 120         enum => [qw(i6300esb ib700)],
 121         description => "Watchdog type to emulate.",
 122         default => 'i6300esb',
 123         optional => 1,
 124     },
 125     action => {
 126         type => 'string',
 127         enum => [qw(reset shutdown poweroff pause debug none)],
 128         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 129         optional => 1,
 130     },
 131 };
 132 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 133
 134 my $agent_fmt = {
 135     enabled => {
 136         description => "Enable/disable Qemu GuestAgent.",
 137         type => 'boolean',
 138         default => 0,
 139         default_key => 1,
 140     },
 141     fstrim_cloned_disks => {
 142         description => "Run fstrim after cloning/moving a disk.",
 143         type => 'boolean',
 144         optional => 1,
 145         default => 0
 146     },
 147     type => {
 148         description => "Select the agent type",
 149         type => 'string',
 150         default => 'virtio',
 151         optional => 1,
 152         enum => [qw(virtio isa)],
 153     },
 154 };
 155
 156 my $vga_fmt = {
 157     type => {
 158         description => "Select the VGA type.",
 159         type => 'string',
 160         default => 'std',
 161         optional => 1,
 162         default_key => 1,
 163         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 164     },
 165     memory => {
 166         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 167         type => 'integer',
 168         optional => 1,
 169         minimum => 4,
 170         maximum => 512,
 171     },
 172 };
 173
 174 my $ivshmem_fmt = {
 175     size => {
 176         type => 'integer',
 177         minimum => 1,
 178         description => "The size of the file in MB.",
 179     },
 180     name => {
 181         type => 'string',
 182         pattern => '[a-zA-Z0-9\-]+',
 183         optional => 1,
 184         format_description => 'string',
 185         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 186     },
 187 };
 188
 189 my $audio_fmt = {
 190     device => {
 191         type => 'string',
 192         enum => [qw(ich9-intel-hda intel-hda AC97)],
 193         description =>  "Configure an audio device."
 194     },
 195     driver =>  {
 196         type => 'string',
 197         enum => ['spice'],
 198         default => 'spice',
 199         optional => 1,
 200         description => "Driver backend for the audio device."
 201     },
 202 };
 203
 204 my $spice_enhancements_fmt = {
 205     foldersharing => {
 206         type => 'boolean',
 207         optional => 1,
 208         default => '0',
 209         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 210     },
 211     videostreaming =>  {
 212         type => 'string',
 213         enum => ['off', 'all', 'filter'],
 214         default => 'off',
 215         optional => 1,
 216         description => "Enable video streaming. Uses compression for detected video streams."
 217     },
 218 };
 219
 220 my $rng_fmt = {
 221     source => {
 222         type => 'string',
 223         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 224         default_key => 1,
 225         description => "The file on the host to gather entropy from. In most"
 226                      . " cases /dev/urandom should be preferred over /dev/random"
 227                      . " to avoid entropy-starvation issues on the host. Using"
 228                      . " urandom does *not* decrease security in any meaningful"
 229                      . " way, as it's still seeded from real entropy, and the"
 230                      . " bytes provided will most likely be mixed with real"
 231                      . " entropy on the guest as well. /dev/hwrng can be used"
 232                      . " to pass through a hardware RNG from the host.",
 233     },
 234     max_bytes => {
 235         type => 'integer',
 236         description => "Maximum bytes of entropy injected into the guest every"
 237                      . " 'period' milliseconds. Prefer a lower value when using"
 238                      . " /dev/random as source. Use 0 to disable limiting"
 239                      . " (potentially dangerous!).",
 240         optional => 1,
 241
 242         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 243         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 244         # of overwhelming the host, provided we're reading from /dev/urandom
 245         default => 1024,
 246     },
 247     period => {
 248         type => 'integer',
 249         description => "Every 'period' milliseconds the entropy-injection quota"
 250                      . " is reset, allowing the guest to retrieve another"
 251                      . " 'max_bytes' of entropy.",
 252         optional => 1,
 253         default => 1000,
 254     },
 255 };
 256
 257 my $confdesc = {
 258     onboot => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Specifies whether a VM will be started during system bootup.",
 262         default => 0,
 263     },
 264     autostart => {
 265         optional => 1,
 266         type => 'boolean',
 267         description => "Automatic restart after crash (currently ignored).",
 268         default => 0,
 269     },
 270     hotplug => {
 271         optional => 1,
 272         type => 'string', format => 'pve-hotplug-features',
 273         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 274         default => 'network,disk,usb',
 275     },
 276     reboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 280         default => 1,
 281     },
 282     lock => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Lock/unlock the VM.",
 286         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 287     },
 288     cpulimit => {
 289         optional => 1,
 290         type => 'number',
 291         description => "Limit of CPU usage.",
 292         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 293         minimum => 0,
 294         maximum => 128,
 295         default => 0,
 296     },
 297     cpuunits => {
 298         optional => 1,
 299         type => 'integer',
 300         description => "CPU weight for a VM.",
 301         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 302         minimum => 2,
 303         maximum => 262144,
 304         default => 1024,
 305     },
 306     memory => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 310         minimum => 16,
 311         default => 512,
 312     },
 313     balloon => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 317         minimum => 0,
 318     },
 319     shares => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 323         minimum => 0,
 324         maximum => 50000,
 325         default => 1000,
 326     },
 327     keyboard => {
 328         optional => 1,
 329         type => 'string',
 330         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 331                        "It should not be necessary to set it.",
 332         enum => PVE::Tools::kvmkeymaplist(),
 333         default => undef,
 334     },
 335     name => {
 336         optional => 1,
 337         type => 'string', format => 'dns-name',
 338         description => "Set a name for the VM. Only used on the configuration web interface.",
 339     },
 340     scsihw => {
 341         optional => 1,
 342         type => 'string',
 343         description => "SCSI controller model",
 344         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 345         default => 'lsi',
 346     },
 347     description => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 351     },
 352     ostype => {
 353         optional => 1,
 354         type => 'string',
 355         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 356         description => "Specify guest operating system.",
 357         verbose_description => <<EODESC,
 358 Specify guest operating system. This is used to enable special
 359 optimization/features for specific operating systems:
 360
 361 [horizontal]
 362 other;; unspecified OS
 363 wxp;; Microsoft Windows XP
 364 w2k;; Microsoft Windows 2000
 365 w2k3;; Microsoft Windows 2003
 366 w2k8;; Microsoft Windows 2008
 367 wvista;; Microsoft Windows Vista
 368 win7;; Microsoft Windows 7
 369 win8;; Microsoft Windows 8/2012/2012r2
 370 win10;; Microsoft Windows 10/2016
 371 l24;; Linux 2.4 Kernel
 372 l26;; Linux 2.6 - 5.X Kernel
 373 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 374 EODESC
 375     },
 376     boot => {
 377         optional => 1,
 378         type => 'string',
 379         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 380         pattern => '[acdn]{1,4}',
 381         default => 'cdn',
 382     },
 383     bootdisk => {
 384         optional => 1,
 385         type => 'string', format => 'pve-qm-bootdisk',
 386         description => "Enable booting from specified disk.",
 387         pattern => '(ide|sata|scsi|virtio)\d+',
 388     },
 389     smp => {
 390         optional => 1,
 391         type => 'integer',
 392         description => "The number of CPUs. Please use option -sockets instead.",
 393         minimum => 1,
 394         default => 1,
 395     },
 396     sockets => {
 397         optional => 1,
 398         type => 'integer',
 399         description => "The number of CPU sockets.",
 400         minimum => 1,
 401         default => 1,
 402     },
 403     cores => {
 404         optional => 1,
 405         type => 'integer',
 406         description => "The number of cores per socket.",
 407         minimum => 1,
 408         default => 1,
 409     },
 410     numa => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable NUMA.",
 414         default => 0,
 415     },
 416     hugepages => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Enable/disable hugepages memory.",
 420         enum => [qw(any 2 1024)],
 421     },
 422     vcpus => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "Number of hotplugged vcpus.",
 426         minimum => 1,
 427         default => 0,
 428     },
 429     acpi => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable ACPI.",
 433         default => 1,
 434     },
 435     agent => {
 436         optional => 1,
 437         description => "Enable/disable Qemu GuestAgent and its properties.",
 438         type => 'string',
 439         format => $agent_fmt,
 440     },
 441     kvm => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable KVM hardware virtualization.",
 445         default => 1,
 446     },
 447     tdf => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Enable/disable time drift fix.",
 451         default => 0,
 452     },
 453     localtime => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 457     },
 458     freeze => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 462     },
 463     vga => {
 464         optional => 1,
 465         type => 'string', format => $vga_fmt,
 466         description => "Configure the VGA hardware.",
 467         verbose_description => "Configure the VGA Hardware. If you want to use ".
 468             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 469             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 470             "is 'std' for all OS types besides some Windows versions (XP and " .
 471             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 472             "display server. For win* OS you can select how many independent " .
 473             "displays you want, Linux guests can add displays them self.\n".
 474             "You can also run without any graphic card, using a serial device as terminal.",
 475     },
 476     watchdog => {
 477         optional => 1,
 478         type => 'string', format => 'pve-qm-watchdog',
 479         description => "Create a virtual hardware watchdog device.",
 480         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 481             " (by a guest action), the watchdog must be periodically polled " .
 482             "by an agent inside the guest or else the watchdog will reset " .
 483             "the guest (or execute the respective action specified)",
 484     },
 485     startdate => {
 486         optional => 1,
 487         type => 'string',
 488         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 489         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 490         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 491         default => 'now',
 492     },
 493     startup =>  get_standard_option('pve-startup-order'),
 494     template => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable Template.",
 498         default => 0,
 499     },
 500     args => {
 501         optional => 1,
 502         type => 'string',
 503         description => "Arbitrary arguments passed to kvm.",
 504         verbose_description => <<EODESCR,
 505 Arbitrary arguments passed to kvm, for example:
 506
 507 args: -no-reboot -no-hpet
 508
 509 NOTE: this option is for experts only.
 510 EODESCR
 511     },
 512     tablet => {
 513         optional => 1,
 514         type => 'boolean',
 515         default => 1,
 516         description => "Enable/disable the USB tablet device.",
 517         verbose_description => "Enable/disable the USB tablet device. This device is " .
 518             "usually needed to allow absolute mouse positioning with VNC. " .
 519             "Else the mouse runs out of sync with normal VNC clients. " .
 520             "If you're running lots of console-only guests on one host, " .
 521             "you may consider disabling this to save some context switches. " .
 522             "This is turned off by default if you use spice (-vga=qxl).",
 523     },
 524     migrate_speed => {
 525         optional => 1,
 526         type => 'integer',
 527         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 528         minimum => 0,
 529         default => 0,
 530     },
 531     migrate_downtime => {
 532         optional => 1,
 533         type => 'number',
 534         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 535         minimum => 0,
 536         default => 0.1,
 537     },
 538     cdrom => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-ide',
 541         typetext => '<volume>',
 542         description => "This is an alias for option -ide2",
 543     },
 544     cpu => {
 545         optional => 1,
 546         description => "Emulated CPU type.",
 547         type => 'string',
 548         format => 'pve-vm-cpu-conf',
 549     },
 550     parent => get_standard_option('pve-snapshot-name', {
 551         optional => 1,
 552         description => "Parent snapshot name. This is used internally, and should not be modified.",
 553     }),
 554     snaptime => {
 555         optional => 1,
 556         description => "Timestamp for snapshots.",
 557         type => 'integer',
 558         minimum => 0,
 559     },
 560     vmstate => {
 561         optional => 1,
 562         type => 'string', format => 'pve-volume-id',
 563         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 564     },
 565     vmstatestorage => get_standard_option('pve-storage-id', {
 566         description => "Default storage for VM state volumes/files.",
 567         optional => 1,
 568     }),
 569     runningmachine => get_standard_option('pve-qemu-machine', {
 570         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 571     }),
 572     machine => get_standard_option('pve-qemu-machine'),
 573     arch => {
 574         description => "Virtual processor architecture. Defaults to the host.",
 575         optional => 1,
 576         type => 'string',
 577         enum => [qw(x86_64 aarch64)],
 578     },
 579     smbios1 => {
 580         description => "Specify SMBIOS type 1 fields.",
 581         type => 'string', format => 'pve-qm-smbios1',
 582         maxLength => 512,
 583         optional => 1,
 584     },
 585     protection => {
 586         optional => 1,
 587         type => 'boolean',
 588         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 589         default => 0,
 590     },
 591     bios => {
 592         optional => 1,
 593         type => 'string',
 594         enum => [ qw(seabios ovmf) ],
 595         description => "Select BIOS implementation.",
 596         default => 'seabios',
 597     },
 598     vmgenid => {
 599         type => 'string',
 600         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 601         format_description => 'UUID',
 602         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 603         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 604             " 128-bit integer value identifier to the guest OS. This allows to".
 605             " notify the guest operating system when the virtual machine is".
 606             " executed with a different configuration (e.g. snapshot execution".
 607             " or creation from a template). The guest operating system notices".
 608             " the change, and is then able to react as appropriate by marking".
 609             " its copies of distributed databases as dirty, re-initializing its".
 610             " random number generator, etc.\n".
 611             "Note that auto-creation only works when done throug API/CLI create".
 612             " or update methods, but not when manually editing the config file.",
 613         default => "1 (autogenerated)",
 614         optional => 1,
 615     },
 616     hookscript => {
 617         type => 'string',
 618         format => 'pve-volume-id',
 619         optional => 1,
 620         description => "Script that will be executed during various steps in the vms lifetime.",
 621     },
 622     ivshmem => {
 623         type => 'string',
 624         format => $ivshmem_fmt,
 625         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 626         optional => 1,
 627     },
 628     audio0 => {
 629         type => 'string',
 630         format => $audio_fmt,
 631         description => "Configure a audio device, useful in combination with QXL/Spice.",
 632         optional => 1
 633     },
 634     spice_enhancements => {
 635         type => 'string',
 636         format => $spice_enhancements_fmt,
 637         description => "Configure additional enhancements for SPICE.",
 638         optional => 1
 639     },
 640     tags => {
 641         type => 'string', format => 'pve-tag-list',
 642         description => 'Tags of the VM. This is only meta information.',
 643         optional => 1,
 644     },
 645     rng0 => {
 646         type => 'string',
 647         format => $rng_fmt,
 648         description => "Configure a VirtIO-based Random Number Generator.",
 649         optional => 1,
 650     },
 651 };
 652
 653 my $cicustom_fmt = {
 654     meta => {
 655         type => 'string',
 656         optional => 1,
 657         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 658         format => 'pve-volume-id',
 659         format_description => 'volume',
 660     },
 661     network => {
 662         type => 'string',
 663         optional => 1,
 664         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 665         format => 'pve-volume-id',
 666         format_description => 'volume',
 667     },
 668     user => {
 669         type => 'string',
 670         optional => 1,
 671         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 672         format => 'pve-volume-id',
 673         format_description => 'volume',
 674     },
 675 };
 676 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 677
 678 my $confdesc_cloudinit = {
 679     citype => {
 680         optional => 1,
 681         type => 'string',
 682         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 683         enum => ['configdrive2', 'nocloud'],
 684     },
 685     ciuser => {
 686         optional => 1,
 687         type => 'string',
 688         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 689     },
 690     cipassword => {
 691         optional => 1,
 692         type => 'string',
 693         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 694     },
 695     cicustom => {
 696         optional => 1,
 697         type => 'string',
 698         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 699         format => 'pve-qm-cicustom',
 700     },
 701     searchdomain => {
 702         optional => 1,
 703         type => 'string',
 704         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 705     },
 706     nameserver => {
 707         optional => 1,
 708         type => 'string', format => 'address-list',
 709         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 710     },
 711     sshkeys => {
 712         optional => 1,
 713         type => 'string',
 714         format => 'urlencoded',
 715         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 716     },
 717 };
 718
 719 # what about other qemu settings ?
 720 #cpu => 'string',
 721 #machine => 'string',
 722 #fda => 'file',
 723 #fdb => 'file',
 724 #mtdblock => 'file',
 725 #sd => 'file',
 726 #pflash => 'file',
 727 #snapshot => 'bool',
 728 #bootp => 'file',
 729 ##tftp => 'dir',
 730 ##smb => 'dir',
 731 #kernel => 'file',
 732 #append => 'string',
 733 #initrd => 'file',
 734 ##soundhw => 'string',
 735
 736 while (my ($k, $v) = each %$confdesc) {
 737     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 738 }
 739
 740 my $MAX_USB_DEVICES = 5;
 741 my $MAX_NETS = 32;
 742 my $MAX_HOSTPCI_DEVICES = 16;
 743 my $MAX_SERIAL_PORTS = 4;
 744 my $MAX_PARALLEL_PORTS = 3;
 745 my $MAX_NUMA = 8;
 746
 747 my $numa_fmt = {
 748     cpus => {
 749         type => "string",
 750         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 751         description => "CPUs accessing this NUMA node.",
 752         format_description => "id[-id];...",
 753     },
 754     memory => {
 755         type => "number",
 756         description => "Amount of memory this NUMA node provides.",
 757         optional => 1,
 758     },
 759     hostnodes => {
 760         type => "string",
 761         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 762         description => "Host NUMA nodes to use.",
 763         format_description => "id[-id];...",
 764         optional => 1,
 765     },
 766     policy => {
 767         type => 'string',
 768         enum => [qw(preferred bind interleave)],
 769         description => "NUMA allocation policy.",
 770         optional => 1,
 771     },
 772 };
 773 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 774 my $numadesc = {
 775     optional => 1,
 776     type => 'string', format => $numa_fmt,
 777     description => "NUMA topology.",
 778 };
 779 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 780
 781 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 782     $confdesc->{"numa$i"} = $numadesc;
 783 }
 784
 785 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 786                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 787                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 788 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 789
 790 my $net_fmt_bridge_descr = <<__EOD__;
 791 Bridge to attach the network device to. The Proxmox VE standard bridge
 792 is called 'vmbr0'.
 793
 794 If you do not specify a bridge, we create a kvm user (NATed) network
 795 device, which provides DHCP and DNS services. The following addresses
 796 are used:
 797
 798  10.0.2.2   Gateway
 799  10.0.2.3   DNS Server
 800  10.0.2.4   SMB Server
 801
 802 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 803 __EOD__
 804
 805 my $net_fmt = {
 806     macaddr  => get_standard_option('mac-addr', {
 807         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 808     }),
 809     model => {
 810         type => 'string',
 811         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 812         enum => $nic_model_list,
 813         default_key => 1,
 814     },
 815     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 816     bridge => {
 817         type => 'string',
 818         description => $net_fmt_bridge_descr,
 819         format_description => 'bridge',
 820         optional => 1,
 821     },
 822     queues => {
 823         type => 'integer',
 824         minimum => 0, maximum => 16,
 825         description => 'Number of packet queues to be used on the device.',
 826         optional => 1,
 827     },
 828     rate => {
 829         type => 'number',
 830         minimum => 0,
 831         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 832         optional => 1,
 833     },
 834     tag => {
 835         type => 'integer',
 836         minimum => 1, maximum => 4094,
 837         description => 'VLAN tag to apply to packets on this interface.',
 838         optional => 1,
 839     },
 840     trunks => {
 841         type => 'string',
 842         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 843         description => 'VLAN trunks to pass through this interface.',
 844         format_description => 'vlanid[;vlanid...]',
 845         optional => 1,
 846     },
 847     firewall => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be protected by the firewall.',
 850         optional => 1,
 851     },
 852     link_down => {
 853         type => 'boolean',
 854         description => 'Whether this interface should be disconnected (like pulling the plug).',
 855         optional => 1,
 856     },
 857 };
 858
 859 my $netdesc = {
 860     optional => 1,
 861     type => 'string', format => $net_fmt,
 862     description => "Specify network devices.",
 863 };
 864
 865 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 866
 867 my $ipconfig_fmt = {
 868     ip => {
 869         type => 'string',
 870         format => 'pve-ipv4-config',
 871         format_description => 'IPv4Format/CIDR',
 872         description => 'IPv4 address in CIDR format.',
 873         optional => 1,
 874         default => 'dhcp',
 875     },
 876     gw => {
 877         type => 'string',
 878         format => 'ipv4',
 879         format_description => 'GatewayIPv4',
 880         description => 'Default gateway for IPv4 traffic.',
 881         optional => 1,
 882         requires => 'ip',
 883     },
 884     ip6 => {
 885         type => 'string',
 886         format => 'pve-ipv6-config',
 887         format_description => 'IPv6Format/CIDR',
 888         description => 'IPv6 address in CIDR format.',
 889         optional => 1,
 890         default => 'dhcp',
 891     },
 892     gw6 => {
 893         type => 'string',
 894         format => 'ipv6',
 895         format_description => 'GatewayIPv6',
 896         description => 'Default gateway for IPv6 traffic.',
 897         optional => 1,
 898         requires => 'ip6',
 899     },
 900 };
 901 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 902 my $ipconfigdesc = {
 903     optional => 1,
 904     type => 'string', format => 'pve-qm-ipconfig',
 905     description => <<'EODESCR',
 906 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 907
 908 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 909
 910 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 911 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 912
 913 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 914 EODESCR
 915 };
 916 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 917
 918 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 919     $confdesc->{"net$i"} = $netdesc;
 920     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 921 }
 922
 923 foreach my $key (keys %$confdesc_cloudinit) {
 924     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 925 }
 926
 927 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 928 sub verify_volume_id_or_qm_path {
 929     my ($volid, $noerr) = @_;
 930
 931     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 932         return $volid;
 933     }
 934
 935     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 936     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 937     if ($@) {
 938         return undef if $noerr;
 939         die $@;
 940     }
 941     return $volid;
 942 }
 943
 944 my $usb_fmt = {
 945     host => {
 946         default_key => 1,
 947         type => 'string', format => 'pve-qm-usb-device',
 948         format_description => 'HOSTUSBDEVICE|spice',
 949         description => <<EODESCR,
 950 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 951
 952  'bus-port(.port)*' (decimal numbers) or
 953  'vendor_id:product_id' (hexadeciaml numbers) or
 954  'spice'
 955
 956 You can use the 'lsusb -t' command to list existing usb devices.
 957
 958 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 959
 960 The value 'spice' can be used to add a usb redirection devices for spice.
 961 EODESCR
 962     },
 963     usb3 => {
 964         optional => 1,
 965         type => 'boolean',
 966         description => "Specifies whether if given host option is a USB3 device or port.",
 967         default => 0,
 968     },
 969 };
 970
 971 my $usbdesc = {
 972     optional => 1,
 973     type => 'string', format => $usb_fmt,
 974     description => "Configure an USB device (n is 0 to 4).",
 975 };
 976 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 977
 978 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 979 my $hostpci_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string',
 983         pattern => qr/$PCIRE(;$PCIRE)*/,
 984         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 985         description => <<EODESCR,
 986 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 987 of PCI virtual functions of the host. HOSTPCIID syntax is:
 988
 989 'bus:dev.func' (hexadecimal numbers)
 990
 991 You can us the 'lspci' command to list existing PCI devices.
 992 EODESCR
 993     },
 994     rombar => {
 995         type => 'boolean',
 996         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 997         optional => 1,
 998         default => 1,
 999     },
1000     romfile => {
1001         type => 'string',
1002         pattern => '[^,;]+',
1003         format_description => 'string',
1004         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1005         optional => 1,
1006     },
1007     pcie => {
1008         type => 'boolean',
1009         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1010         optional => 1,
1011         default => 0,
1012     },
1013     'x-vga' => {
1014         type => 'boolean',
1015         description =>  "Enable vfio-vga device support.",
1016         optional => 1,
1017         default => 0,
1018     },
1019     'mdev' => {
1020         type => 'string',
1021         format_description => 'string',
1022         pattern => '[^/\.:]+',
1023         optional => 1,
1024         description => <<EODESCR
1025 The type of mediated device to use.
1026 An instance of this type will be created on startup of the VM and
1027 will be cleaned up when the VM stops.
1028 EODESCR
1029     }
1030 };
1031 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1032
1033 my $hostpcidesc = {
1034         optional => 1,
1035         type => 'string', format => 'pve-qm-hostpci',
1036         description => "Map host PCI devices into guest.",
1037         verbose_description =>  <<EODESCR,
1038 Map host PCI devices into guest.
1039
1040 NOTE: This option allows direct access to host hardware. So it is no longer
1041 possible to migrate such machines - use with special care.
1042
1043 CAUTION: Experimental! User reported problems with this option.
1044 EODESCR
1045 };
1046 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1047
1048 my $serialdesc = {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '(/dev/.+|socket)',
1052         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1053         verbose_description =>  <<EODESCR,
1054 Create a serial device inside the VM (n is 0 to 3), and pass through a
1055 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1056 host side (use 'qm terminal' to open a terminal connection).
1057
1058 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1059
1060 CAUTION: Experimental! User reported problems with this option.
1061 EODESCR
1062 };
1063
1064 my $paralleldesc= {
1065         optional => 1,
1066         type => 'string',
1067         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1068         description =>  "Map host parallel devices (n is 0 to 2).",
1069         verbose_description =>  <<EODESCR,
1070 Map host parallel devices (n is 0 to 2).
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1073
1074 CAUTION: Experimental! User reported problems with this option.
1075 EODESCR
1076 };
1077
1078 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1079     $confdesc->{"parallel$i"} = $paralleldesc;
1080 }
1081
1082 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1083     $confdesc->{"serial$i"} = $serialdesc;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1087     $confdesc->{"hostpci$i"} = $hostpcidesc;
1088 }
1089
1090 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1091     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1092 }
1093
1094 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1095     $confdesc->{"usb$i"} = $usbdesc;
1096 }
1097
1098 my $kvm_api_version = 0;
1099
1100 sub kvm_version {
1101     return $kvm_api_version if $kvm_api_version;
1102
1103     open my $fh, '<', '/dev/kvm'
1104         or return undef;
1105
1106     # 0xae00 => KVM_GET_API_VERSION
1107     $kvm_api_version = ioctl($fh, 0xae00, 0);
1108
1109     return $kvm_api_version;
1110 }
1111
1112 my $kvm_user_version = {};
1113 my $kvm_mtime = {};
1114
1115 sub kvm_user_version {
1116     my ($binary) = @_;
1117
1118     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1119     my $st = stat($binary);
1120
1121     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1122     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1123         $cachedmtime == $st->mtime;
1124
1125     $kvm_user_version->{$binary} = 'unknown';
1126     $kvm_mtime->{$binary} = $st->mtime;
1127
1128     my $code = sub {
1129         my $line = shift;
1130         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1131             $kvm_user_version->{$binary} = $2;
1132         }
1133     };
1134
1135     eval { run_command([$binary, '--version'], outfunc => $code); };
1136     warn $@ if $@;
1137
1138     return $kvm_user_version->{$binary};
1139
1140 }
1141
1142 sub kernel_has_vhost_net {
1143     return -c '/dev/vhost-net';
1144 }
1145
1146 sub option_exists {
1147     my $key = shift;
1148     return defined($confdesc->{$key});
1149 }
1150
1151 my $cdrom_path;
1152 sub get_cdrom_path {
1153
1154     return  $cdrom_path if $cdrom_path;
1155
1156     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1157     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1158     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1159 }
1160
1161 sub get_iso_path {
1162     my ($storecfg, $vmid, $cdrom) = @_;
1163
1164     if ($cdrom eq 'cdrom') {
1165         return get_cdrom_path();
1166     } elsif ($cdrom eq 'none') {
1167         return '';
1168     } elsif ($cdrom =~ m|^/|) {
1169         return $cdrom;
1170     } else {
1171         return PVE::Storage::path($storecfg, $cdrom);
1172     }
1173 }
1174
1175 # try to convert old style file names to volume IDs
1176 sub filename_to_volume_id {
1177     my ($vmid, $file, $media) = @_;
1178
1179      if (!($file eq 'none' || $file eq 'cdrom' ||
1180           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1181
1182         return undef if $file =~ m|/|;
1183
1184         if ($media && $media eq 'cdrom') {
1185             $file = "local:iso/$file";
1186         } else {
1187             $file = "local:$vmid/$file";
1188         }
1189     }
1190
1191     return $file;
1192 }
1193
1194 sub verify_media_type {
1195     my ($opt, $vtype, $media) = @_;
1196
1197     return if !$media;
1198
1199     my $etype;
1200     if ($media eq 'disk') {
1201         $etype = 'images';
1202     } elsif ($media eq 'cdrom') {
1203         $etype = 'iso';
1204     } else {
1205         die "internal error";
1206     }
1207
1208     return if ($vtype eq $etype);
1209
1210     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1211 }
1212
1213 sub cleanup_drive_path {
1214     my ($opt, $storecfg, $drive) = @_;
1215
1216     # try to convert filesystem paths to volume IDs
1217
1218     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1219         ($drive->{file} !~ m|^/dev/.+|) &&
1220         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1221         ($drive->{file} !~ m/^\d+$/)) {
1222         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1223         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1224         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1225         verify_media_type($opt, $vtype, $drive->{media});
1226         $drive->{file} = $volid;
1227     }
1228
1229     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1230 }
1231
1232 sub parse_hotplug_features {
1233     my ($data) = @_;
1234
1235     my $res = {};
1236
1237     return $res if $data eq '0';
1238
1239     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1240
1241     foreach my $feature (PVE::Tools::split_list($data)) {
1242         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1243             $res->{$1} = 1;
1244         } else {
1245             die "invalid hotplug feature '$feature'\n";
1246         }
1247     }
1248     return $res;
1249 }
1250
1251 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1252 sub pve_verify_hotplug_features {
1253     my ($value, $noerr) = @_;
1254
1255     return $value if parse_hotplug_features($value);
1256
1257     return undef if $noerr;
1258
1259     die "unable to parse hotplug option\n";
1260 }
1261
1262 sub scsi_inquiry {
1263     my($fh, $noerr) = @_;
1264
1265     my $SG_IO = 0x2285;
1266     my $SG_GET_VERSION_NUM = 0x2282;
1267
1268     my $versionbuf = "\x00" x 8;
1269     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1270     if (!$ret) {
1271         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1272         return undef;
1273     }
1274     my $version = unpack("I", $versionbuf);
1275     if ($version < 30000) {
1276         die "scsi generic interface too old\n"  if !$noerr;
1277         return undef;
1278     }
1279
1280     my $buf = "\x00" x 36;
1281     my $sensebuf = "\x00" x 8;
1282     my $cmd = pack("C x3 C x1", 0x12, 36);
1283
1284     # see /usr/include/scsi/sg.h
1285     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1286
1287     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1288                       length($sensebuf), 0, length($buf), $buf,
1289                       $cmd, $sensebuf, 6000);
1290
1291     $ret = ioctl($fh, $SG_IO, $packet);
1292     if (!$ret) {
1293         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1294         return undef;
1295     }
1296
1297     my @res = unpack($sg_io_hdr_t, $packet);
1298     if ($res[17] || $res[18]) {
1299         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1300         return undef;
1301     }
1302
1303     my $res = {};
1304     (my $byte0, my $byte1, $res->{vendor},
1305      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1306
1307     $res->{removable} = $byte1 & 128 ? 1 : 0;
1308     $res->{type} = $byte0 & 31;
1309
1310     return $res;
1311 }
1312
1313 sub path_is_scsi {
1314     my ($path) = @_;
1315
1316     my $fh = IO::File->new("+<$path") || return undef;
1317     my $res = scsi_inquiry($fh, 1);
1318     close($fh);
1319
1320     return $res;
1321 }
1322
1323 sub print_tabletdevice_full {
1324     my ($conf, $arch) = @_;
1325
1326     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1327
1328     # we use uhci for old VMs because tablet driver was buggy in older qemu
1329     my $usbbus;
1330     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1331         $usbbus = 'ehci';
1332     } else {
1333         $usbbus = 'uhci';
1334     }
1335
1336     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1337 }
1338
1339 sub print_keyboarddevice_full {
1340     my ($conf, $arch, $machine) = @_;
1341
1342     return undef if $arch ne 'aarch64';
1343
1344     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1345 }
1346
1347 sub print_drivedevice_full {
1348     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1349
1350     my $device = '';
1351     my $maxdev = 0;
1352
1353     if ($drive->{interface} eq 'virtio') {
1354         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1355         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1356         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1357     } elsif ($drive->{interface} eq 'scsi') {
1358
1359         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1360         my $unit = $drive->{index} % $maxdev;
1361         my $devicetype = 'hd';
1362         my $path = '';
1363         if (drive_is_cdrom($drive)) {
1364             $devicetype = 'cd';
1365         } else {
1366             if ($drive->{file} =~ m|^/|) {
1367                 $path = $drive->{file};
1368                 if (my $info = path_is_scsi($path)) {
1369                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1370                         $devicetype = 'block';
1371                     } elsif ($info->{type} == 1) { # tape
1372                         $devicetype = 'generic';
1373                     }
1374                 }
1375             } else {
1376                  $path = PVE::Storage::path($storecfg, $drive->{file});
1377             }
1378
1379             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1380             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1381             if ($path =~ m/^iscsi\:\/\// &&
1382                !min_version($version, 4, 1)) {
1383                 $devicetype = 'generic';
1384             }
1385         }
1386
1387         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1388             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1389         } else {
1390             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1391         }
1392
1393         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1394             $device .= ",rotation_rate=1";
1395         }
1396         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1397
1398     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1399         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1400         my $controller = int($drive->{index} / $maxdev);
1401         my $unit = $drive->{index} % $maxdev;
1402         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1403
1404         $device = "ide-$devicetype";
1405         if ($drive->{interface} eq 'ide') {
1406             $device .= ",bus=ide.$controller,unit=$unit";
1407         } else {
1408             $device .= ",bus=ahci$controller.$unit";
1409         }
1410         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1411
1412         if ($devicetype eq 'hd') {
1413             if (my $model = $drive->{model}) {
1414                 $model = URI::Escape::uri_unescape($model);
1415                 $device .= ",model=$model";
1416             }
1417             if ($drive->{ssd}) {
1418                 $device .= ",rotation_rate=1";
1419             }
1420         }
1421         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1422     } elsif ($drive->{interface} eq 'usb') {
1423         die "implement me";
1424         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1425     } else {
1426         die "unsupported interface type";
1427     }
1428
1429     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1430
1431     if (my $serial = $drive->{serial}) {
1432         $serial = URI::Escape::uri_unescape($serial);
1433         $device .= ",serial=$serial";
1434     }
1435
1436
1437     return $device;
1438 }
1439
1440 sub get_initiator_name {
1441     my $initiator;
1442
1443     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1444     while (defined(my $line = <$fh>)) {
1445         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1446         $initiator = $1;
1447         last;
1448     }
1449     $fh->close();
1450
1451     return $initiator;
1452 }
1453
1454 sub print_drive_commandline_full {
1455     my ($storecfg, $vmid, $drive) = @_;
1456
1457     my $path;
1458     my $volid = $drive->{file};
1459     my $format;
1460
1461     if (drive_is_cdrom($drive)) {
1462         $path = get_iso_path($storecfg, $vmid, $volid);
1463     } else {
1464         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1465         if ($storeid) {
1466             $path = PVE::Storage::path($storecfg, $volid);
1467             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1468             $format = qemu_img_format($scfg, $volname);
1469         } else {
1470             $path = $volid;
1471             $format = "raw";
1472         }
1473    }
1474
1475     my $opts = '';
1476     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1477     foreach my $o (@qemu_drive_options) {
1478         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1479     }
1480
1481     # snapshot only accepts on|off
1482     if (defined($drive->{snapshot})) {
1483         my $v = $drive->{snapshot} ? 'on' : 'off';
1484         $opts .= ",snapshot=$v";
1485     }
1486
1487     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1488         my ($dir, $qmpname) = @$type;
1489         if (my $v = $drive->{"mbps$dir"}) {
1490             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1491         }
1492         if (my $v = $drive->{"mbps${dir}_max"}) {
1493             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1494         }
1495         if (my $v = $drive->{"bps${dir}_max_length"}) {
1496             $opts .= ",throttling.bps$qmpname-max-length=$v";
1497         }
1498         if (my $v = $drive->{"iops${dir}"}) {
1499             $opts .= ",throttling.iops$qmpname=$v";
1500         }
1501         if (my $v = $drive->{"iops${dir}_max"}) {
1502             $opts .= ",throttling.iops$qmpname-max=$v";
1503         }
1504         if (my $v = $drive->{"iops${dir}_max_length"}) {
1505             $opts .= ",throttling.iops$qmpname-max-length=$v";
1506         }
1507     }
1508
1509     $opts .= ",format=$format" if $format && !$drive->{format};
1510
1511     my $cache_direct = 0;
1512
1513     if (my $cache = $drive->{cache}) {
1514         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1515     } elsif (!drive_is_cdrom($drive)) {
1516         $opts .= ",cache=none";
1517         $cache_direct = 1;
1518     }
1519
1520     # aio native works only with O_DIRECT
1521     if (!$drive->{aio}) {
1522         if($cache_direct) {
1523             $opts .= ",aio=native";
1524         } else {
1525             $opts .= ",aio=threads";
1526         }
1527     }
1528
1529     if (!drive_is_cdrom($drive)) {
1530         my $detectzeroes;
1531         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1532             $detectzeroes = 'off';
1533         } elsif ($drive->{discard}) {
1534             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1535         } else {
1536             # This used to be our default with discard not being specified:
1537             $detectzeroes = 'on';
1538         }
1539         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1540     }
1541
1542     my $pathinfo = $path ? "file=$path," : '';
1543
1544     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1545 }
1546
1547 sub print_netdevice_full {
1548     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1549
1550     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1551
1552     my $device = $net->{model};
1553     if ($net->{model} eq 'virtio') {
1554          $device = 'virtio-net-pci';
1555      };
1556
1557     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1558     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1559     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1560         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1561         my $vectors = $net->{queues} * 2 + 2;
1562         $tmpstr .= ",vectors=$vectors,mq=on";
1563     }
1564     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1565
1566     if ($use_old_bios_files) {
1567         my $romfile;
1568         if ($device eq 'virtio-net-pci') {
1569             $romfile = 'pxe-virtio.rom';
1570         } elsif ($device eq 'e1000') {
1571             $romfile = 'pxe-e1000.rom';
1572         } elsif ($device eq 'ne2k') {
1573             $romfile = 'pxe-ne2k_pci.rom';
1574         } elsif ($device eq 'pcnet') {
1575             $romfile = 'pxe-pcnet.rom';
1576         } elsif ($device eq 'rtl8139') {
1577             $romfile = 'pxe-rtl8139.rom';
1578         }
1579         $tmpstr .= ",romfile=$romfile" if $romfile;
1580     }
1581
1582     return $tmpstr;
1583 }
1584
1585 sub print_netdev_full {
1586     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1587
1588     my $i = '';
1589     if ($netid =~ m/^net(\d+)$/) {
1590         $i = int($1);
1591     }
1592
1593     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1594
1595     my $ifname = "tap${vmid}i$i";
1596
1597     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1598     die "interface name '$ifname' is too long (max 15 character)\n"
1599         if length($ifname) >= 16;
1600
1601     my $vhostparam = '';
1602     if (is_native($arch)) {
1603         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1604     }
1605
1606     my $vmname = $conf->{name} || "vm$vmid";
1607
1608     my $netdev = "";
1609     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1610
1611     if ($net->{bridge}) {
1612         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1613     } else {
1614         $netdev = "type=user,id=$netid,hostname=$vmname";
1615     }
1616
1617     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1618
1619     return $netdev;
1620 }
1621
1622 my $vga_map = {
1623     'cirrus' => 'cirrus-vga',
1624     'std' => 'VGA',
1625     'vmware' => 'vmware-svga',
1626     'virtio' => 'virtio-vga',
1627 };
1628
1629 sub print_vga_device {
1630     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1631
1632     my $type = $vga_map->{$vga->{type}};
1633     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1634         $type = 'virtio-gpu';
1635     }
1636     my $vgamem_mb = $vga->{memory};
1637
1638     my $max_outputs = '';
1639     if ($qxlnum) {
1640         $type = $id ? 'qxl' : 'qxl-vga';
1641
1642         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1643             # set max outputs so linux can have up to 4 qxl displays with one device
1644             if (min_version($machine_version, 4, 1)) {
1645                 $max_outputs = ",max_outputs=4";
1646             }
1647         }
1648     }
1649
1650     die "no devicetype for $vga->{type}\n" if !$type;
1651
1652     my $memory = "";
1653     if ($vgamem_mb) {
1654         if ($vga->{type} eq 'virtio') {
1655             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1656             $memory = ",max_hostmem=$bytes";
1657         } elsif ($qxlnum) {
1658             # from https://www.spice-space.org/multiple-monitors.html
1659             $memory = ",vgamem_mb=$vga->{memory}";
1660             my $ram = $vgamem_mb * 4;
1661             my $vram = $vgamem_mb * 2;
1662             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1663         } else {
1664             $memory = ",vgamem_mb=$vga->{memory}";
1665         }
1666     } elsif ($qxlnum && $id) {
1667         $memory = ",ram_size=67108864,vram_size=33554432";
1668     }
1669
1670     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1671     my $vgaid = "vga" . ($id // '');
1672     my $pciaddr;
1673
1674     if ($q35 && $vgaid eq 'vga') {
1675         # the first display uses pcie.0 bus on q35 machines
1676         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1677     } else {
1678         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1679     }
1680
1681     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1682 }
1683
1684 sub parse_number_sets {
1685     my ($set) = @_;
1686     my $res = [];
1687     foreach my $part (split(/;/, $set)) {
1688         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1689             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1690             push @$res, [ $1, $2 ];
1691         } else {
1692             die "invalid range: $part\n";
1693         }
1694     }
1695     return $res;
1696 }
1697
1698 sub parse_numa {
1699     my ($data) = @_;
1700
1701     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1702     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1703     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1704     return $res;
1705 }
1706
1707 sub parse_hostpci {
1708     my ($value) = @_;
1709
1710     return undef if !$value;
1711
1712     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1713
1714     my @idlist = split(/;/, $res->{host});
1715     delete $res->{host};
1716     foreach my $id (@idlist) {
1717         my $devs = PVE::SysFSTools::lspci($id);
1718         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1719         push @{$res->{pciid}}, @$devs;
1720     }
1721     return $res;
1722 }
1723
1724 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1725 sub parse_net {
1726     my ($data) = @_;
1727
1728     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1729     if ($@) {
1730         warn $@;
1731         return undef;
1732     }
1733     if (!defined($res->{macaddr})) {
1734         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1735         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1736     }
1737     return $res;
1738 }
1739
1740 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1741 sub parse_ipconfig {
1742     my ($data) = @_;
1743
1744     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1745     if ($@) {
1746         warn $@;
1747         return undef;
1748     }
1749
1750     if ($res->{gw} && !$res->{ip}) {
1751         warn 'gateway specified without specifying an IP address';
1752         return undef;
1753     }
1754     if ($res->{gw6} && !$res->{ip6}) {
1755         warn 'IPv6 gateway specified without specifying an IPv6 address';
1756         return undef;
1757     }
1758     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1759         warn 'gateway specified together with DHCP';
1760         return undef;
1761     }
1762     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1763         # gw6 + auto/dhcp
1764         warn "IPv6 gateway specified together with $res->{ip6} address";
1765         return undef;
1766     }
1767
1768     if (!$res->{ip} && !$res->{ip6}) {
1769         return { ip => 'dhcp', ip6 => 'dhcp' };
1770     }
1771
1772     return $res;
1773 }
1774
1775 sub print_net {
1776     my $net = shift;
1777
1778     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1779 }
1780
1781 sub add_random_macs {
1782     my ($settings) = @_;
1783
1784     foreach my $opt (keys %$settings) {
1785         next if $opt !~ m/^net(\d+)$/;
1786         my $net = parse_net($settings->{$opt});
1787         next if !$net;
1788         $settings->{$opt} = print_net($net);
1789     }
1790 }
1791
1792 sub vm_is_volid_owner {
1793     my ($storecfg, $vmid, $volid) = @_;
1794
1795     if ($volid !~  m|^/|) {
1796         my ($path, $owner);
1797         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1798         if ($owner && ($owner == $vmid)) {
1799             return 1;
1800         }
1801     }
1802
1803     return undef;
1804 }
1805
1806 sub vmconfig_register_unused_drive {
1807     my ($storecfg, $vmid, $conf, $drive) = @_;
1808
1809     if (drive_is_cloudinit($drive)) {
1810         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1811         warn $@ if $@;
1812     } elsif (!drive_is_cdrom($drive)) {
1813         my $volid = $drive->{file};
1814         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1815             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1816         }
1817     }
1818 }
1819
1820 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1821 my $smbios1_fmt = {
1822     uuid => {
1823         type => 'string',
1824         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1825         format_description => 'UUID',
1826         description => "Set SMBIOS1 UUID.",
1827         optional => 1,
1828     },
1829     version => {
1830         type => 'string',
1831         pattern => '[A-Za-z0-9+\/]+={0,2}',
1832         format_description => 'Base64 encoded string',
1833         description => "Set SMBIOS1 version.",
1834         optional => 1,
1835     },
1836     serial => {
1837         type => 'string',
1838         pattern => '[A-Za-z0-9+\/]+={0,2}',
1839         format_description => 'Base64 encoded string',
1840         description => "Set SMBIOS1 serial number.",
1841         optional => 1,
1842     },
1843     manufacturer => {
1844         type => 'string',
1845         pattern => '[A-Za-z0-9+\/]+={0,2}',
1846         format_description => 'Base64 encoded string',
1847         description => "Set SMBIOS1 manufacturer.",
1848         optional => 1,
1849     },
1850     product => {
1851         type => 'string',
1852         pattern => '[A-Za-z0-9+\/]+={0,2}',
1853         format_description => 'Base64 encoded string',
1854         description => "Set SMBIOS1 product ID.",
1855         optional => 1,
1856     },
1857     sku => {
1858         type => 'string',
1859         pattern => '[A-Za-z0-9+\/]+={0,2}',
1860         format_description => 'Base64 encoded string',
1861         description => "Set SMBIOS1 SKU string.",
1862         optional => 1,
1863     },
1864     family => {
1865         type => 'string',
1866         pattern => '[A-Za-z0-9+\/]+={0,2}',
1867         format_description => 'Base64 encoded string',
1868         description => "Set SMBIOS1 family string.",
1869         optional => 1,
1870     },
1871     base64 => {
1872         type => 'boolean',
1873         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1874         optional => 1,
1875     },
1876 };
1877
1878 sub parse_smbios1 {
1879     my ($data) = @_;
1880
1881     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1882     warn $@ if $@;
1883     return $res;
1884 }
1885
1886 sub print_smbios1 {
1887     my ($smbios1) = @_;
1888     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1889 }
1890
1891 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1892
1893 sub parse_watchdog {
1894     my ($value) = @_;
1895
1896     return undef if !$value;
1897
1898     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1899     warn $@ if $@;
1900     return $res;
1901 }
1902
1903 sub parse_guest_agent {
1904     my ($value) = @_;
1905
1906     return {} if !defined($value->{agent});
1907
1908     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1909     warn $@ if $@;
1910
1911     # if the agent is disabled ignore the other potentially set properties
1912     return {} if !$res->{enabled};
1913     return $res;
1914 }
1915
1916 sub parse_vga {
1917     my ($value) = @_;
1918
1919     return {} if !$value;
1920     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1921     warn $@ if $@;
1922     return $res;
1923 }
1924
1925 sub parse_rng {
1926     my ($value) = @_;
1927
1928     return undef if !$value;
1929
1930     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1931     warn $@ if $@;
1932     return $res;
1933 }
1934
1935 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1936 sub verify_usb_device {
1937     my ($value, $noerr) = @_;
1938
1939     return $value if parse_usb_device($value);
1940
1941     return undef if $noerr;
1942
1943     die "unable to parse usb device\n";
1944 }
1945
1946 # add JSON properties for create and set function
1947 sub json_config_properties {
1948     my $prop = shift;
1949
1950     foreach my $opt (keys %$confdesc) {
1951         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1952         $prop->{$opt} = $confdesc->{$opt};
1953     }
1954
1955     return $prop;
1956 }
1957
1958 # return copy of $confdesc_cloudinit to generate documentation
1959 sub cloudinit_config_properties {
1960
1961     return dclone($confdesc_cloudinit);
1962 }
1963
1964 sub check_type {
1965     my ($key, $value) = @_;
1966
1967     die "unknown setting '$key'\n" if !$confdesc->{$key};
1968
1969     my $type = $confdesc->{$key}->{type};
1970
1971     if (!defined($value)) {
1972         die "got undefined value\n";
1973     }
1974
1975     if ($value =~ m/[\n\r]/) {
1976         die "property contains a line feed\n";
1977     }
1978
1979     if ($type eq 'boolean') {
1980         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1981         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1982         die "type check ('boolean') failed - got '$value'\n";
1983     } elsif ($type eq 'integer') {
1984         return int($1) if $value =~ m/^(\d+)$/;
1985         die "type check ('integer') failed - got '$value'\n";
1986     } elsif ($type eq 'number') {
1987         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1988         die "type check ('number') failed - got '$value'\n";
1989     } elsif ($type eq 'string') {
1990         if (my $fmt = $confdesc->{$key}->{format}) {
1991             PVE::JSONSchema::check_format($fmt, $value);
1992             return $value;
1993         }
1994         $value =~ s/^\"(.*)\"$/$1/;
1995         return $value;
1996     } else {
1997         die "internal error"
1998     }
1999 }
2000
2001 sub destroy_vm {
2002     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2003
2004     my $conf = PVE::QemuConfig->load_config($vmid);
2005
2006     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2007
2008     if ($conf->{template}) {
2009         # check if any base image is still used by a linked clone
2010         foreach_drive($conf, sub {
2011                 my ($ds, $drive) = @_;
2012                 return if drive_is_cdrom($drive);
2013
2014                 my $volid = $drive->{file};
2015                 return if !$volid || $volid =~ m|^/|;
2016
2017                 die "base volume '$volid' is still in use by linked cloned\n"
2018                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2019
2020         });
2021     }
2022
2023     # only remove disks owned by this VM
2024     foreach_drive($conf, sub {
2025         my ($ds, $drive) = @_;
2026         return if drive_is_cdrom($drive, 1);
2027
2028         my $volid = $drive->{file};
2029         return if !$volid || $volid =~ m|^/|;
2030
2031         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2032         return if !$path || !$owner || ($owner != $vmid);
2033
2034         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2035         warn "Could not remove disk '$volid', check manually: $@" if $@;
2036     });
2037
2038     # also remove unused disk
2039     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2040     PVE::Storage::foreach_volid($vmdisks, sub {
2041         my ($volid, $sid, $volname, $d) = @_;
2042         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2043         warn $@ if $@;
2044     });
2045
2046     if (defined $replacement_conf) {
2047         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2048     } else {
2049         PVE::QemuConfig->destroy_config($vmid);
2050     }
2051 }
2052
2053 sub parse_vm_config {
2054     my ($filename, $raw) = @_;
2055
2056     return undef if !defined($raw);
2057
2058     my $res = {
2059         digest => Digest::SHA::sha1_hex($raw),
2060         snapshots => {},
2061         pending => {},
2062     };
2063
2064     $filename =~ m|/qemu-server/(\d+)\.conf$|
2065         || die "got strange filename '$filename'";
2066
2067     my $vmid = $1;
2068
2069     my $conf = $res;
2070     my $descr;
2071     my $section = '';
2072
2073     my @lines = split(/\n/, $raw);
2074     foreach my $line (@lines) {
2075         next if $line =~ m/^\s*$/;
2076
2077         if ($line =~ m/^\[PENDING\]\s*$/i) {
2078             $section = 'pending';
2079             if (defined($descr)) {
2080                 $descr =~ s/\s+$//;
2081                 $conf->{description} = $descr;
2082             }
2083             $descr = undef;
2084             $conf = $res->{$section} = {};
2085             next;
2086
2087         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2088             $section = $1;
2089             if (defined($descr)) {
2090                 $descr =~ s/\s+$//;
2091                 $conf->{description} = $descr;
2092             }
2093             $descr = undef;
2094             $conf = $res->{snapshots}->{$section} = {};
2095             next;
2096         }
2097
2098         if ($line =~ m/^\#(.*)\s*$/) {
2099             $descr = '' if !defined($descr);
2100             $descr .= PVE::Tools::decode_text($1) . "\n";
2101             next;
2102         }
2103
2104         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2105             $descr = '' if !defined($descr);
2106             $descr .= PVE::Tools::decode_text($2);
2107         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2108             $conf->{snapstate} = $1;
2109         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2110             my $key = $1;
2111             my $value = $2;
2112             $conf->{$key} = $value;
2113         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2114             my $value = $1;
2115             if ($section eq 'pending') {
2116                 $conf->{delete} = $value; # we parse this later
2117             } else {
2118                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2119             }
2120         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2121             my $key = $1;
2122             my $value = $2;
2123             eval { $value = check_type($key, $value); };
2124             if ($@) {
2125                 warn "vm $vmid - unable to parse value of '$key' - $@";
2126             } else {
2127                 $key = 'ide2' if $key eq 'cdrom';
2128                 my $fmt = $confdesc->{$key}->{format};
2129                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2130                     my $v = parse_drive($key, $value);
2131                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2132                         $v->{file} = $volid;
2133                         $value = print_drive($v);
2134                     } else {
2135                         warn "vm $vmid - unable to parse value of '$key'\n";
2136                         next;
2137                     }
2138                 }
2139
2140                 $conf->{$key} = $value;
2141             }
2142         }
2143     }
2144
2145     if (defined($descr)) {
2146         $descr =~ s/\s+$//;
2147         $conf->{description} = $descr;
2148     }
2149     delete $res->{snapstate}; # just to be sure
2150
2151     return $res;
2152 }
2153
2154 sub write_vm_config {
2155     my ($filename, $conf) = @_;
2156
2157     delete $conf->{snapstate}; # just to be sure
2158
2159     if ($conf->{cdrom}) {
2160         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2161         $conf->{ide2} = $conf->{cdrom};
2162         delete $conf->{cdrom};
2163     }
2164
2165     # we do not use 'smp' any longer
2166     if ($conf->{sockets}) {
2167         delete $conf->{smp};
2168     } elsif ($conf->{smp}) {
2169         $conf->{sockets} = $conf->{smp};
2170         delete $conf->{cores};
2171         delete $conf->{smp};
2172     }
2173
2174     my $used_volids = {};
2175
2176     my $cleanup_config = sub {
2177         my ($cref, $pending, $snapname) = @_;
2178
2179         foreach my $key (keys %$cref) {
2180             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2181                 $key eq 'snapstate' || $key eq 'pending';
2182             my $value = $cref->{$key};
2183             if ($key eq 'delete') {
2184                 die "propertry 'delete' is only allowed in [PENDING]\n"
2185                     if !$pending;
2186                 # fixme: check syntax?
2187                 next;
2188             }
2189             eval { $value = check_type($key, $value); };
2190             die "unable to parse value of '$key' - $@" if $@;
2191
2192             $cref->{$key} = $value;
2193
2194             if (!$snapname && is_valid_drivename($key)) {
2195                 my $drive = parse_drive($key, $value);
2196                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2197             }
2198         }
2199     };
2200
2201     &$cleanup_config($conf);
2202
2203     &$cleanup_config($conf->{pending}, 1);
2204
2205     foreach my $snapname (keys %{$conf->{snapshots}}) {
2206         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2207         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2208     }
2209
2210     # remove 'unusedX' settings if we re-add a volume
2211     foreach my $key (keys %$conf) {
2212         my $value = $conf->{$key};
2213         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2214             delete $conf->{$key};
2215         }
2216     }
2217
2218     my $generate_raw_config = sub {
2219         my ($conf, $pending) = @_;
2220
2221         my $raw = '';
2222
2223         # add description as comment to top of file
2224         if (defined(my $descr = $conf->{description})) {
2225             if ($descr) {
2226                 foreach my $cl (split(/\n/, $descr)) {
2227                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2228                 }
2229             } else {
2230                 $raw .= "#\n" if $pending;
2231             }
2232         }
2233
2234         foreach my $key (sort keys %$conf) {
2235             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2236             $raw .= "$key: $conf->{$key}\n";
2237         }
2238         return $raw;
2239     };
2240
2241     my $raw = &$generate_raw_config($conf);
2242
2243     if (scalar(keys %{$conf->{pending}})){
2244         $raw .= "\n[PENDING]\n";
2245         $raw .= &$generate_raw_config($conf->{pending}, 1);
2246     }
2247
2248     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2249         $raw .= "\n[$snapname]\n";
2250         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2251     }
2252
2253     return $raw;
2254 }
2255
2256 sub load_defaults {
2257
2258     my $res = {};
2259
2260     # we use static defaults from our JSON schema configuration
2261     foreach my $key (keys %$confdesc) {
2262         if (defined(my $default = $confdesc->{$key}->{default})) {
2263             $res->{$key} = $default;
2264         }
2265     }
2266
2267     return $res;
2268 }
2269
2270 sub config_list {
2271     my $vmlist = PVE::Cluster::get_vmlist();
2272     my $res = {};
2273     return $res if !$vmlist || !$vmlist->{ids};
2274     my $ids = $vmlist->{ids};
2275     my $nodename = nodename();
2276
2277     foreach my $vmid (keys %$ids) {
2278         my $d = $ids->{$vmid};
2279         next if !$d->{node} || $d->{node} ne $nodename;
2280         next if !$d->{type} || $d->{type} ne 'qemu';
2281         $res->{$vmid}->{exists} = 1;
2282     }
2283     return $res;
2284 }
2285
2286 # test if VM uses local resources (to prevent migration)
2287 sub check_local_resources {
2288     my ($conf, $noerr) = @_;
2289
2290     my @loc_res = ();
2291
2292     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2293     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2294
2295     push @loc_res, "ivshmem" if $conf->{ivshmem};
2296
2297     foreach my $k (keys %$conf) {
2298         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2299         # sockets are safe: they will recreated be on the target side post-migrate
2300         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2301         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2302     }
2303
2304     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2305
2306     return \@loc_res;
2307 }
2308
2309 # check if used storages are available on all nodes (use by migrate)
2310 sub check_storage_availability {
2311     my ($storecfg, $conf, $node) = @_;
2312
2313     foreach_drive($conf, sub {
2314         my ($ds, $drive) = @_;
2315
2316         my $volid = $drive->{file};
2317         return if !$volid;
2318
2319         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2320         return if !$sid;
2321
2322         # check if storage is available on both nodes
2323         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2324         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2325    });
2326 }
2327
2328 # list nodes where all VM images are available (used by has_feature API)
2329 sub shared_nodes {
2330     my ($conf, $storecfg) = @_;
2331
2332     my $nodelist = PVE::Cluster::get_nodelist();
2333     my $nodehash = { map { $_ => 1 } @$nodelist };
2334     my $nodename = nodename();
2335
2336     foreach_drive($conf, sub {
2337         my ($ds, $drive) = @_;
2338
2339         my $volid = $drive->{file};
2340         return if !$volid;
2341
2342         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2343         if ($storeid) {
2344             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2345             if ($scfg->{disable}) {
2346                 $nodehash = {};
2347             } elsif (my $avail = $scfg->{nodes}) {
2348                 foreach my $node (keys %$nodehash) {
2349                     delete $nodehash->{$node} if !$avail->{$node};
2350                 }
2351             } elsif (!$scfg->{shared}) {
2352                 foreach my $node (keys %$nodehash) {
2353                     delete $nodehash->{$node} if $node ne $nodename
2354                 }
2355             }
2356         }
2357     });
2358
2359     return $nodehash
2360 }
2361
2362 sub check_local_storage_availability {
2363     my ($conf, $storecfg) = @_;
2364
2365     my $nodelist = PVE::Cluster::get_nodelist();
2366     my $nodehash = { map { $_ => {} } @$nodelist };
2367
2368     foreach_drive($conf, sub {
2369         my ($ds, $drive) = @_;
2370
2371         my $volid = $drive->{file};
2372         return if !$volid;
2373
2374         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2375         if ($storeid) {
2376             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2377
2378             if ($scfg->{disable}) {
2379                 foreach my $node (keys %$nodehash) {
2380                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2381                 }
2382             } elsif (my $avail = $scfg->{nodes}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     if (!$avail->{$node}) {
2385                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2386                     }
2387                 }
2388             }
2389         }
2390     });
2391
2392     foreach my $node (values %$nodehash) {
2393         if (my $unavail = $node->{unavailable_storages}) {
2394             $node->{unavailable_storages} = [ sort keys %$unavail ];
2395         }
2396     }
2397
2398     return $nodehash
2399 }
2400
2401 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2402 sub check_running {
2403     my ($vmid, $nocheck, $node) = @_;
2404
2405     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2406     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2407 }
2408
2409 sub vzlist {
2410
2411     my $vzlist = config_list();
2412
2413     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2414
2415     while (defined(my $de = $fd->read)) {
2416         next if $de !~ m/^(\d+)\.pid$/;
2417         my $vmid = $1;
2418         next if !defined($vzlist->{$vmid});
2419         if (my $pid = check_running($vmid)) {
2420             $vzlist->{$vmid}->{pid} = $pid;
2421         }
2422     }
2423
2424     return $vzlist;
2425 }
2426
2427 our $vmstatus_return_properties = {
2428     vmid => get_standard_option('pve-vmid'),
2429     status => {
2430         description => "Qemu process status.",
2431         type => 'string',
2432         enum => ['stopped', 'running'],
2433     },
2434     maxmem => {
2435         description => "Maximum memory in bytes.",
2436         type => 'integer',
2437         optional => 1,
2438         renderer => 'bytes',
2439     },
2440     maxdisk => {
2441         description => "Root disk size in bytes.",
2442         type => 'integer',
2443         optional => 1,
2444         renderer => 'bytes',
2445     },
2446     name => {
2447         description => "VM name.",
2448         type => 'string',
2449         optional => 1,
2450     },
2451     qmpstatus => {
2452         description => "Qemu QMP agent status.",
2453         type => 'string',
2454         optional => 1,
2455     },
2456     pid => {
2457         description => "PID of running qemu process.",
2458         type => 'integer',
2459         optional => 1,
2460     },
2461     uptime => {
2462         description => "Uptime.",
2463         type => 'integer',
2464         optional => 1,
2465         renderer => 'duration',
2466     },
2467     cpus => {
2468         description => "Maximum usable CPUs.",
2469         type => 'number',
2470         optional => 1,
2471     },
2472     lock => {
2473         description => "The current config lock, if any.",
2474         type => 'string',
2475         optional => 1,
2476     },
2477     tags => {
2478         description  => "The current configured tags, if any",
2479         type => 'string',
2480         optional => 1,
2481     },
2482 };
2483
2484 my $last_proc_pid_stat;
2485
2486 # get VM status information
2487 # This must be fast and should not block ($full == false)
2488 # We only query KVM using QMP if $full == true (this can be slow)
2489 sub vmstatus {
2490     my ($opt_vmid, $full) = @_;
2491
2492     my $res = {};
2493
2494     my $storecfg = PVE::Storage::config();
2495
2496     my $list = vzlist();
2497     my $defaults = load_defaults();
2498
2499     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2500
2501     my $cpucount = $cpuinfo->{cpus} || 1;
2502
2503     foreach my $vmid (keys %$list) {
2504         next if $opt_vmid && ($vmid ne $opt_vmid);
2505
2506         my $conf = PVE::QemuConfig->load_config($vmid);
2507
2508         my $d = { vmid => $vmid };
2509         $d->{pid} = $list->{$vmid}->{pid};
2510
2511         # fixme: better status?
2512         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2513
2514         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2515         if (defined($size)) {
2516             $d->{disk} = 0; # no info available
2517             $d->{maxdisk} = $size;
2518         } else {
2519             $d->{disk} = 0;
2520             $d->{maxdisk} = 0;
2521         }
2522
2523         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2524             * ($conf->{cores} || $defaults->{cores});
2525         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2526         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2527
2528         $d->{name} = $conf->{name} || "VM $vmid";
2529         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2530             : $defaults->{memory}*(1024*1024);
2531
2532         if ($conf->{balloon}) {
2533             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2534             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2535                 : $defaults->{shares};
2536         }
2537
2538         $d->{uptime} = 0;
2539         $d->{cpu} = 0;
2540         $d->{mem} = 0;
2541
2542         $d->{netout} = 0;
2543         $d->{netin} = 0;
2544
2545         $d->{diskread} = 0;
2546         $d->{diskwrite} = 0;
2547
2548         $d->{template} = PVE::QemuConfig->is_template($conf);
2549
2550         $d->{serial} = 1 if conf_has_serial($conf);
2551         $d->{lock} = $conf->{lock} if $conf->{lock};
2552         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2553
2554         $res->{$vmid} = $d;
2555     }
2556
2557     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2558     foreach my $dev (keys %$netdev) {
2559         next if $dev !~ m/^tap([1-9]\d*)i/;
2560         my $vmid = $1;
2561         my $d = $res->{$vmid};
2562         next if !$d;
2563
2564         $d->{netout} += $netdev->{$dev}->{receive};
2565         $d->{netin} += $netdev->{$dev}->{transmit};
2566
2567         if ($full) {
2568             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2569             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2570         }
2571
2572     }
2573
2574     my $ctime = gettimeofday;
2575
2576     foreach my $vmid (keys %$list) {
2577
2578         my $d = $res->{$vmid};
2579         my $pid = $d->{pid};
2580         next if !$pid;
2581
2582         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2583         next if !$pstat; # not running
2584
2585         my $used = $pstat->{utime} + $pstat->{stime};
2586
2587         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2588
2589         if ($pstat->{vsize}) {
2590             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2591         }
2592
2593         my $old = $last_proc_pid_stat->{$pid};
2594         if (!$old) {
2595             $last_proc_pid_stat->{$pid} = {
2596                 time => $ctime,
2597                 used => $used,
2598                 cpu => 0,
2599             };
2600             next;
2601         }
2602
2603         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2604
2605         if ($dtime > 1000) {
2606             my $dutime = $used -  $old->{used};
2607
2608             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2609             $last_proc_pid_stat->{$pid} = {
2610                 time => $ctime,
2611                 used => $used,
2612                 cpu => $d->{cpu},
2613             };
2614         } else {
2615             $d->{cpu} = $old->{cpu};
2616         }
2617     }
2618
2619     return $res if !$full;
2620
2621     my $qmpclient = PVE::QMPClient->new();
2622
2623     my $ballooncb = sub {
2624         my ($vmid, $resp) = @_;
2625
2626         my $info = $resp->{'return'};
2627         return if !$info->{max_mem};
2628
2629         my $d = $res->{$vmid};
2630
2631         # use memory assigned to VM
2632         $d->{maxmem} = $info->{max_mem};
2633         $d->{balloon} = $info->{actual};
2634
2635         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2636             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2637             $d->{freemem} = $info->{free_mem};
2638         }
2639
2640         $d->{ballooninfo} = $info;
2641     };
2642
2643     my $blockstatscb = sub {
2644         my ($vmid, $resp) = @_;
2645         my $data = $resp->{'return'} || [];
2646         my $totalrdbytes = 0;
2647         my $totalwrbytes = 0;
2648
2649         for my $blockstat (@$data) {
2650             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2651             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2652
2653             $blockstat->{device} =~ s/drive-//;
2654             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2655         }
2656         $res->{$vmid}->{diskread} = $totalrdbytes;
2657         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2658     };
2659
2660     my $statuscb = sub {
2661         my ($vmid, $resp) = @_;
2662
2663         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2664         # this fails if ballon driver is not loaded, so this must be
2665         # the last commnand (following command are aborted if this fails).
2666         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2667
2668         my $status = 'unknown';
2669         if (!defined($status = $resp->{'return'}->{status})) {
2670             warn "unable to get VM status\n";
2671             return;
2672         }
2673
2674         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2675     };
2676
2677     foreach my $vmid (keys %$list) {
2678         next if $opt_vmid && ($vmid ne $opt_vmid);
2679         next if !$res->{$vmid}->{pid}; # not running
2680         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2681     }
2682
2683     $qmpclient->queue_execute(undef, 2);
2684
2685     foreach my $vmid (keys %$list) {
2686         next if $opt_vmid && ($vmid ne $opt_vmid);
2687         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2688     }
2689
2690     return $res;
2691 }
2692
2693 sub conf_has_serial {
2694     my ($conf) = @_;
2695
2696     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2697         if ($conf->{"serial$i"}) {
2698             return 1;
2699         }
2700     }
2701
2702     return 0;
2703 }
2704
2705 sub conf_has_audio {
2706     my ($conf, $id) = @_;
2707
2708     $id //= 0;
2709     my $audio = $conf->{"audio$id"};
2710     return undef if !defined($audio);
2711
2712     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2713     my $audiodriver = $audioproperties->{driver} // 'spice';
2714
2715     return {
2716         dev => $audioproperties->{device},
2717         dev_id => "audiodev$id",
2718         backend => $audiodriver,
2719         backend_id => "$audiodriver-backend${id}",
2720     };
2721 }
2722
2723 sub vga_conf_has_spice {
2724     my ($vga) = @_;
2725
2726     my $vgaconf = parse_vga($vga);
2727     my $vgatype = $vgaconf->{type};
2728     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2729
2730     return $1 || 1;
2731 }
2732
2733 sub is_native($) {
2734     my ($arch) = @_;
2735     return get_host_arch() eq $arch;
2736 }
2737
2738 sub get_vm_arch {
2739     my ($conf) = @_;
2740     return $conf->{arch} // get_host_arch();
2741 }
2742
2743 my $default_machines = {
2744     x86_64 => 'pc',
2745     aarch64 => 'virt',
2746 };
2747
2748 sub get_vm_machine {
2749     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2750
2751     my $machine = $forcemachine || $conf->{machine};
2752
2753     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2754         $arch //= 'x86_64';
2755         $machine ||= $default_machines->{$arch};
2756         if ($add_pve_version) {
2757             $kvmversion //= kvm_user_version();
2758             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2759             $machine .= "+pve$pvever";
2760         }
2761     }
2762
2763     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2764         # for version-pinned machines that do not include a pve-version (e.g.
2765         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2766         $machine .= '+pve0';
2767     }
2768
2769     return $machine;
2770 }
2771
2772 sub get_ovmf_files($) {
2773     my ($arch) = @_;
2774
2775     my $ovmf = $OVMF->{$arch}
2776         or die "no OVMF images known for architecture '$arch'\n";
2777
2778     return @$ovmf;
2779 }
2780
2781 my $Arch2Qemu = {
2782     aarch64 => '/usr/bin/qemu-system-aarch64',
2783     x86_64 => '/usr/bin/qemu-system-x86_64',
2784 };
2785 sub get_command_for_arch($) {
2786     my ($arch) = @_;
2787     return '/usr/bin/kvm' if is_native($arch);
2788
2789     my $cmd = $Arch2Qemu->{$arch}
2790         or die "don't know how to emulate architecture '$arch'\n";
2791     return $cmd;
2792 }
2793
2794 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2795 # to use in a QEMU command line (-cpu element), first array_intersect the result
2796 # of query_supported_ with query_understood_. This is necessary because:
2797 #
2798 # a) query_understood_ returns flags the host cannot use and
2799 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2800 #    flags, but CPU settings - with most of them being flags. Those settings
2801 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2802 #
2803 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2804 # expensive. If you need the value returned from this, you can get it much
2805 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2806 # $accel being 'kvm' or 'tcg'.
2807 #
2808 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2809 # changes, automatically populating pmxcfs.
2810 #
2811 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2812 # since kvm and tcg machines support different flags
2813 #
2814 sub query_supported_cpu_flags {
2815     my ($arch) = @_;
2816
2817     $arch //= get_host_arch();
2818     my $default_machine = $default_machines->{$arch};
2819
2820     my $flags = {};
2821
2822     # FIXME: Once this is merged, the code below should work for ARM as well:
2823     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2824     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2825         $arch eq "aarch64";
2826
2827     my $kvm_supported = defined(kvm_version());
2828     my $qemu_cmd = get_command_for_arch($arch);
2829     my $fakevmid = -1;
2830     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2831
2832     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2833     my $query_supported_run_qemu = sub {
2834         my ($kvm) = @_;
2835
2836         my $flags = {};
2837         my $cmd = [
2838             $qemu_cmd,
2839             '-machine', $default_machine,
2840             '-display', 'none',
2841             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2842             '-mon', 'chardev=qmp,mode=control',
2843             '-pidfile', $pidfile,
2844             '-S', '-daemonize'
2845         ];
2846
2847         if (!$kvm) {
2848             push @$cmd, '-accel', 'tcg';
2849         }
2850
2851         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2852         die "QEMU flag querying VM exited with code " . $rc if $rc;
2853
2854         eval {
2855             my $cmd_result = mon_cmd(
2856                 $fakevmid,
2857                 'query-cpu-model-expansion',
2858                 type => 'full',
2859                 model => { name => 'host' }
2860             );
2861
2862             my $props = $cmd_result->{model}->{props};
2863             foreach my $prop (keys %$props) {
2864                 next if $props->{$prop} ne '1';
2865                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2866                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2867                 # We only keep those with underscores, to match /proc/cpuinfo
2868                 $prop =~ s/\.|-/_/g;
2869                 $flags->{$prop} = 1;
2870             }
2871         };
2872         my $err = $@;
2873
2874         # force stop with 10 sec timeout and 'nocheck'
2875         # always stop, even if QMP failed
2876         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2877
2878         die $err if $err;
2879
2880         return [ sort keys %$flags ];
2881     };
2882
2883     # We need to query QEMU twice, since KVM and TCG have different supported flags
2884     PVE::QemuConfig->lock_config($fakevmid, sub {
2885         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2886         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2887
2888         if ($kvm_supported) {
2889             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2890             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2891         }
2892     });
2893
2894     return $flags;
2895 }
2896
2897 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2898 my $understood_cpu_flag_dir = "/usr/share/kvm";
2899 sub query_understood_cpu_flags {
2900     my $arch = get_host_arch();
2901     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2902
2903     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2904         if ! -e $filepath;
2905
2906     my $raw = file_get_contents($filepath);
2907     $raw =~ s/^\s+|\s+$//g;
2908     my @flags = split(/\s+/, $raw);
2909
2910     return \@flags;
2911 }
2912
2913 sub config_to_command {
2914     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2915
2916     my $cmd = [];
2917     my $globalFlags = [];
2918     my $machineFlags = [];
2919     my $rtcFlags = [];
2920     my $devices = [];
2921     my $pciaddr = '';
2922     my $bridges = {};
2923     my $ostype = $conf->{ostype};
2924     my $winversion = windows_version($ostype);
2925     my $kvm = $conf->{kvm};
2926     my $nodename = nodename();
2927
2928     my $arch = get_vm_arch($conf);
2929     my $kvm_binary = get_command_for_arch($arch);
2930     my $kvmver = kvm_user_version($kvm_binary);
2931
2932     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2933         $kvmver //= "undefined";
2934         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2935     }
2936
2937     my $add_pve_version = min_version($kvmver, 4, 1);
2938
2939     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2940     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2941     $kvm //= 1 if is_native($arch);
2942
2943     $machine_version =~ m/(\d+)\.(\d+)/;
2944     my ($machine_major, $machine_minor) = ($1, $2);
2945     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2946         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2947
2948     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2949         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2950         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2951           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2952     }
2953
2954     # if a specific +pve version is required for a feature, use $version_guard
2955     # instead of min_version to allow machines to be run with the minimum
2956     # required version
2957     my $required_pve_version = 0;
2958     my $version_guard = sub {
2959         my ($major, $minor, $pve) = @_;
2960         return 0 if !min_version($machine_version, $major, $minor, $pve);
2961         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2962         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2963         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2964         return 1;
2965     };
2966
2967     if ($kvm) {
2968         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2969             if !defined kvm_version();
2970     }
2971
2972     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2973     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2974     my $use_old_bios_files = undef;
2975     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2976
2977     my $cpuunits = defined($conf->{cpuunits}) ?
2978             $conf->{cpuunits} : $defaults->{cpuunits};
2979
2980     push @$cmd, $kvm_binary;
2981
2982     push @$cmd, '-id', $vmid;
2983
2984     my $vmname = $conf->{name} || "vm$vmid";
2985
2986     push @$cmd, '-name', $vmname;
2987
2988     my $use_virtio = 0;
2989
2990     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2991     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2992     push @$cmd, '-mon', "chardev=qmp,mode=control";
2993
2994     if (min_version($machine_version, 2, 12)) {
2995         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2996         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2997     }
2998
2999     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3000
3001     push @$cmd, '-daemonize';
3002
3003     if ($conf->{smbios1}) {
3004         my $smbios_conf = parse_smbios1($conf->{smbios1});
3005         if ($smbios_conf->{base64}) {
3006             # Do not pass base64 flag to qemu
3007             delete $smbios_conf->{base64};
3008             my $smbios_string = "";
3009             foreach my $key (keys %$smbios_conf) {
3010                 my $value;
3011                 if ($key eq "uuid") {
3012                     $value = $smbios_conf->{uuid}
3013                 } else {
3014                     $value = decode_base64($smbios_conf->{$key});
3015                 }
3016                 # qemu accepts any binary data, only commas need escaping by double comma
3017                 $value =~ s/,/,,/g;
3018                 $smbios_string .= "," . $key . "=" . $value if $value;
3019             }
3020             push @$cmd, '-smbios', "type=1" . $smbios_string;
3021         } else {
3022             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3023         }
3024     }
3025
3026     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3027     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3028         die "uefi base image not found\n" if ! -f $ovmf_code;
3029
3030         my $path;
3031         my $format;
3032         if (my $efidisk = $conf->{efidisk0}) {
3033             my $d = parse_drive('efidisk0', $efidisk);
3034             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3035             $format = $d->{format};
3036             if ($storeid) {
3037                 $path = PVE::Storage::path($storecfg, $d->{file});
3038                 if (!defined($format)) {
3039                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3040                     $format = qemu_img_format($scfg, $volname);
3041                 }
3042             } else {
3043                 $path = $d->{file};
3044                 die "efidisk format must be specified\n"
3045                     if !defined($format);
3046             }
3047         } else {
3048             warn "no efidisk configured! Using temporary efivars disk.\n";
3049             $path = "/tmp/$vmid-ovmf.fd";
3050             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3051             $format = 'raw';
3052         }
3053
3054         my $size_str = "";
3055
3056         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3057             $size_str = ",size=" . (-s $ovmf_vars);
3058         }
3059
3060         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3061         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3062     }
3063
3064     # load q35 config
3065     if ($q35) {
3066         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3067         if (min_version($machine_version, 4, 0)) {
3068             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3069         } else {
3070             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3071         }
3072     }
3073
3074     if ($conf->{vmgenid}) {
3075         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3076     }
3077
3078     # add usb controllers
3079     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3080     push @$devices, @usbcontrollers if @usbcontrollers;
3081     my $vga = parse_vga($conf->{vga});
3082
3083     my $qxlnum = vga_conf_has_spice($conf->{vga});
3084     $vga->{type} = 'qxl' if $qxlnum;
3085
3086     if (!$vga->{type}) {
3087         if ($arch eq 'aarch64') {
3088             $vga->{type} = 'virtio';
3089         } elsif (min_version($machine_version, 2, 9)) {
3090             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3091         } else {
3092             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3093         }
3094     }
3095
3096     # enable absolute mouse coordinates (needed by vnc)
3097     my $tablet;
3098     if (defined($conf->{tablet})) {
3099         $tablet = $conf->{tablet};
3100     } else {
3101         $tablet = $defaults->{tablet};
3102         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3103         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3104     }
3105
3106     if ($tablet) {
3107         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3108         my $kbd = print_keyboarddevice_full($conf, $arch);
3109         push @$devices, '-device', $kbd if defined($kbd);
3110     }
3111
3112     my $kvm_off = 0;
3113     my $gpu_passthrough;
3114
3115     # host pci devices
3116     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3117         my $id = "hostpci$i";
3118         my $d = parse_hostpci($conf->{$id});
3119         next if !$d;
3120
3121         if (my $pcie = $d->{pcie}) {
3122             die "q35 machine model is not enabled" if !$q35;
3123             # win7 wants to have the pcie devices directly on the pcie bus
3124             # instead of in the root port
3125             if ($winversion == 7) {
3126                 $pciaddr = print_pcie_addr("${id}bus0");
3127             } else {
3128                 # add more root ports if needed, 4 are present by default
3129                 # by pve-q35 cfgs, rest added here on demand.
3130                 if ($i > 3) {
3131                     push @$devices, '-device', print_pcie_root_port($i);
3132                 }
3133                 $pciaddr = print_pcie_addr($id);
3134             }
3135         } else {
3136             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3137         }
3138
3139         my $xvga = '';
3140         if ($d->{'x-vga'}) {
3141             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3142             $kvm_off = 1;
3143             $vga->{type} = 'none' if !defined($conf->{vga});
3144             $gpu_passthrough = 1;
3145         }
3146
3147         my $pcidevices = $d->{pciid};
3148         my $multifunction = 1 if @$pcidevices > 1;
3149
3150         my $sysfspath;
3151         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3152             my $pci_id = $pcidevices->[0]->{id};
3153             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3154             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3155         } elsif ($d->{mdev}) {
3156             warn "ignoring mediated device '$id' with multifunction device\n";
3157         }
3158
3159         my $j=0;
3160         foreach my $pcidevice (@$pcidevices) {
3161             my $devicestr = "vfio-pci";
3162
3163             if ($sysfspath) {
3164                 $devicestr .= ",sysfsdev=$sysfspath";
3165             } else {
3166                 $devicestr .= ",host=$pcidevice->{id}";
3167             }
3168
3169             my $mf_addr = $multifunction ? ".$j" : '';
3170             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3171
3172             if ($j == 0) {
3173                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3174                 $devicestr .= "$xvga";
3175                 $devicestr .= ",multifunction=on" if $multifunction;
3176                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3177             }
3178
3179             push @$devices, '-device', $devicestr;
3180             $j++;
3181         }
3182     }
3183
3184     # usb devices
3185     my $usb_dev_features = {};
3186     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3187
3188     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3189     push @$devices, @usbdevices if @usbdevices;
3190     # serial devices
3191     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3192         if (my $path = $conf->{"serial$i"}) {
3193             if ($path eq 'socket') {
3194                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3195                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3196                 # On aarch64, serial0 is the UART device. Qemu only allows
3197                 # connecting UART devices via the '-serial' command line, as
3198                 # the device has a fixed slot on the hardware...
3199                 if ($arch eq 'aarch64' && $i == 0) {
3200                     push @$devices, '-serial', "chardev:serial$i";
3201                 } else {
3202                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3203                 }
3204             } else {
3205                 die "no such serial device\n" if ! -c $path;
3206                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3207                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3208             }
3209         }
3210     }
3211
3212     # parallel devices
3213     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3214         if (my $path = $conf->{"parallel$i"}) {
3215             die "no such parallel device\n" if ! -c $path;
3216             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3217             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3218             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3219         }
3220     }
3221
3222     if (my $audio = conf_has_audio($conf)) {
3223
3224         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3225
3226         my $id = $audio->{dev_id};
3227         if ($audio->{dev} eq 'AC97') {
3228             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3229         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3230             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3231             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3232             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3233         } else {
3234             die "unkown audio device '$audio->{dev}', implement me!";
3235         }
3236
3237         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3238     }
3239
3240     my $sockets = 1;
3241     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3242     $sockets = $conf->{sockets} if  $conf->{sockets};
3243
3244     my $cores = $conf->{cores} || 1;
3245
3246     my $maxcpus = $sockets * $cores;
3247
3248     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3249
3250     my $allowed_vcpus = $cpuinfo->{cpus};
3251
3252     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3253         if ($allowed_vcpus < $maxcpus);
3254
3255     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3256
3257         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3258         for (my $i = 2; $i <= $vcpus; $i++)  {
3259             my $cpustr = print_cpu_device($conf,$i);
3260             push @$cmd, '-device', $cpustr;
3261         }
3262
3263     } else {
3264
3265         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3266     }
3267     push @$cmd, '-nodefaults';
3268
3269     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3270
3271     my $bootindex_hash = {};
3272     my $i = 1;
3273     foreach my $o (split(//, $bootorder)) {
3274         $bootindex_hash->{$o} = $i*100;
3275         $i++;
3276     }
3277
3278     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3279
3280     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3281
3282     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3283
3284     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3285         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3286         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3287         push @$cmd,  '-vnc', "unix:$socket,password";
3288     } else {
3289         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3290         push @$cmd, '-nographic';
3291     }
3292
3293     # time drift fix
3294     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3295
3296     my $useLocaltime = $conf->{localtime};
3297
3298     if ($winversion >= 5) { # windows
3299         $useLocaltime = 1 if !defined($conf->{localtime});
3300
3301         # use time drift fix when acpi is enabled
3302         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3303             $tdf = 1 if !defined($conf->{tdf});
3304         }
3305     }
3306
3307     if ($winversion >= 6) {
3308         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3309         push @$cmd, '-no-hpet';
3310     }
3311
3312     push @$rtcFlags, 'driftfix=slew' if $tdf;
3313
3314     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3315         push @$rtcFlags, "base=$conf->{startdate}";
3316     } elsif ($useLocaltime) {
3317         push @$rtcFlags, 'base=localtime';
3318     }
3319
3320     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3321
3322     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3323
3324     push @$cmd, '-S' if $conf->{freeze};
3325
3326     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3327
3328     my $guest_agent = parse_guest_agent($conf);
3329
3330     if ($guest_agent->{enabled}) {
3331         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3332         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3333
3334         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3335             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3336             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3337             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3338         } elsif ($guest_agent->{type} eq 'isa') {
3339             push @$devices, '-device', "isa-serial,chardev=qga0";
3340         }
3341     }
3342
3343     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3344     if ($rng && &$version_guard(4, 1, 2)) {
3345         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3346         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3347
3348         my $limiter_str = "";
3349         if ($max_bytes) {
3350             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3351         }
3352
3353         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3354         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3355             if ! -e $rng->{source};
3356
3357         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3358
3359         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3360         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3361     }
3362
3363     my $spice_port;
3364
3365     if ($qxlnum) {
3366         if ($qxlnum > 1) {
3367             if ($winversion){
3368                 for(my $i = 1; $i < $qxlnum; $i++){
3369                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3370                 }
3371             } else {
3372                 # assume other OS works like Linux
3373                 my ($ram, $vram) = ("134217728", "67108864");
3374                 if ($vga->{memory}) {
3375                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3376                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3377                 }
3378                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3379                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3380             }
3381         }
3382
3383         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3384
3385         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3386         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3387         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3388
3389         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3390         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3391         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3392
3393         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3394         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3395
3396         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3397         if ($spice_enhancement->{foldersharing}) {
3398             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3399             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3400         }
3401
3402         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3403         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3404         push @$devices, '-spice', "$spice_opts";
3405     }
3406
3407     # enable balloon by default, unless explicitly disabled
3408     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3409         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3410         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3411     }
3412
3413     if ($conf->{watchdog}) {
3414         my $wdopts = parse_watchdog($conf->{watchdog});
3415         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3416         my $watchdog = $wdopts->{model} || 'i6300esb';
3417         push @$devices, '-device', "$watchdog$pciaddr";
3418         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3419     }
3420
3421     my $vollist = [];
3422     my $scsicontroller = {};
3423     my $ahcicontroller = {};
3424     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3425
3426     # Add iscsi initiator name if available
3427     if (my $initiator = get_initiator_name()) {
3428         push @$devices, '-iscsi', "initiator-name=$initiator";
3429     }
3430
3431     foreach_drive($conf, sub {
3432         my ($ds, $drive) = @_;
3433
3434         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3435             push @$vollist, $drive->{file};
3436         }
3437
3438         # ignore efidisk here, already added in bios/fw handling code above
3439         return if $drive->{interface} eq 'efidisk';
3440
3441         $use_virtio = 1 if $ds =~ m/^virtio/;
3442
3443         if (drive_is_cdrom ($drive)) {
3444             if ($bootindex_hash->{d}) {
3445                 $drive->{bootindex} = $bootindex_hash->{d};
3446                 $bootindex_hash->{d} += 1;
3447             }
3448         } else {
3449             if ($bootindex_hash->{c}) {
3450                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3451                 $bootindex_hash->{c} += 1;
3452             }
3453         }
3454
3455         if($drive->{interface} eq 'virtio'){
3456            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3457         }
3458
3459         if ($drive->{interface} eq 'scsi') {
3460
3461             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3462
3463             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3464                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3465
3466             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3467             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3468
3469             my $iothread = '';
3470             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3471                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3472                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3473             } elsif ($drive->{iothread}) {
3474                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3475             }
3476
3477             my $queues = '';
3478             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3479                 $queues = ",num_queues=$drive->{queues}";
3480             }
3481
3482             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3483             $scsicontroller->{$controller}=1;
3484         }
3485
3486         if ($drive->{interface} eq 'sata') {
3487            my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3488            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3489            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3490            $ahcicontroller->{$controller}=1;
3491         }
3492
3493         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3494         push @$devices, '-drive',$drive_cmd;
3495         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3496     });
3497
3498     for (my $i = 0; $i < $MAX_NETS; $i++) {
3499          next if !$conf->{"net$i"};
3500          my $d = parse_net($conf->{"net$i"});
3501          next if !$d;
3502
3503          $use_virtio = 1 if $d->{model} eq 'virtio';
3504
3505          if ($bootindex_hash->{n}) {
3506             $d->{bootindex} = $bootindex_hash->{n};
3507             $bootindex_hash->{n} += 1;
3508          }
3509
3510          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3511          push @$devices, '-netdev', $netdevfull;
3512
3513          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3514          push @$devices, '-device', $netdevicefull;
3515     }
3516
3517     if ($conf->{ivshmem}) {
3518         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3519
3520         my $bus;
3521         if ($q35) {
3522             $bus = print_pcie_addr("ivshmem");
3523         } else {
3524             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3525         }
3526
3527         my $ivshmem_name = $ivshmem->{name} // $vmid;
3528         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3529
3530         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3531         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3532     }
3533
3534     # pci.4 is nested in pci.1
3535     $bridges->{1} = 1 if $bridges->{4};
3536
3537     if (!$q35) {
3538         # add pci bridges
3539         if (min_version($machine_version, 2, 3)) {
3540            $bridges->{1} = 1;
3541            $bridges->{2} = 1;
3542         }
3543
3544         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3545
3546     }
3547
3548     for my $k (sort {$b cmp $a} keys %$bridges) {
3549         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3550         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3551         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3552         if ($q35) {
3553             # add after -readconfig pve-q35.cfg
3554             splice @$devices, 2, 0, '-device', $devstr;
3555         } else {
3556             unshift @$devices, '-device', $devstr if $k > 0;
3557         }
3558     }
3559
3560     if (!$kvm) {
3561         push @$machineFlags, 'accel=tcg';
3562     }
3563
3564     my $machine_type_min = $machine_type;
3565     if ($add_pve_version) {
3566         $machine_type_min =~ s/\+pve\d+$//;
3567         $machine_type_min .= "+pve$required_pve_version";
3568     }
3569     push @$machineFlags, "type=${machine_type_min}";
3570
3571     push @$cmd, @$devices;
3572     push @$cmd, '-rtc', join(',', @$rtcFlags)
3573         if scalar(@$rtcFlags);
3574     push @$cmd, '-machine', join(',', @$machineFlags)
3575         if scalar(@$machineFlags);
3576     push @$cmd, '-global', join(',', @$globalFlags)
3577         if scalar(@$globalFlags);
3578
3579     if (my $vmstate = $conf->{vmstate}) {
3580         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3581         push @$vollist, $vmstate;
3582         push @$cmd, '-loadstate', $statepath;
3583         print "activating and using '$vmstate' as vmstate\n";
3584     }
3585
3586     # add custom args
3587     if ($conf->{args}) {
3588         my $aa = PVE::Tools::split_args($conf->{args});
3589         push @$cmd, @$aa;
3590     }
3591
3592     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3593 }
3594
3595 sub spice_port {
3596     my ($vmid) = @_;
3597
3598     my $res = mon_cmd($vmid, 'query-spice');
3599
3600     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3601 }
3602
3603 sub vm_devices_list {
3604     my ($vmid) = @_;
3605
3606     my $res = mon_cmd($vmid, 'query-pci');
3607     my $devices_to_check = [];
3608     my $devices = {};
3609     foreach my $pcibus (@$res) {
3610         push @$devices_to_check, @{$pcibus->{devices}},
3611     }
3612
3613     while (@$devices_to_check) {
3614         my $to_check = [];
3615         for my $d (@$devices_to_check) {
3616             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3617             next if !$d->{'pci_bridge'};
3618
3619             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3620             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3621         }
3622         $devices_to_check = $to_check;
3623     }
3624
3625     my $resblock = mon_cmd($vmid, 'query-block');
3626     foreach my $block (@$resblock) {
3627         if($block->{device} =~ m/^drive-(\S+)/){
3628                 $devices->{$1} = 1;
3629         }
3630     }
3631
3632     my $resmice = mon_cmd($vmid, 'query-mice');
3633     foreach my $mice (@$resmice) {
3634         if ($mice->{name} eq 'QEMU HID Tablet') {
3635             $devices->{tablet} = 1;
3636             last;
3637         }
3638     }
3639
3640     # for usb devices there is no query-usb
3641     # but we can iterate over the entries in
3642     # qom-list path=/machine/peripheral
3643     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3644     foreach my $per (@$resperipheral) {
3645         if ($per->{name} =~ m/^usb\d+$/) {
3646             $devices->{$per->{name}} = 1;
3647         }
3648     }
3649
3650     return $devices;
3651 }
3652
3653 sub vm_deviceplug {
3654     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3655
3656     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3657
3658     my $devices_list = vm_devices_list($vmid);
3659     return 1 if defined($devices_list->{$deviceid});
3660
3661     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3662
3663     if ($deviceid eq 'tablet') {
3664
3665         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3666
3667     } elsif ($deviceid eq 'keyboard') {
3668
3669         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3670
3671     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3672
3673         die "usb hotplug currently not reliable\n";
3674         # since we can't reliably hot unplug all added usb devices
3675         # and usb passthrough disables live migration
3676         # we disable usb hotplugging for now
3677         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3678
3679     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3680
3681         qemu_iothread_add($vmid, $deviceid, $device);
3682
3683         qemu_driveadd($storecfg, $vmid, $device);
3684         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3685
3686         qemu_deviceadd($vmid, $devicefull);
3687         eval { qemu_deviceaddverify($vmid, $deviceid); };
3688         if (my $err = $@) {
3689             eval { qemu_drivedel($vmid, $deviceid); };
3690             warn $@ if $@;
3691             die $err;
3692         }
3693
3694     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3695
3696
3697         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3698         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3699         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3700
3701         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3702
3703         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3704             qemu_iothread_add($vmid, $deviceid, $device);
3705             $devicefull .= ",iothread=iothread-$deviceid";
3706         }
3707
3708         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3709             $devicefull .= ",num_queues=$device->{queues}";
3710         }
3711
3712         qemu_deviceadd($vmid, $devicefull);
3713         qemu_deviceaddverify($vmid, $deviceid);
3714
3715     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3716
3717         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3718         qemu_driveadd($storecfg, $vmid, $device);
3719
3720         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3721         eval { qemu_deviceadd($vmid, $devicefull); };
3722         if (my $err = $@) {
3723             eval { qemu_drivedel($vmid, $deviceid); };
3724             warn $@ if $@;
3725             die $err;
3726         }
3727
3728     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3729
3730         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3731
3732         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3733         my $use_old_bios_files = undef;
3734         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3735
3736         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3737         qemu_deviceadd($vmid, $netdevicefull);
3738         eval {
3739             qemu_deviceaddverify($vmid, $deviceid);
3740             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3741         };
3742         if (my $err = $@) {
3743             eval { qemu_netdevdel($vmid, $deviceid); };
3744             warn $@ if $@;
3745             die $err;
3746         }
3747
3748     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3749
3750         my $bridgeid = $2;
3751         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3752         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3753
3754         qemu_deviceadd($vmid, $devicefull);
3755         qemu_deviceaddverify($vmid, $deviceid);
3756
3757     } else {
3758         die "can't hotplug device '$deviceid'\n";
3759     }
3760
3761     return 1;
3762 }
3763
3764 # fixme: this should raise exceptions on error!
3765 sub vm_deviceunplug {
3766     my ($vmid, $conf, $deviceid) = @_;
3767
3768     my $devices_list = vm_devices_list($vmid);
3769     return 1 if !defined($devices_list->{$deviceid});
3770
3771     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3772
3773     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3774
3775         qemu_devicedel($vmid, $deviceid);
3776
3777     } elsif ($deviceid =~ m/^usb\d+$/) {
3778
3779         die "usb hotplug currently not reliable\n";
3780         # when unplugging usb devices this way,
3781         # there may be remaining usb controllers/hubs
3782         # so we disable it for now
3783         qemu_devicedel($vmid, $deviceid);
3784         qemu_devicedelverify($vmid, $deviceid);
3785
3786     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3787
3788         qemu_devicedel($vmid, $deviceid);
3789         qemu_devicedelverify($vmid, $deviceid);
3790         qemu_drivedel($vmid, $deviceid);
3791         qemu_iothread_del($conf, $vmid, $deviceid);
3792
3793     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3794
3795         qemu_devicedel($vmid, $deviceid);
3796         qemu_devicedelverify($vmid, $deviceid);
3797         qemu_iothread_del($conf, $vmid, $deviceid);
3798
3799     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3800
3801         qemu_devicedel($vmid, $deviceid);
3802         qemu_drivedel($vmid, $deviceid);
3803         qemu_deletescsihw($conf, $vmid, $deviceid);
3804
3805     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3806
3807         qemu_devicedel($vmid, $deviceid);
3808         qemu_devicedelverify($vmid, $deviceid);
3809         qemu_netdevdel($vmid, $deviceid);
3810
3811     } else {
3812         die "can't unplug device '$deviceid'\n";
3813     }
3814
3815     return 1;
3816 }
3817
3818 sub qemu_deviceadd {
3819     my ($vmid, $devicefull) = @_;
3820
3821     $devicefull = "driver=".$devicefull;
3822     my %options =  split(/[=,]/, $devicefull);
3823
3824     mon_cmd($vmid, "device_add" , %options);
3825 }
3826
3827 sub qemu_devicedel {
3828     my ($vmid, $deviceid) = @_;
3829
3830     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3831 }
3832
3833 sub qemu_iothread_add {
3834     my($vmid, $deviceid, $device) = @_;
3835
3836     if ($device->{iothread}) {
3837         my $iothreads = vm_iothreads_list($vmid);
3838         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3839     }
3840 }
3841
3842 sub qemu_iothread_del {
3843     my($conf, $vmid, $deviceid) = @_;
3844
3845     my $confid = $deviceid;
3846     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3847         $confid = 'scsi' . $1;
3848     }
3849     my $device = parse_drive($confid, $conf->{$confid});
3850     if ($device->{iothread}) {
3851         my $iothreads = vm_iothreads_list($vmid);
3852         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3853     }
3854 }
3855
3856 sub qemu_objectadd {
3857     my($vmid, $objectid, $qomtype) = @_;
3858
3859     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3860
3861     return 1;
3862 }
3863
3864 sub qemu_objectdel {
3865     my($vmid, $objectid) = @_;
3866
3867     mon_cmd($vmid, "object-del", id => $objectid);
3868
3869     return 1;
3870 }
3871
3872 sub qemu_driveadd {
3873     my ($storecfg, $vmid, $device) = @_;
3874
3875     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3876     $drive =~ s/\\/\\\\/g;
3877     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3878
3879     # If the command succeeds qemu prints: "OK"
3880     return 1 if $ret =~ m/OK/s;
3881
3882     die "adding drive failed: $ret\n";
3883 }
3884
3885 sub qemu_drivedel {
3886     my($vmid, $deviceid) = @_;
3887
3888     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3889     $ret =~ s/^\s+//;
3890
3891     return 1 if $ret eq "";
3892
3893     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3894     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3895
3896     die "deleting drive $deviceid failed : $ret\n";
3897 }
3898
3899 sub qemu_deviceaddverify {
3900     my ($vmid, $deviceid) = @_;
3901
3902     for (my $i = 0; $i <= 5; $i++) {
3903          my $devices_list = vm_devices_list($vmid);
3904          return 1 if defined($devices_list->{$deviceid});
3905          sleep 1;
3906     }
3907
3908     die "error on hotplug device '$deviceid'\n";
3909 }
3910
3911
3912 sub qemu_devicedelverify {
3913     my ($vmid, $deviceid) = @_;
3914
3915     # need to verify that the device is correctly removed as device_del
3916     # is async and empty return is not reliable
3917
3918     for (my $i = 0; $i <= 5; $i++) {
3919          my $devices_list = vm_devices_list($vmid);
3920          return 1 if !defined($devices_list->{$deviceid});
3921          sleep 1;
3922     }
3923
3924     die "error on hot-unplugging device '$deviceid'\n";
3925 }
3926
3927 sub qemu_findorcreatescsihw {
3928     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3929
3930     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3931
3932     my $scsihwid="$controller_prefix$controller";
3933     my $devices_list = vm_devices_list($vmid);
3934
3935     if(!defined($devices_list->{$scsihwid})) {
3936         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3937     }
3938
3939     return 1;
3940 }
3941
3942 sub qemu_deletescsihw {
3943     my ($conf, $vmid, $opt) = @_;
3944
3945     my $device = parse_drive($opt, $conf->{$opt});
3946
3947     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3948         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3949         return 1;
3950     }
3951
3952     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3953
3954     my $devices_list = vm_devices_list($vmid);
3955     foreach my $opt (keys %{$devices_list}) {
3956         if (is_valid_drivename($opt)) {
3957             my $drive = parse_drive($opt, $conf->{$opt});
3958             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3959                 return 1;
3960             }
3961         }
3962     }
3963
3964     my $scsihwid="scsihw$controller";
3965
3966     vm_deviceunplug($vmid, $conf, $scsihwid);
3967
3968     return 1;
3969 }
3970
3971 sub qemu_add_pci_bridge {
3972     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3973
3974     my $bridges = {};
3975
3976     my $bridgeid;
3977
3978     print_pci_addr($device, $bridges, $arch, $machine_type);
3979
3980     while (my ($k, $v) = each %$bridges) {
3981         $bridgeid = $k;
3982     }
3983     return 1 if !defined($bridgeid) || $bridgeid < 1;
3984
3985     my $bridge = "pci.$bridgeid";
3986     my $devices_list = vm_devices_list($vmid);
3987
3988     if (!defined($devices_list->{$bridge})) {
3989         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3990     }
3991
3992     return 1;
3993 }
3994
3995 sub qemu_set_link_status {
3996     my ($vmid, $device, $up) = @_;
3997
3998     mon_cmd($vmid, "set_link", name => $device,
3999                up => $up ? JSON::true : JSON::false);
4000 }
4001
4002 sub qemu_netdevadd {
4003     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4004
4005     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4006     my %options =  split(/[=,]/, $netdev);
4007
4008     mon_cmd($vmid, "netdev_add",  %options);
4009     return 1;
4010 }
4011
4012 sub qemu_netdevdel {
4013     my ($vmid, $deviceid) = @_;
4014
4015     mon_cmd($vmid, "netdev_del", id => $deviceid);
4016 }
4017
4018 sub qemu_usb_hotplug {
4019     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4020
4021     return if !$device;
4022
4023     # remove the old one first
4024     vm_deviceunplug($vmid, $conf, $deviceid);
4025
4026     # check if xhci controller is necessary and available
4027     if ($device->{usb3}) {
4028
4029         my $devicelist = vm_devices_list($vmid);
4030
4031         if (!$devicelist->{xhci}) {
4032             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4033             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4034         }
4035     }
4036     my $d = parse_usb_device($device->{host});
4037     $d->{usb3} = $device->{usb3};
4038
4039     # add the new one
4040     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4041 }
4042
4043 sub qemu_cpu_hotplug {
4044     my ($vmid, $conf, $vcpus) = @_;
4045
4046     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4047
4048     my $sockets = 1;
4049     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4050     $sockets = $conf->{sockets} if  $conf->{sockets};
4051     my $cores = $conf->{cores} || 1;
4052     my $maxcpus = $sockets * $cores;
4053
4054     $vcpus = $maxcpus if !$vcpus;
4055
4056     die "you can't add more vcpus than maxcpus\n"
4057         if $vcpus > $maxcpus;
4058
4059     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4060
4061     if ($vcpus < $currentvcpus) {
4062
4063         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4064
4065             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4066                 qemu_devicedel($vmid, "cpu$i");
4067                 my $retry = 0;
4068                 my $currentrunningvcpus = undef;
4069                 while (1) {
4070                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4071                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4072                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4073                     $retry++;
4074                     sleep 1;
4075                 }
4076                 #update conf after each succesfull cpu unplug
4077                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4078                 PVE::QemuConfig->write_config($vmid, $conf);
4079             }
4080         } else {
4081             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4082         }
4083
4084         return;
4085     }
4086
4087     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4088     die "vcpus in running vm does not match its configuration\n"
4089         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4090
4091     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4092
4093         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4094             my $cpustr = print_cpu_device($conf, $i);
4095             qemu_deviceadd($vmid, $cpustr);
4096
4097             my $retry = 0;
4098             my $currentrunningvcpus = undef;
4099             while (1) {
4100                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4101                 last if scalar(@{$currentrunningvcpus}) == $i;
4102                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4103                 sleep 1;
4104                 $retry++;
4105             }
4106             #update conf after each succesfull cpu hotplug
4107             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4108             PVE::QemuConfig->write_config($vmid, $conf);
4109         }
4110     } else {
4111
4112         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4113             mon_cmd($vmid, "cpu-add", id => int($i));
4114         }
4115     }
4116 }
4117
4118 sub qemu_block_set_io_throttle {
4119     my ($vmid, $deviceid,
4120         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4121         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4122         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4123         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4124
4125     return if !check_running($vmid) ;
4126
4127     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4128         bps => int($bps),
4129         bps_rd => int($bps_rd),
4130         bps_wr => int($bps_wr),
4131         iops => int($iops),
4132         iops_rd => int($iops_rd),
4133         iops_wr => int($iops_wr),
4134         bps_max => int($bps_max),
4135         bps_rd_max => int($bps_rd_max),
4136         bps_wr_max => int($bps_wr_max),
4137         iops_max => int($iops_max),
4138         iops_rd_max => int($iops_rd_max),
4139         iops_wr_max => int($iops_wr_max),
4140         bps_max_length => int($bps_max_length),
4141         bps_rd_max_length => int($bps_rd_max_length),
4142         bps_wr_max_length => int($bps_wr_max_length),
4143         iops_max_length => int($iops_max_length),
4144         iops_rd_max_length => int($iops_rd_max_length),
4145         iops_wr_max_length => int($iops_wr_max_length),
4146     );
4147
4148 }
4149
4150 # old code, only used to shutdown old VM after update
4151 sub __read_avail {
4152     my ($fh, $timeout) = @_;
4153
4154     my $sel = new IO::Select;
4155     $sel->add($fh);
4156
4157     my $res = '';
4158     my $buf;
4159
4160     my @ready;
4161     while (scalar (@ready = $sel->can_read($timeout))) {
4162         my $count;
4163         if ($count = $fh->sysread($buf, 8192)) {
4164             if ($buf =~ /^(.*)\(qemu\) $/s) {
4165                 $res .= $1;
4166                 last;
4167             } else {
4168                 $res .= $buf;
4169             }
4170         } else {
4171             if (!defined($count)) {
4172                 die "$!\n";
4173             }
4174             last;
4175         }
4176     }
4177
4178     die "monitor read timeout\n" if !scalar(@ready);
4179
4180     return $res;
4181 }
4182
4183 sub qemu_block_resize {
4184     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4185
4186     my $running = check_running($vmid);
4187
4188     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4189
4190     return if !$running;
4191
4192     my $padding = (1024 - $size % 1024) % 1024;
4193     $size = $size + $padding;
4194
4195     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4196
4197 }
4198
4199 sub qemu_volume_snapshot {
4200     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4201
4202     my $running = check_running($vmid);
4203
4204     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4205         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4206     } else {
4207         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4208     }
4209 }
4210
4211 sub qemu_volume_snapshot_delete {
4212     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4213
4214     my $running = check_running($vmid);
4215
4216     if($running) {
4217
4218         $running = undef;
4219         my $conf = PVE::QemuConfig->load_config($vmid);
4220         foreach_drive($conf, sub {
4221             my ($ds, $drive) = @_;
4222             $running = 1 if $drive->{file} eq $volid;
4223         });
4224     }
4225
4226     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4227         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4228     } else {
4229         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4230     }
4231 }
4232
4233 sub set_migration_caps {
4234     my ($vmid) = @_;
4235
4236     my $cap_ref = [];
4237
4238     my $enabled_cap = {
4239         "auto-converge" => 1,
4240         "xbzrle" => 1,
4241         "x-rdma-pin-all" => 0,
4242         "zero-blocks" => 0,
4243         "compress" => 0
4244     };
4245
4246     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4247
4248     for my $supported_capability (@$supported_capabilities) {
4249         push @$cap_ref, {
4250             capability => $supported_capability->{capability},
4251             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4252         };
4253     }
4254
4255     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4256 }
4257
4258 my $fast_plug_option = {
4259     'lock' => 1,
4260     'name' => 1,
4261     'onboot' => 1,
4262     'shares' => 1,
4263     'startup' => 1,
4264     'description' => 1,
4265     'protection' => 1,
4266     'vmstatestorage' => 1,
4267     'hookscript' => 1,
4268     'tags' => 1,
4269 };
4270
4271 # hotplug changes in [PENDING]
4272 # $selection hash can be used to only apply specified options, for
4273 # example: { cores => 1 } (only apply changed 'cores')
4274 # $errors ref is used to return error messages
4275 sub vmconfig_hotplug_pending {
4276     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4277
4278     my $defaults = load_defaults();
4279     my $arch = get_vm_arch($conf);
4280     my $machine_type = get_vm_machine($conf, undef, $arch);
4281
4282     # commit values which do not have any impact on running VM first
4283     # Note: those option cannot raise errors, we we do not care about
4284     # $selection and always apply them.
4285
4286     my $add_error = sub {
4287         my ($opt, $msg) = @_;
4288         $errors->{$opt} = "hotplug problem - $msg";
4289     };
4290
4291     my $changes = 0;
4292     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4293         if ($fast_plug_option->{$opt}) {
4294             $conf->{$opt} = $conf->{pending}->{$opt};
4295             delete $conf->{pending}->{$opt};
4296             $changes = 1;
4297         }
4298     }
4299
4300     if ($changes) {
4301         PVE::QemuConfig->write_config($vmid, $conf);
4302     }
4303
4304     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4305
4306     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4307     foreach my $opt (sort keys %$pending_delete_hash) {
4308         next if $selection && !$selection->{$opt};
4309         my $force = $pending_delete_hash->{$opt}->{force};
4310         eval {
4311             if ($opt eq 'hotplug') {
4312                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4313             } elsif ($opt eq 'tablet') {
4314                 die "skip\n" if !$hotplug_features->{usb};
4315                 if ($defaults->{tablet}) {
4316                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4317                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4318                         if $arch eq 'aarch64';
4319                 } else {
4320                     vm_deviceunplug($vmid, $conf, 'tablet');
4321                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4322                 }
4323             } elsif ($opt =~ m/^usb\d+/) {
4324                 die "skip\n";
4325                 # since we cannot reliably hot unplug usb devices
4326                 # we are disabling it
4327                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4328                 vm_deviceunplug($vmid, $conf, $opt);
4329             } elsif ($opt eq 'vcpus') {
4330                 die "skip\n" if !$hotplug_features->{cpu};
4331                 qemu_cpu_hotplug($vmid, $conf, undef);
4332             } elsif ($opt eq 'balloon') {
4333                 # enable balloon device is not hotpluggable
4334                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4335                 # here we reset the ballooning value to memory
4336                 my $balloon = $conf->{memory} || $defaults->{memory};
4337                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4338             } elsif ($fast_plug_option->{$opt}) {
4339                 # do nothing
4340             } elsif ($opt =~ m/^net(\d+)$/) {
4341                 die "skip\n" if !$hotplug_features->{network};
4342                 vm_deviceunplug($vmid, $conf, $opt);
4343             } elsif (is_valid_drivename($opt)) {
4344                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4345                 vm_deviceunplug($vmid, $conf, $opt);
4346                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4347             } elsif ($opt =~ m/^memory$/) {
4348                 die "skip\n" if !$hotplug_features->{memory};
4349                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4350             } elsif ($opt eq 'cpuunits') {
4351                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4352             } elsif ($opt eq 'cpulimit') {
4353                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4354             } else {
4355                 die "skip\n";
4356             }
4357         };
4358         if (my $err = $@) {
4359             &$add_error($opt, $err) if $err ne "skip\n";
4360         } else {
4361             delete $conf->{$opt};
4362             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4363         }
4364     }
4365
4366     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4367     $apply_pending_cloudinit = sub {
4368         return if $apply_pending_cloudinit_done; # once is enough
4369         $apply_pending_cloudinit_done = 1; # once is enough
4370
4371         my ($key, $value) = @_;
4372
4373         my @cloudinit_opts = keys %$confdesc_cloudinit;
4374         foreach my $opt (keys %{$conf->{pending}}) {
4375             next if !grep { $_ eq $opt } @cloudinit_opts;
4376             $conf->{$opt} = delete $conf->{pending}->{$opt};
4377         }
4378
4379         my $new_conf = { %$conf };
4380         $new_conf->{$key} = $value;
4381         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4382     };
4383
4384     foreach my $opt (keys %{$conf->{pending}}) {
4385         next if $selection && !$selection->{$opt};
4386         my $value = $conf->{pending}->{$opt};
4387         eval {
4388             if ($opt eq 'hotplug') {
4389                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4390             } elsif ($opt eq 'tablet') {
4391                 die "skip\n" if !$hotplug_features->{usb};
4392                 if ($value == 1) {
4393                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4394                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4395                         if $arch eq 'aarch64';
4396                 } elsif ($value == 0) {
4397                     vm_deviceunplug($vmid, $conf, 'tablet');
4398                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4399                 }
4400             } elsif ($opt =~ m/^usb\d+$/) {
4401                 die "skip\n";
4402                 # since we cannot reliably hot unplug usb devices
4403                 # we are disabling it
4404                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4405                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4406                 die "skip\n" if !$d;
4407                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4408             } elsif ($opt eq 'vcpus') {
4409                 die "skip\n" if !$hotplug_features->{cpu};
4410                 qemu_cpu_hotplug($vmid, $conf, $value);
4411             } elsif ($opt eq 'balloon') {
4412                 # enable/disable balloning device is not hotpluggable
4413                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4414                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4415                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4416
4417                 # allow manual ballooning if shares is set to zero
4418                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4419                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4420                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4421                 }
4422             } elsif ($opt =~ m/^net(\d+)$/) {
4423                 # some changes can be done without hotplug
4424                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4425                                     $vmid, $opt, $value, $arch, $machine_type);
4426             } elsif (is_valid_drivename($opt)) {
4427                 die "skip\n" if $opt eq 'efidisk0';
4428                 # some changes can be done without hotplug
4429                 my $drive = parse_drive($opt, $value);
4430                 if (drive_is_cloudinit($drive)) {
4431                     &$apply_pending_cloudinit($opt, $value);
4432                 }
4433                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4434                                      $vmid, $opt, $value, $arch, $machine_type);
4435             } elsif ($opt =~ m/^memory$/) { #dimms
4436                 die "skip\n" if !$hotplug_features->{memory};
4437                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4438             } elsif ($opt eq 'cpuunits') {
4439                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4440             } elsif ($opt eq 'cpulimit') {
4441                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4442                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4443             } else {
4444                 die "skip\n";  # skip non-hot-pluggable options
4445             }
4446         };
4447         if (my $err = $@) {
4448             &$add_error($opt, $err) if $err ne "skip\n";
4449         } else {
4450             $conf->{$opt} = $value;
4451             delete $conf->{pending}->{$opt};
4452         }
4453     }
4454
4455     PVE::QemuConfig->write_config($vmid, $conf);
4456 }
4457
4458 sub try_deallocate_drive {
4459     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4460
4461     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4462         my $volid = $drive->{file};
4463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4464             my $sid = PVE::Storage::parse_volume_id($volid);
4465             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4466
4467             # check if the disk is really unused
4468             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4469                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4470             PVE::Storage::vdisk_free($storecfg, $volid);
4471             return 1;
4472         } else {
4473             # If vm is not owner of this disk remove from config
4474             return 1;
4475         }
4476     }
4477
4478     return undef;
4479 }
4480
4481 sub vmconfig_delete_or_detach_drive {
4482     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4483
4484     my $drive = parse_drive($opt, $conf->{$opt});
4485
4486     my $rpcenv = PVE::RPCEnvironment::get();
4487     my $authuser = $rpcenv->get_user();
4488
4489     if ($force) {
4490         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4491         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4492     } else {
4493         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4494     }
4495 }
4496
4497
4498
4499 sub vmconfig_apply_pending {
4500     my ($vmid, $conf, $storecfg, $errors) = @_;
4501
4502     my $add_apply_error = sub {
4503         my ($opt, $msg) = @_;
4504         my $err_msg = "unable to apply pending change $opt : $msg";
4505         $errors->{$opt} = $err_msg;
4506         warn $err_msg;
4507     };
4508
4509     # cold plug
4510
4511     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4512     foreach my $opt (sort keys %$pending_delete_hash) {
4513         my $force = $pending_delete_hash->{$opt}->{force};
4514         eval {
4515             if ($opt =~ m/^unused/) {
4516                 die "internal error";
4517             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4518                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4519             }
4520         };
4521         if (my $err = $@) {
4522             $add_apply_error->($opt, $err);
4523         } else {
4524             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4525             delete $conf->{$opt};
4526         }
4527     }
4528
4529     PVE::QemuConfig->cleanup_pending($conf);
4530
4531     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4532         next if $opt eq 'delete'; # just to be sure
4533         eval {
4534             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4535                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4536             }
4537         };
4538         if (my $err = $@) {
4539             $add_apply_error->($opt, $err);
4540         } else {
4541             $conf->{$opt} = delete $conf->{pending}->{$opt};
4542         }
4543     }
4544
4545     # write all changes at once to avoid unnecessary i/o
4546     PVE::QemuConfig->write_config($vmid, $conf);
4547 }
4548
4549 sub vmconfig_update_net {
4550     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4551
4552     my $newnet = parse_net($value);
4553
4554     if ($conf->{$opt}) {
4555         my $oldnet = parse_net($conf->{$opt});
4556
4557         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4558             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4559             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4560             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4561
4562             # for non online change, we try to hot-unplug
4563             die "skip\n" if !$hotplug;
4564             vm_deviceunplug($vmid, $conf, $opt);
4565         } else {
4566
4567             die "internal error" if $opt !~ m/net(\d+)/;
4568             my $iface = "tap${vmid}i$1";
4569
4570             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4571                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4572                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4573                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4574                 PVE::Network::tap_unplug($iface);
4575
4576                 if ($have_sdn) {
4577                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4578                 } else {
4579                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4580                 }
4581             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4582                 # Rate can be applied on its own but any change above needs to
4583                 # include the rate in tap_plug since OVS resets everything.
4584                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4585             }
4586
4587             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4588                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4589             }
4590
4591             return 1;
4592         }
4593     }
4594
4595     if ($hotplug) {
4596         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4597     } else {
4598         die "skip\n";
4599     }
4600 }
4601
4602 sub vmconfig_update_disk {
4603     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4604
4605     my $drive = parse_drive($opt, $value);
4606
4607     if ($conf->{$opt}) {
4608
4609         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4610
4611             my $media = $drive->{media} || 'disk';
4612             my $oldmedia = $old_drive->{media} || 'disk';
4613             die "unable to change media type\n" if $media ne $oldmedia;
4614
4615             if (!drive_is_cdrom($old_drive)) {
4616
4617                 if ($drive->{file} ne $old_drive->{file}) {
4618
4619                     die "skip\n" if !$hotplug;
4620
4621                     # unplug and register as unused
4622                     vm_deviceunplug($vmid, $conf, $opt);
4623                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4624
4625                 } else {
4626                     # update existing disk
4627
4628                     # skip non hotpluggable value
4629                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4630                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4631                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4632                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4633                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4634                         die "skip\n";
4635                     }
4636
4637                     # apply throttle
4638                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4639                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4640                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4641                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4642                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4643                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4644                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4645                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4646                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4647                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4648                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4649                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4650                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4651                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4652                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4653                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4654                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4655                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4656
4657                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4658                                                    ($drive->{mbps} || 0)*1024*1024,
4659                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4660                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4661                                                    $drive->{iops} || 0,
4662                                                    $drive->{iops_rd} || 0,
4663                                                    $drive->{iops_wr} || 0,
4664                                                    ($drive->{mbps_max} || 0)*1024*1024,
4665                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4666                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4667                                                    $drive->{iops_max} || 0,
4668                                                    $drive->{iops_rd_max} || 0,
4669                                                    $drive->{iops_wr_max} || 0,
4670                                                    $drive->{bps_max_length} || 1,
4671                                                    $drive->{bps_rd_max_length} || 1,
4672                                                    $drive->{bps_wr_max_length} || 1,
4673                                                    $drive->{iops_max_length} || 1,
4674                                                    $drive->{iops_rd_max_length} || 1,
4675                                                    $drive->{iops_wr_max_length} || 1);
4676
4677                     }
4678
4679                     return 1;
4680                 }
4681
4682             } else { # cdrom
4683
4684                 if ($drive->{file} eq 'none') {
4685                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4686                     if (drive_is_cloudinit($old_drive)) {
4687                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4688                     }
4689                 } else {
4690                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4691
4692                     # force eject if locked
4693                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4694
4695                     if ($path) {
4696                         mon_cmd($vmid, "blockdev-change-medium",
4697                             id => "$opt", filename => "$path");
4698                     }
4699                 }
4700
4701                 return 1;
4702             }
4703         }
4704     }
4705
4706     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4707     # hotplug new disks
4708     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4709     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4710 }
4711
4712 # called in locked context by incoming migration
4713 sub vm_migrate_alloc_nbd_disks {
4714     my ($storecfg, $vmid, $conf, $targetstorage, $replicated_volumes) = @_;
4715
4716     my $local_volumes = {};
4717     foreach_drive($conf, sub {
4718         my ($ds, $drive) = @_;
4719
4720         return if drive_is_cdrom($drive);
4721
4722         my $volid = $drive->{file};
4723
4724         return if !$volid;
4725
4726         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4727
4728         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4729         return if $scfg->{shared};
4730         $local_volumes->{$ds} = [$volid, $storeid, $volname];
4731     });
4732
4733     my $format = undef;
4734
4735     my $nbd = {};
4736     foreach my $opt (sort keys %$local_volumes) {
4737         my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4738         if ($replicated_volumes->{$volid}) {
4739             # re-use existing, replicated volume with bitmap on source side
4740             $nbd->{$opt} = $conf->{${opt}};
4741             print "re-using replicated volume: $opt - $volid\n";
4742             next;
4743         }
4744         my $drive = parse_drive($opt, $conf->{$opt});
4745
4746         # If a remote storage is specified and the format of the original
4747         # volume is not available there, fall back to the default format.
4748         # Otherwise use the same format as the original.
4749         if ($targetstorage && $targetstorage ne "1") {
4750             $storeid = $targetstorage;
4751             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4752             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4753             my $fileFormat = qemu_img_format($scfg, $volname);
4754             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4755         } else {
4756             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4757             $format = qemu_img_format($scfg, $volname);
4758         }
4759
4760         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4761         my $newdrive = $drive;
4762         $newdrive->{format} = $format;
4763         $newdrive->{file} = $newvolid;
4764         my $drivestr = print_drive($newdrive);
4765         $nbd->{$opt} = $drivestr;
4766         #pass drive to conf for command line
4767         $conf->{$opt} = $drivestr;
4768     }
4769
4770     return $nbd;
4771 }
4772
4773 # see vm_start_nolock for parameters, additionally:
4774 # migrate_opts:
4775 #   targetstorage = storageid/'1' - target storage for disks migrated over NBD
4776 sub vm_start {
4777     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4778
4779     PVE::QemuConfig->lock_config($vmid, sub {
4780         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4781
4782         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4783
4784         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4785
4786         PVE::QemuConfig->check_lock($conf)
4787             if !($params->{skiplock} || $params->{resume});
4788
4789         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4790
4791         $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $conf, $migrate_opts->{targetstorage}, $migrate_opts->{replicated_volumes})
4792             if $migrate_opts->{targetstorage};
4793
4794         vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4795     });
4796 }
4797
4798
4799 # params:
4800 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4801 #   skiplock => 0/1, skip checking for config lock
4802 #   forcemachine => to force Qemu machine (rollback/migration)
4803 #   timeout => in seconds
4804 #   paused => start VM in paused state (backup)
4805 #   resume => resume from hibernation
4806 # migrate_opts:
4807 #   nbd => newly allocated volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4808 #   migratedfrom => source node
4809 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4810 #   network => CIDR of migration network
4811 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4812 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4813 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4814 sub vm_start_nolock {
4815     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4816
4817     my $statefile = $params->{statefile};
4818     my $resume = $params->{resume};
4819
4820     my $migratedfrom = $migrate_opts->{migratedfrom};
4821     my $migration_type = $migrate_opts->{type};
4822
4823     # clean up leftover reboot request files
4824     eval { clear_reboot_request($vmid); };
4825     warn $@ if $@;
4826
4827     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4828         vmconfig_apply_pending($vmid, $conf, $storecfg);
4829         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4830     }
4831
4832     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4833
4834     my $defaults = load_defaults();
4835
4836     # set environment variable useful inside network script
4837     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4838
4839     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4840
4841     my $forcemachine = $params->{forcemachine};
4842     if ($resume) {
4843         # enforce machine type on suspended vm to ensure HW compatibility
4844         $forcemachine = $conf->{runningmachine};
4845         print "Resuming suspended VM\n";
4846     }
4847
4848     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4849
4850     my $migration_ip;
4851     my $get_migration_ip = sub {
4852         my ($nodename) = @_;
4853
4854         return $migration_ip if defined($migration_ip);
4855
4856         my $cidr = $migrate_opts->{network};
4857
4858         if (!defined($cidr)) {
4859             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4860             $cidr = $dc_conf->{migration}->{network};
4861         }
4862
4863         if (defined($cidr)) {
4864             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4865
4866             die "could not get IP: no address configured on local " .
4867                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4868
4869             die "could not get IP: multiple addresses configured on local " .
4870                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4871
4872             $migration_ip = @$ips[0];
4873         }
4874
4875         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4876             if !defined($migration_ip);
4877
4878         return $migration_ip;
4879     };
4880
4881     my $migrate_uri;
4882     if ($statefile) {
4883         if ($statefile eq 'tcp') {
4884             my $localip = "localhost";
4885             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4886             my $nodename = nodename();
4887
4888             if (!defined($migration_type)) {
4889                 if (defined($datacenterconf->{migration}->{type})) {
4890                     $migration_type = $datacenterconf->{migration}->{type};
4891                 } else {
4892                     $migration_type = 'secure';
4893                 }
4894             }
4895
4896             if ($migration_type eq 'insecure') {
4897                 $localip = $get_migration_ip->($nodename);
4898                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4899             }
4900
4901             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4902             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4903             $migrate_uri = "tcp:${localip}:${migrate_port}";
4904             push @$cmd, '-incoming', $migrate_uri;
4905             push @$cmd, '-S';
4906
4907         } elsif ($statefile eq 'unix') {
4908             # should be default for secure migrations as a ssh TCP forward
4909             # tunnel is not deterministic reliable ready and fails regurarly
4910             # to set up in time, so use UNIX socket forwards
4911             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4912             unlink $socket_addr;
4913
4914             $migrate_uri = "unix:$socket_addr";
4915
4916             push @$cmd, '-incoming', $migrate_uri;
4917             push @$cmd, '-S';
4918
4919         } elsif (-e $statefile) {
4920             push @$cmd, '-loadstate', $statefile;
4921         } else {
4922             my $statepath = PVE::Storage::path($storecfg, $statefile);
4923             push @$vollist, $statefile;
4924             push @$cmd, '-loadstate', $statepath;
4925         }
4926     } elsif ($params->{paused}) {
4927         push @$cmd, '-S';
4928     }
4929
4930     # host pci devices
4931     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4932       my $d = parse_hostpci($conf->{"hostpci$i"});
4933       next if !$d;
4934       my $pcidevices = $d->{pciid};
4935       foreach my $pcidevice (@$pcidevices) {
4936             my $pciid = $pcidevice->{id};
4937
4938             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4939             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4940             die "no pci device info for device '$pciid'\n" if !$info;
4941
4942             if ($d->{mdev}) {
4943                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4944                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4945             } else {
4946                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4947                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4948                 die "can't reset pci device '$pciid'\n"
4949                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4950             }
4951       }
4952     }
4953
4954     PVE::Storage::activate_volumes($storecfg, $vollist);
4955
4956     eval {
4957         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4958             outfunc => sub {}, errfunc => sub {});
4959     };
4960     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4961     # timeout should be more than enough here...
4962     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4963
4964     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4965                                               : $defaults->{cpuunits};
4966
4967     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4968     my %run_params = (
4969         timeout => $statefile ? undef : $start_timeout,
4970         umask => 0077,
4971         noerr => 1,
4972     );
4973
4974     # when migrating, prefix QEMU output so other side can pick up any
4975     # errors that might occur and show the user
4976     if ($migratedfrom) {
4977         $run_params{quiet} = 1;
4978         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4979     }
4980
4981     my %properties = (
4982         Slice => 'qemu.slice',
4983         KillMode => 'none',
4984         CPUShares => $cpuunits
4985     );
4986
4987     if (my $cpulimit = $conf->{cpulimit}) {
4988         $properties{CPUQuota} = int($cpulimit * 100);
4989     }
4990     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4991
4992     my $run_qemu = sub {
4993         PVE::Tools::run_fork sub {
4994             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4995
4996             my $exitcode = run_command($cmd, %run_params);
4997             die "QEMU exited with code $exitcode\n" if $exitcode;
4998         };
4999     };
5000
5001     if ($conf->{hugepages}) {
5002
5003         my $code = sub {
5004             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5005             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5006
5007             PVE::QemuServer::Memory::hugepages_mount();
5008             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5009
5010             eval { $run_qemu->() };
5011             if (my $err = $@) {
5012                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5013                 die $err;
5014             }
5015
5016             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5017         };
5018         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5019
5020     } else {
5021         eval { $run_qemu->() };
5022     }
5023
5024     if (my $err = $@) {
5025         # deactivate volumes if start fails
5026         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5027         die "start failed: $err";
5028     }
5029
5030     print "migration listens on $migrate_uri\n" if $migrate_uri;
5031
5032     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5033         eval { mon_cmd($vmid, "cont"); };
5034         warn $@ if $@;
5035     }
5036
5037     #start nbd server for storage migration
5038     if (my $nbd = $migrate_opts->{nbd}) {
5039         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5040
5041         my $migrate_storage_uri;
5042         # nbd_protocol_version > 0 for unix socket support
5043         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5044             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5045             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5046             $migrate_storage_uri = "nbd:unix:$socket_path";
5047         } else {
5048             my $nodename = nodename();
5049             my $localip = $get_migration_ip->($nodename);
5050             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5051             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5052
5053             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5054             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5055             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5056         }
5057
5058         foreach my $opt (sort keys %$nbd) {
5059             my $drivestr = $nbd->{$opt};
5060             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5061             print "storage migration listens on $migrate_storage_uri:exportname=drive-$opt volume:$drivestr\n";
5062         }
5063     }
5064
5065     if ($migratedfrom) {
5066         eval {
5067             set_migration_caps($vmid);
5068         };
5069         warn $@ if $@;
5070
5071         if ($spice_port) {
5072             print "spice listens on port $spice_port\n";
5073             if ($migrate_opts->{spice_ticket}) {
5074                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5075                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5076             }
5077         }
5078
5079     } else {
5080         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5081             if !$statefile && $conf->{balloon};
5082
5083         foreach my $opt (keys %$conf) {
5084             next if $opt !~  m/^net\d+$/;
5085             my $nicconf = parse_net($conf->{$opt});
5086             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5087         }
5088     }
5089
5090     mon_cmd($vmid, 'qom-set',
5091                 path => "machine/peripheral/balloon0",
5092                 property => "guest-stats-polling-interval",
5093                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5094
5095     if ($resume) {
5096         print "Resumed VM, removing state\n";
5097         if (my $vmstate = $conf->{vmstate}) {
5098             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5099             PVE::Storage::vdisk_free($storecfg, $vmstate);
5100         }
5101         delete $conf->@{qw(lock vmstate runningmachine)};
5102         PVE::QemuConfig->write_config($vmid, $conf);
5103     }
5104
5105     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5106 }
5107
5108 sub vm_commandline {
5109     my ($storecfg, $vmid, $snapname) = @_;
5110
5111     my $conf = PVE::QemuConfig->load_config($vmid);
5112     my $forcemachine;
5113
5114     if ($snapname) {
5115         my $snapshot = $conf->{snapshots}->{$snapname};
5116         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5117
5118         # check for a 'runningmachine' in snapshot
5119         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5120
5121         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5122
5123         $conf = $snapshot;
5124     }
5125
5126     my $defaults = load_defaults();
5127
5128     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5129
5130     return PVE::Tools::cmd2string($cmd);
5131 }
5132
5133 sub vm_reset {
5134     my ($vmid, $skiplock) = @_;
5135
5136     PVE::QemuConfig->lock_config($vmid, sub {
5137
5138         my $conf = PVE::QemuConfig->load_config($vmid);
5139
5140         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5141
5142         mon_cmd($vmid, "system_reset");
5143     });
5144 }
5145
5146 sub get_vm_volumes {
5147     my ($conf) = @_;
5148
5149     my $vollist = [];
5150     foreach_volid($conf, sub {
5151         my ($volid, $attr) = @_;
5152
5153         return if $volid =~ m|^/|;
5154
5155         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5156         return if !$sid;
5157
5158         push @$vollist, $volid;
5159     });
5160
5161     return $vollist;
5162 }
5163
5164 sub vm_stop_cleanup {
5165     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5166
5167     eval {
5168
5169         if (!$keepActive) {
5170             my $vollist = get_vm_volumes($conf);
5171             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5172         }
5173
5174         foreach my $ext (qw(mon qmp pid vnc qga)) {
5175             unlink "/var/run/qemu-server/${vmid}.$ext";
5176         }
5177
5178         if ($conf->{ivshmem}) {
5179             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5180             # just delete it for now, VMs which have this already open do not
5181             # are affected, but new VMs will get a separated one. If this
5182             # becomes an issue we either add some sort of ref-counting or just
5183             # add a "don't delete on stop" flag to the ivshmem format.
5184             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5185         }
5186
5187         foreach my $key (keys %$conf) {
5188             next if $key !~ m/^hostpci(\d+)$/;
5189             my $hostpciindex = $1;
5190             my $d = parse_hostpci($conf->{$key});
5191             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5192
5193             foreach my $pci (@{$d->{pciid}}) {
5194                 my $pciid = $pci->{id};
5195                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5196             }
5197         }
5198
5199         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5200     };
5201     warn $@ if $@; # avoid errors - just warn
5202 }
5203
5204 # call only in locked context
5205 sub _do_vm_stop {
5206     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5207
5208     my $pid = check_running($vmid, $nocheck);
5209     return if !$pid;
5210
5211     my $conf;
5212     if (!$nocheck) {
5213         $conf = PVE::QemuConfig->load_config($vmid);
5214         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5215         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5216             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5217             $timeout = $opts->{down} if $opts->{down};
5218         }
5219         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5220     }
5221
5222     eval {
5223         if ($shutdown) {
5224             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5225                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5226             } else {
5227                 mon_cmd($vmid, "system_powerdown");
5228             }
5229         } else {
5230             mon_cmd($vmid, "quit");
5231         }
5232     };
5233     my $err = $@;
5234
5235     if (!$err) {
5236         $timeout = 60 if !defined($timeout);
5237
5238         my $count = 0;
5239         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5240             $count++;
5241             sleep 1;
5242         }
5243
5244         if ($count >= $timeout) {
5245             if ($force) {
5246                 warn "VM still running - terminating now with SIGTERM\n";
5247                 kill 15, $pid;
5248             } else {
5249                 die "VM quit/powerdown failed - got timeout\n";
5250             }
5251         } else {
5252             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5253             return;
5254         }
5255     } else {
5256         if ($force) {
5257             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5258             kill 15, $pid;
5259         } else {
5260             die "VM quit/powerdown failed\n";
5261         }
5262     }
5263
5264     # wait again
5265     $timeout = 10;
5266
5267     my $count = 0;
5268     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5269         $count++;
5270         sleep 1;
5271     }
5272
5273     if ($count >= $timeout) {
5274         warn "VM still running - terminating now with SIGKILL\n";
5275         kill 9, $pid;
5276         sleep 1;
5277     }
5278
5279     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5280 }
5281
5282 # Note: use $nocheck to skip tests if VM configuration file exists.
5283 # We need that when migration VMs to other nodes (files already moved)
5284 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5285 sub vm_stop {
5286     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5287
5288     $force = 1 if !defined($force) && !$shutdown;
5289
5290     if ($migratedfrom){
5291         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5292         kill 15, $pid if $pid;
5293         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5294         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5295         return;
5296     }
5297
5298     PVE::QemuConfig->lock_config($vmid, sub {
5299         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5300    });
5301 }
5302
5303 sub vm_reboot {
5304     my ($vmid, $timeout) = @_;
5305
5306     PVE::QemuConfig->lock_config($vmid, sub {
5307         eval {
5308
5309             # only reboot if running, as qmeventd starts it again on a stop event
5310             return if !check_running($vmid);
5311
5312             create_reboot_request($vmid);
5313
5314             my $storecfg = PVE::Storage::config();
5315             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5316
5317         };
5318         if (my $err = $@) {
5319             # avoid that the next normal shutdown will be confused for a reboot
5320             clear_reboot_request($vmid);
5321             die $err;
5322         }
5323    });
5324 }
5325
5326 # note: if using the statestorage parameter, the caller has to check privileges
5327 sub vm_suspend {
5328     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5329
5330     my $conf;
5331     my $path;
5332     my $storecfg;
5333     my $vmstate;
5334
5335     PVE::QemuConfig->lock_config($vmid, sub {
5336
5337         $conf = PVE::QemuConfig->load_config($vmid);
5338
5339         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5340         PVE::QemuConfig->check_lock($conf)
5341             if !($skiplock || $is_backing_up);
5342
5343         die "cannot suspend to disk during backup\n"
5344             if $is_backing_up && $includestate;
5345
5346         if ($includestate) {
5347             $conf->{lock} = 'suspending';
5348             my $date = strftime("%Y-%m-%d", localtime(time()));
5349             $storecfg = PVE::Storage::config();
5350             if (!$statestorage) {
5351                 $statestorage = find_vmstate_storage($conf, $storecfg);
5352                 # check permissions for the storage
5353                 my $rpcenv = PVE::RPCEnvironment::get();
5354                 if ($rpcenv->{type} ne 'cli') {
5355                     my $authuser = $rpcenv->get_user();
5356                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5357                 }
5358             }
5359
5360
5361             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5362             $path = PVE::Storage::path($storecfg, $vmstate);
5363             PVE::QemuConfig->write_config($vmid, $conf);
5364         } else {
5365             mon_cmd($vmid, "stop");
5366         }
5367     });
5368
5369     if ($includestate) {
5370         # save vm state
5371         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5372
5373         eval {
5374             mon_cmd($vmid, "savevm-start", statefile => $path);
5375             for(;;) {
5376                 my $state = mon_cmd($vmid, "query-savevm");
5377                 if (!$state->{status}) {
5378                     die "savevm not active\n";
5379                 } elsif ($state->{status} eq 'active') {
5380                     sleep(1);
5381                     next;
5382                 } elsif ($state->{status} eq 'completed') {
5383                     print "State saved, quitting\n";
5384                     last;
5385                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5386                     die "query-savevm failed with error '$state->{error}'\n"
5387                 } else {
5388                     die "query-savevm returned status '$state->{status}'\n";
5389                 }
5390             }
5391         };
5392         my $err = $@;
5393
5394         PVE::QemuConfig->lock_config($vmid, sub {
5395             $conf = PVE::QemuConfig->load_config($vmid);
5396             if ($err) {
5397                 # cleanup, but leave suspending lock, to indicate something went wrong
5398                 eval {
5399                     mon_cmd($vmid, "savevm-end");
5400                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5401                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5402                     delete $conf->@{qw(vmstate runningmachine)};
5403                     PVE::QemuConfig->write_config($vmid, $conf);
5404                 };
5405                 warn $@ if $@;
5406                 die $err;
5407             }
5408
5409             die "lock changed unexpectedly\n"
5410                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5411
5412             mon_cmd($vmid, "quit");
5413             $conf->{lock} = 'suspended';
5414             PVE::QemuConfig->write_config($vmid, $conf);
5415         });
5416     }
5417 }
5418
5419 sub vm_resume {
5420     my ($vmid, $skiplock, $nocheck) = @_;
5421
5422     PVE::QemuConfig->lock_config($vmid, sub {
5423         my $res = mon_cmd($vmid, 'query-status');
5424         my $resume_cmd = 'cont';
5425
5426         if ($res->{status} && $res->{status} eq 'suspended') {
5427             $resume_cmd = 'system_wakeup';
5428         }
5429
5430         if (!$nocheck) {
5431
5432             my $conf = PVE::QemuConfig->load_config($vmid);
5433
5434             PVE::QemuConfig->check_lock($conf)
5435                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5436         }
5437
5438         mon_cmd($vmid, $resume_cmd);
5439     });
5440 }
5441
5442 sub vm_sendkey {
5443     my ($vmid, $skiplock, $key) = @_;
5444
5445     PVE::QemuConfig->lock_config($vmid, sub {
5446
5447         my $conf = PVE::QemuConfig->load_config($vmid);
5448
5449         # there is no qmp command, so we use the human monitor command
5450         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5451         die $res if $res ne '';
5452     });
5453 }
5454
5455 # vzdump restore implementaion
5456
5457 sub tar_archive_read_firstfile {
5458     my $archive = shift;
5459
5460     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5461
5462     # try to detect archive type first
5463     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5464         die "unable to open file '$archive'\n";
5465     my $firstfile = <$fh>;
5466     kill 15, $pid;
5467     close $fh;
5468
5469     die "ERROR: archive contaions no data\n" if !$firstfile;
5470     chomp $firstfile;
5471
5472     return $firstfile;
5473 }
5474
5475 sub tar_restore_cleanup {
5476     my ($storecfg, $statfile) = @_;
5477
5478     print STDERR "starting cleanup\n";
5479
5480     if (my $fd = IO::File->new($statfile, "r")) {
5481         while (defined(my $line = <$fd>)) {
5482             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5483                 my $volid = $2;
5484                 eval {
5485                     if ($volid =~ m|^/|) {
5486                         unlink $volid || die 'unlink failed\n';
5487                     } else {
5488                         PVE::Storage::vdisk_free($storecfg, $volid);
5489                     }
5490                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5491                 };
5492                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5493             } else {
5494                 print STDERR "unable to parse line in statfile - $line";
5495             }
5496         }
5497         $fd->close();
5498     }
5499 }
5500
5501 sub restore_file_archive {
5502     my ($archive, $vmid, $user, $opts) = @_;
5503
5504     my $format = $opts->{format};
5505     my $comp;
5506
5507     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5508         $format = 'tar' if !$format;
5509         $comp = 'gzip';
5510     } elsif ($archive =~ m/\.tar$/) {
5511         $format = 'tar' if !$format;
5512     } elsif ($archive =~ m/.tar.lzo$/) {
5513         $format = 'tar' if !$format;
5514         $comp = 'lzop';
5515     } elsif ($archive =~ m/\.vma$/) {
5516         $format = 'vma' if !$format;
5517     } elsif ($archive =~ m/\.vma\.gz$/) {
5518         $format = 'vma' if !$format;
5519         $comp = 'gzip';
5520     } elsif ($archive =~ m/\.vma\.lzo$/) {
5521         $format = 'vma' if !$format;
5522         $comp = 'lzop';
5523     } else {
5524         $format = 'vma' if !$format; # default
5525     }
5526
5527     # try to detect archive format
5528     if ($format eq 'tar') {
5529         return restore_tar_archive($archive, $vmid, $user, $opts);
5530     } else {
5531         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5532     }
5533 }
5534
5535 # hepler to remove disks that will not be used after restore
5536 my $restore_cleanup_oldconf = sub {
5537     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5538
5539     foreach_drive($oldconf, sub {
5540         my ($ds, $drive) = @_;
5541
5542         return if drive_is_cdrom($drive, 1);
5543
5544         my $volid = $drive->{file};
5545         return if !$volid || $volid =~ m|^/|;
5546
5547         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5548         return if !$path || !$owner || ($owner != $vmid);
5549
5550         # Note: only delete disk we want to restore
5551         # other volumes will become unused
5552         if ($virtdev_hash->{$ds}) {
5553             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5554             if (my $err = $@) {
5555                 warn $err;
5556             }
5557         }
5558     });
5559
5560     # delete vmstate files, after the restore we have no snapshots anymore
5561     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5562         my $snap = $oldconf->{snapshots}->{$snapname};
5563         if ($snap->{vmstate}) {
5564             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5565             if (my $err = $@) {
5566                 warn $err;
5567             }
5568         }
5569     }
5570 };
5571
5572 # Helper to parse vzdump backup device hints
5573 #
5574 # $rpcenv: Environment, used to ckeck storage permissions
5575 # $user: User ID, to check storage permissions
5576 # $storecfg: Storage configuration
5577 # $fh: the file handle for reading the configuration
5578 # $devinfo: should contain device sizes for all backu-up'ed devices
5579 # $options: backup options (pool, default storage)
5580 #
5581 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5582 my $parse_backup_hints = sub {
5583     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5584
5585     my $virtdev_hash = {};
5586
5587     while (defined(my $line = <$fh>)) {
5588         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5589             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5590             die "archive does not contain data for drive '$virtdev'\n"
5591                 if !$devinfo->{$devname};
5592
5593             if (defined($options->{storage})) {
5594                 $storeid = $options->{storage} || 'local';
5595             } elsif (!$storeid) {
5596                 $storeid = 'local';
5597             }
5598             $format = 'raw' if !$format;
5599             $devinfo->{$devname}->{devname} = $devname;
5600             $devinfo->{$devname}->{virtdev} = $virtdev;
5601             $devinfo->{$devname}->{format} = $format;
5602             $devinfo->{$devname}->{storeid} = $storeid;
5603
5604             # check permission on storage
5605             my $pool = $options->{pool}; # todo: do we need that?
5606             if ($user ne 'root@pam') {
5607                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5608             }
5609
5610             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5611         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5612             my $virtdev = $1;
5613             my $drive = parse_drive($virtdev, $2);
5614             if (drive_is_cloudinit($drive)) {
5615                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5616                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5617                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5618
5619                 $virtdev_hash->{$virtdev} = {
5620                     format => $format,
5621                     storeid => $options->{storage} // $storeid,
5622                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5623                     is_cloudinit => 1,
5624                 };
5625             }
5626         }
5627     }
5628
5629     return $virtdev_hash;
5630 };
5631
5632 # Helper to allocate and activate all volumes required for a restore
5633 #
5634 # $storecfg: Storage configuration
5635 # $virtdev_hash: as returned by parse_backup_hints()
5636 #
5637 # Returns: { $virtdev => $volid }
5638 my $restore_allocate_devices = sub {
5639     my ($storecfg, $virtdev_hash, $vmid) = @_;
5640
5641     my $map = {};
5642     foreach my $virtdev (sort keys %$virtdev_hash) {
5643         my $d = $virtdev_hash->{$virtdev};
5644         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5645         my $storeid = $d->{storeid};
5646         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5647
5648         # test if requested format is supported
5649         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5650         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5651         $d->{format} = $defFormat if !$supported;
5652
5653         my $name;
5654         if ($d->{is_cloudinit}) {
5655             $name = "vm-$vmid-cloudinit";
5656             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5657         }
5658
5659         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5660
5661         print STDERR "new volume ID is '$volid'\n";
5662         $d->{volid} = $volid;
5663
5664         PVE::Storage::activate_volumes($storecfg, [$volid]);
5665
5666         $map->{$virtdev} = $volid;
5667     }
5668
5669     return $map;
5670 };
5671
5672 my $restore_update_config_line = sub {
5673     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5674
5675     return if $line =~ m/^\#qmdump\#/;
5676     return if $line =~ m/^\#vzdump\#/;
5677     return if $line =~ m/^lock:/;
5678     return if $line =~ m/^unused\d+:/;
5679     return if $line =~ m/^parent:/;
5680
5681     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5682     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5683         # try to convert old 1.X settings
5684         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5685         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5686             my ($model, $macaddr) = split(/\=/, $devconfig);
5687             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5688             my $net = {
5689                 model => $model,
5690                 bridge => "vmbr$ind",
5691                 macaddr => $macaddr,
5692             };
5693             my $netstr = print_net($net);
5694
5695             print $outfd "net$cookie->{netcount}: $netstr\n";
5696             $cookie->{netcount}++;
5697         }
5698     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5699         my ($id, $netstr) = ($1, $2);
5700         my $net = parse_net($netstr);
5701         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5702         $netstr = print_net($net);
5703         print $outfd "$id: $netstr\n";
5704     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5705         my $virtdev = $1;
5706         my $value = $3;
5707         my $di = parse_drive($virtdev, $value);
5708         if (defined($di->{backup}) && !$di->{backup}) {
5709             print $outfd "#$line";
5710         } elsif ($map->{$virtdev}) {
5711             delete $di->{format}; # format can change on restore
5712             $di->{file} = $map->{$virtdev};
5713             $value = print_drive($di);
5714             print $outfd "$virtdev: $value\n";
5715         } else {
5716             print $outfd $line;
5717         }
5718     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5719         my $vmgenid = $1;
5720         if ($vmgenid ne '0') {
5721             # always generate a new vmgenid if there was a valid one setup
5722             $vmgenid = generate_uuid();
5723         }
5724         print $outfd "vmgenid: $vmgenid\n";
5725     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5726         my ($uuid, $uuid_str);
5727         UUID::generate($uuid);
5728         UUID::unparse($uuid, $uuid_str);
5729         my $smbios1 = parse_smbios1($2);
5730         $smbios1->{uuid} = $uuid_str;
5731         print $outfd $1.print_smbios1($smbios1)."\n";
5732     } else {
5733         print $outfd $line;
5734     }
5735 };
5736
5737 my $restore_deactivate_volumes = sub {
5738     my ($storecfg, $devinfo) = @_;
5739
5740     my $vollist = [];
5741     foreach my $devname (keys %$devinfo) {
5742         my $volid = $devinfo->{$devname}->{volid};
5743         push @$vollist, $volid if $volid;
5744     }
5745
5746     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5747 };
5748
5749 my $restore_destroy_volumes = sub {
5750     my ($storecfg, $devinfo) = @_;
5751
5752     foreach my $devname (keys %$devinfo) {
5753         my $volid = $devinfo->{$devname}->{volid};
5754         next if !$volid;
5755         eval {
5756             if ($volid =~ m|^/|) {
5757                 unlink $volid || die 'unlink failed\n';
5758             } else {
5759                 PVE::Storage::vdisk_free($storecfg, $volid);
5760             }
5761             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5762         };
5763         print STDERR "unable to cleanup '$volid' - $@" if $@;
5764     }
5765 };
5766
5767 sub scan_volids {
5768     my ($cfg, $vmid) = @_;
5769
5770     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5771
5772     my $volid_hash = {};
5773     foreach my $storeid (keys %$info) {
5774         foreach my $item (@{$info->{$storeid}}) {
5775             next if !($item->{volid} && $item->{size});
5776             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5777             $volid_hash->{$item->{volid}} = $item;
5778         }
5779     }
5780
5781     return $volid_hash;
5782 }
5783
5784 sub update_disk_config {
5785     my ($vmid, $conf, $volid_hash) = @_;
5786
5787     my $changes;
5788     my $prefix = "VM $vmid:";
5789
5790     # used and unused disks
5791     my $referenced = {};
5792
5793     # Note: it is allowed to define multiple storages with same path (alias), so
5794     # we need to check both 'volid' and real 'path' (two different volid can point
5795     # to the same path).
5796
5797     my $referencedpath = {};
5798
5799     # update size info
5800     PVE::QemuConfig->foreach_volume($conf, sub {
5801         my ($opt, $drive) = @_;
5802
5803         my $volid = $drive->{file};
5804         return if !$volid;
5805
5806         # mark volid as "in-use" for next step
5807         $referenced->{$volid} = 1;
5808         if ($volid_hash->{$volid} &&
5809             (my $path = $volid_hash->{$volid}->{path})) {
5810             $referencedpath->{$path} = 1;
5811         }
5812
5813         return if drive_is_cdrom($drive);
5814         return if !$volid_hash->{$volid};
5815
5816         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5817         if (defined($updated)) {
5818             $changes = 1;
5819             $conf->{$opt} = print_drive($updated);
5820             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5821         }
5822     });
5823
5824     # remove 'unusedX' entry if volume is used
5825     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5826         my ($opt, $drive) = @_;
5827
5828         my $volid = $drive->{file};
5829         return if !$volid;
5830
5831         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5832         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5833             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5834             $changes = 1;
5835             delete $conf->{$opt};
5836         }
5837
5838         $referenced->{$volid} = 1;
5839         $referencedpath->{$path} = 1 if $path;
5840     });
5841
5842     foreach my $volid (sort keys %$volid_hash) {
5843         next if $volid =~ m/vm-$vmid-state-/;
5844         next if $referenced->{$volid};
5845         my $path = $volid_hash->{$volid}->{path};
5846         next if !$path; # just to be sure
5847         next if $referencedpath->{$path};
5848         $changes = 1;
5849         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5850         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5851         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5852     }
5853
5854     return $changes;
5855 }
5856
5857 sub rescan {
5858     my ($vmid, $nolock, $dryrun) = @_;
5859
5860     my $cfg = PVE::Storage::config();
5861
5862     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5863     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5864     foreach my $stor (keys %{$cfg->{ids}}) {
5865         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5866     }
5867
5868     print "rescan volumes...\n";
5869     my $volid_hash = scan_volids($cfg, $vmid);
5870
5871     my $updatefn =  sub {
5872         my ($vmid) = @_;
5873
5874         my $conf = PVE::QemuConfig->load_config($vmid);
5875
5876         PVE::QemuConfig->check_lock($conf);
5877
5878         my $vm_volids = {};
5879         foreach my $volid (keys %$volid_hash) {
5880             my $info = $volid_hash->{$volid};
5881             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5882         }
5883
5884         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5885
5886         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5887     };
5888
5889     if (defined($vmid)) {
5890         if ($nolock) {
5891             &$updatefn($vmid);
5892         } else {
5893             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5894         }
5895     } else {
5896         my $vmlist = config_list();
5897         foreach my $vmid (keys %$vmlist) {
5898             if ($nolock) {
5899                 &$updatefn($vmid);
5900             } else {
5901                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5902             }
5903         }
5904     }
5905 }
5906
5907 sub restore_proxmox_backup_archive {
5908     my ($archive, $vmid, $user, $options) = @_;
5909
5910     my $storecfg = PVE::Storage::config();
5911
5912     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5913     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5914
5915     my $server = $scfg->{server};
5916     my $datastore = $scfg->{datastore};
5917     my $username = $scfg->{username} // 'root@pam';
5918     my $fingerprint = $scfg->{fingerprint};
5919
5920     my $repo = "$username\@$server:$datastore";
5921     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5922     local $ENV{PBS_PASSWORD} = $password;
5923     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5924
5925     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5926         PVE::Storage::parse_volname($storecfg, $archive);
5927
5928     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5929
5930     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5931
5932     my $tmpdir = "/var/tmp/vzdumptmp$$";
5933     rmtree $tmpdir;
5934     mkpath $tmpdir;
5935
5936     my $conffile = PVE::QemuConfig->config_file($vmid);
5937     my $tmpfn = "$conffile.$$.tmp";
5938      # disable interrupts (always do cleanups)
5939     local $SIG{INT} =
5940         local $SIG{TERM} =
5941         local $SIG{QUIT} =
5942         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5943
5944     # Note: $oldconf is undef if VM does not exists
5945     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5946     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5947
5948     my $rpcenv = PVE::RPCEnvironment::get();
5949     my $devinfo = {};
5950
5951     eval {
5952         # enable interrupts
5953         local $SIG{INT} =
5954             local $SIG{TERM} =
5955             local $SIG{QUIT} =
5956             local $SIG{HUP} =
5957             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5958
5959         my $cfgfn = "$tmpdir/qemu-server.conf";
5960         my $firewall_config_fn = "$tmpdir/fw.conf";
5961         my $index_fn = "$tmpdir/index.json";
5962
5963         my $cmd = "restore";
5964
5965         my $param = [$pbs_backup_name, "index.json", $index_fn];
5966         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5967         my $index = PVE::Tools::file_get_contents($index_fn);
5968         $index = decode_json($index);
5969
5970         # print Dumper($index);
5971         foreach my $info (@{$index->{files}}) {
5972             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
5973                 my $devname = $1;
5974                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
5975                     $devinfo->{$devname}->{size} = $1;
5976                 } else {
5977                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
5978                 }
5979             }
5980         }
5981
5982         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
5983         if (!$is_qemu_server_backup) {
5984             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
5985         }
5986         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
5987
5988         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
5989         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5990
5991         if ($has_firewall_config) {
5992             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
5993             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5994
5995             my $pve_firewall_dir = '/etc/pve/firewall';
5996             mkdir $pve_firewall_dir; # make sure the dir exists
5997             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
5998         }
5999
6000         my $fh = IO::File->new($cfgfn, "r") ||
6001             "unable to read qemu-server.conf - $!\n";
6002
6003         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6004
6005         # fixme: rate limit?
6006
6007         # create empty/temp config
6008         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6009
6010         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6011
6012         # allocate volumes
6013         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6014
6015         foreach my $virtdev (sort keys %$virtdev_hash) {
6016             my $d = $virtdev_hash->{$virtdev};
6017             next if $d->{is_cloudinit}; # no need to restore cloudinit
6018
6019             my $volid = $d->{volid};
6020
6021             my $path = PVE::Storage::path($storecfg, $volid);
6022
6023             my $pbs_restore_cmd = [
6024                 '/usr/bin/pbs-restore',
6025                 '--repository', $repo,
6026                 $pbs_backup_name,
6027                 "$d->{devname}.img.fidx",
6028                 $path,
6029                 '--verbose',
6030                 ];
6031
6032             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6033                 push @$pbs_restore_cmd, '--skip-zero';
6034             }
6035
6036             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6037             print "restore proxmox backup image: $dbg_cmdstring\n";
6038             run_command($pbs_restore_cmd);
6039         }
6040
6041         $fh->seek(0, 0) || die "seek failed - $!\n";
6042
6043         my $outfd = new IO::File ($tmpfn, "w") ||
6044             die "unable to write config for VM $vmid\n";
6045
6046         my $cookie = { netcount => 0 };
6047         while (defined(my $line = <$fh>)) {
6048             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6049         }
6050
6051         $fh->close();
6052         $outfd->close();
6053     };
6054     my $err = $@;
6055
6056     $restore_deactivate_volumes->($storecfg, $devinfo);
6057
6058     rmtree $tmpdir;
6059
6060     if ($err) {
6061         unlink $tmpfn;
6062         $restore_destroy_volumes->($storecfg, $devinfo);
6063         die $err;
6064     }
6065
6066     rename($tmpfn, $conffile) ||
6067         die "unable to commit configuration file '$conffile'\n";
6068
6069     PVE::Cluster::cfs_update(); # make sure we read new file
6070
6071     eval { rescan($vmid, 1); };
6072     warn $@ if $@;
6073 }
6074
6075 sub restore_vma_archive {
6076     my ($archive, $vmid, $user, $opts, $comp) = @_;
6077
6078     my $readfrom = $archive;
6079
6080     my $cfg = PVE::Storage::config();
6081     my $commands = [];
6082     my $bwlimit = $opts->{bwlimit};
6083
6084     my $dbg_cmdstring = '';
6085     my $add_pipe = sub {
6086         my ($cmd) = @_;
6087         push @$commands, $cmd;
6088         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6089         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6090         $readfrom = '-';
6091     };
6092
6093     my $input = undef;
6094     if ($archive eq '-') {
6095         $input = '<&STDIN';
6096     } else {
6097         # If we use a backup from a PVE defined storage we also consider that
6098         # storage's rate limit:
6099         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6100         if (defined($volid)) {
6101             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6102             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6103             if ($readlimit) {
6104                 print STDERR "applying read rate limit: $readlimit\n";
6105                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6106                 $add_pipe->($cstream);
6107             }
6108         }
6109     }
6110
6111     if ($comp) {
6112         my $cmd;
6113         if ($comp eq 'gzip') {
6114             $cmd = ['zcat', $readfrom];
6115         } elsif ($comp eq 'lzop') {
6116             $cmd = ['lzop', '-d', '-c', $readfrom];
6117         } else {
6118             die "unknown compression method '$comp'\n";
6119         }
6120         $add_pipe->($cmd);
6121     }
6122
6123     my $tmpdir = "/var/tmp/vzdumptmp$$";
6124     rmtree $tmpdir;
6125
6126     # disable interrupts (always do cleanups)
6127     local $SIG{INT} =
6128         local $SIG{TERM} =
6129         local $SIG{QUIT} =
6130         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6131
6132     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6133     POSIX::mkfifo($mapfifo, 0600);
6134     my $fifofh;
6135
6136     my $openfifo = sub {
6137         open($fifofh, '>', $mapfifo) || die $!;
6138     };
6139
6140     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6141
6142     my $oldtimeout;
6143     my $timeout = 5;
6144
6145     my $devinfo = {};
6146
6147     my $rpcenv = PVE::RPCEnvironment::get();
6148
6149     my $conffile = PVE::QemuConfig->config_file($vmid);
6150     my $tmpfn = "$conffile.$$.tmp";
6151
6152     # Note: $oldconf is undef if VM does not exist
6153     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6154     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6155
6156     my %storage_limits;
6157
6158     my $print_devmap = sub {
6159         my $cfgfn = "$tmpdir/qemu-server.conf";
6160
6161         # we can read the config - that is already extracted
6162         my $fh = IO::File->new($cfgfn, "r") ||
6163             "unable to read qemu-server.conf - $!\n";
6164
6165         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6166         if (-f $fwcfgfn) {
6167             my $pve_firewall_dir = '/etc/pve/firewall';
6168             mkdir $pve_firewall_dir; # make sure the dir exists
6169             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6170         }
6171
6172         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6173
6174         foreach my $key (keys %storage_limits) {
6175             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6176             next if !$limit;
6177             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6178             $storage_limits{$key} = $limit * 1024;
6179         }
6180
6181         foreach my $devname (keys %$devinfo) {
6182             die "found no device mapping information for device '$devname'\n"
6183                 if !$devinfo->{$devname}->{virtdev};
6184         }
6185
6186         # create empty/temp config
6187         if ($oldconf) {
6188             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6189             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6190         }
6191
6192         # allocate volumes
6193         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6194
6195         # print restore information to $fifofh
6196         foreach my $virtdev (sort keys %$virtdev_hash) {
6197             my $d = $virtdev_hash->{$virtdev};
6198             next if $d->{is_cloudinit}; # no need to restore cloudinit
6199
6200             my $storeid = $d->{storeid};
6201             my $volid = $d->{volid};
6202
6203             my $map_opts = '';
6204             if (my $limit = $storage_limits{$storeid}) {
6205                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6206             }
6207
6208             my $write_zeros = 1;
6209             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6210                 $write_zeros = 0;
6211             }
6212
6213             my $path = PVE::Storage::path($cfg, $volid);
6214
6215             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6216
6217             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6218         }
6219
6220         $fh->seek(0, 0) || die "seek failed - $!\n";
6221
6222         my $outfd = new IO::File ($tmpfn, "w") ||
6223             die "unable to write config for VM $vmid\n";
6224
6225         my $cookie = { netcount => 0 };
6226         while (defined(my $line = <$fh>)) {
6227             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6228         }
6229
6230         $fh->close();
6231         $outfd->close();
6232     };
6233
6234     eval {
6235         # enable interrupts
6236         local $SIG{INT} =
6237             local $SIG{TERM} =
6238             local $SIG{QUIT} =
6239             local $SIG{HUP} =
6240             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6241         local $SIG{ALRM} = sub { die "got timeout\n"; };
6242
6243         $oldtimeout = alarm($timeout);
6244
6245         my $parser = sub {
6246             my $line = shift;
6247
6248             print "$line\n";
6249
6250             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6251                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6252                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6253             } elsif ($line =~ m/^CTIME: /) {
6254                 # we correctly received the vma config, so we can disable
6255                 # the timeout now for disk allocation (set to 10 minutes, so
6256                 # that we always timeout if something goes wrong)
6257                 alarm(600);
6258                 &$print_devmap();
6259                 print $fifofh "done\n";
6260                 my $tmp = $oldtimeout || 0;
6261                 $oldtimeout = undef;
6262                 alarm($tmp);
6263                 close($fifofh);
6264             }
6265         };
6266
6267         print "restore vma archive: $dbg_cmdstring\n";
6268         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6269     };
6270     my $err = $@;
6271
6272     alarm($oldtimeout) if $oldtimeout;
6273
6274     $restore_deactivate_volumes->($cfg, $devinfo);
6275
6276     unlink $mapfifo;
6277     rmtree $tmpdir;
6278
6279     if ($err) {
6280         unlink $tmpfn;
6281         $restore_destroy_volumes->($cfg, $devinfo);
6282         die $err;
6283     }
6284
6285     rename($tmpfn, $conffile) ||
6286         die "unable to commit configuration file '$conffile'\n";
6287
6288     PVE::Cluster::cfs_update(); # make sure we read new file
6289
6290     eval { rescan($vmid, 1); };
6291     warn $@ if $@;
6292 }
6293
6294 sub restore_tar_archive {
6295     my ($archive, $vmid, $user, $opts) = @_;
6296
6297     if ($archive ne '-') {
6298         my $firstfile = tar_archive_read_firstfile($archive);
6299         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6300             if $firstfile ne 'qemu-server.conf';
6301     }
6302
6303     my $storecfg = PVE::Storage::config();
6304
6305     # avoid zombie disks when restoring over an existing VM -> cleanup first
6306     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6307     # skiplock=1 because qmrestore has set the 'create' lock itself already
6308     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6309     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6310
6311     my $tocmd = "/usr/lib/qemu-server/qmextract";
6312
6313     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6314     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6315     $tocmd .= ' --prealloc' if $opts->{prealloc};
6316     $tocmd .= ' --info' if $opts->{info};
6317
6318     # tar option "xf" does not autodetect compression when read from STDIN,
6319     # so we pipe to zcat
6320     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6321         PVE::Tools::shellquote("--to-command=$tocmd");
6322
6323     my $tmpdir = "/var/tmp/vzdumptmp$$";
6324     mkpath $tmpdir;
6325
6326     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6327     local $ENV{VZDUMP_VMID} = $vmid;
6328     local $ENV{VZDUMP_USER} = $user;
6329
6330     my $conffile = PVE::QemuConfig->config_file($vmid);
6331     my $tmpfn = "$conffile.$$.tmp";
6332
6333     # disable interrupts (always do cleanups)
6334     local $SIG{INT} =
6335         local $SIG{TERM} =
6336         local $SIG{QUIT} =
6337         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6338
6339     eval {
6340         # enable interrupts
6341         local $SIG{INT} =
6342             local $SIG{TERM} =
6343             local $SIG{QUIT} =
6344             local $SIG{HUP} =
6345             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6346
6347         if ($archive eq '-') {
6348             print "extracting archive from STDIN\n";
6349             run_command($cmd, input => "<&STDIN");
6350         } else {
6351             print "extracting archive '$archive'\n";
6352             run_command($cmd);
6353         }
6354
6355         return if $opts->{info};
6356
6357         # read new mapping
6358         my $map = {};
6359         my $statfile = "$tmpdir/qmrestore.stat";
6360         if (my $fd = IO::File->new($statfile, "r")) {
6361             while (defined (my $line = <$fd>)) {
6362                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6363                     $map->{$1} = $2 if $1;
6364                 } else {
6365                     print STDERR "unable to parse line in statfile - $line\n";
6366                 }
6367             }
6368             $fd->close();
6369         }
6370
6371         my $confsrc = "$tmpdir/qemu-server.conf";
6372
6373         my $srcfd = new IO::File($confsrc, "r") ||
6374             die "unable to open file '$confsrc'\n";
6375
6376         my $outfd = new IO::File ($tmpfn, "w") ||
6377             die "unable to write config for VM $vmid\n";
6378
6379         my $cookie = { netcount => 0 };
6380         while (defined (my $line = <$srcfd>)) {
6381             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6382         }
6383
6384         $srcfd->close();
6385         $outfd->close();
6386     };
6387     if (my $err = $@) {
6388         unlink $tmpfn;
6389         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6390         die $err;
6391     }
6392
6393     rmtree $tmpdir;
6394
6395     rename $tmpfn, $conffile ||
6396         die "unable to commit configuration file '$conffile'\n";
6397
6398     PVE::Cluster::cfs_update(); # make sure we read new file
6399
6400     eval { rescan($vmid, 1); };
6401     warn $@ if $@;
6402 };
6403
6404 sub foreach_storage_used_by_vm {
6405     my ($conf, $func) = @_;
6406
6407     my $sidhash = {};
6408
6409     foreach_drive($conf, sub {
6410         my ($ds, $drive) = @_;
6411         return if drive_is_cdrom($drive);
6412
6413         my $volid = $drive->{file};
6414
6415         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6416         $sidhash->{$sid} = $sid if $sid;
6417     });
6418
6419     foreach my $sid (sort keys %$sidhash) {
6420         &$func($sid);
6421     }
6422 }
6423
6424 my $qemu_snap_storage = {
6425     rbd => 1,
6426 };
6427 sub do_snapshots_with_qemu {
6428     my ($storecfg, $volid) = @_;
6429
6430     my $storage_name = PVE::Storage::parse_volume_id($volid);
6431     my $scfg = $storecfg->{ids}->{$storage_name};
6432
6433     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6434         return 1;
6435     }
6436
6437     if ($volid =~ m/\.(qcow2|qed)$/){
6438         return 1;
6439     }
6440
6441     return undef;
6442 }
6443
6444 sub qga_check_running {
6445     my ($vmid, $nowarn) = @_;
6446
6447     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6448     if ($@) {
6449         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6450         return 0;
6451     }
6452     return 1;
6453 }
6454
6455 sub template_create {
6456     my ($vmid, $conf, $disk) = @_;
6457
6458     my $storecfg = PVE::Storage::config();
6459
6460     foreach_drive($conf, sub {
6461         my ($ds, $drive) = @_;
6462
6463         return if drive_is_cdrom($drive);
6464         return if $disk && $ds ne $disk;
6465
6466         my $volid = $drive->{file};
6467         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6468
6469         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6470         $drive->{file} = $voliddst;
6471         $conf->{$ds} = print_drive($drive);
6472         PVE::QemuConfig->write_config($vmid, $conf);
6473     });
6474 }
6475
6476 sub convert_iscsi_path {
6477     my ($path) = @_;
6478
6479     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6480         my $portal = $1;
6481         my $target = $2;
6482         my $lun = $3;
6483
6484         my $initiator_name = get_initiator_name();
6485
6486         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6487                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6488     }
6489
6490     die "cannot convert iscsi path '$path', unkown format\n";
6491 }
6492
6493 sub qemu_img_convert {
6494     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6495
6496     my $storecfg = PVE::Storage::config();
6497     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6498     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6499
6500     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6501
6502     my $cachemode;
6503     my $src_path;
6504     my $src_is_iscsi = 0;
6505     my $src_format;
6506
6507     if ($src_storeid) {
6508         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6509         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6510         $src_format = qemu_img_format($src_scfg, $src_volname);
6511         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6512         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6513         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6514     } elsif (-f $src_volid) {
6515         $src_path = $src_volid;
6516         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6517             $src_format = $1;
6518         }
6519     }
6520
6521     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6522
6523     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6524     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6525     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6526     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6527
6528     my $cmd = [];
6529     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6530     push @$cmd, '-l', "snapshot.name=$snapname"
6531         if $snapname && $src_format && $src_format eq "qcow2";
6532     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6533     push @$cmd, '-T', $cachemode if defined($cachemode);
6534
6535     if ($src_is_iscsi) {
6536         push @$cmd, '--image-opts';
6537         $src_path = convert_iscsi_path($src_path);
6538     } elsif ($src_format) {
6539         push @$cmd, '-f', $src_format;
6540     }
6541
6542     if ($dst_is_iscsi) {
6543         push @$cmd, '--target-image-opts';
6544         $dst_path = convert_iscsi_path($dst_path);
6545     } else {
6546         push @$cmd, '-O', $dst_format;
6547     }
6548
6549     push @$cmd, $src_path;
6550
6551     if (!$dst_is_iscsi && $is_zero_initialized) {
6552         push @$cmd, "zeroinit:$dst_path";
6553     } else {
6554         push @$cmd, $dst_path;
6555     }
6556
6557     my $parser = sub {
6558         my $line = shift;
6559         if($line =~ m/\((\S+)\/100\%\)/){
6560             my $percent = $1;
6561             my $transferred = int($size * $percent / 100);
6562             my $remaining = $size - $transferred;
6563
6564             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6565         }
6566
6567     };
6568
6569     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6570     my $err = $@;
6571     die "copy failed: $err" if $err;
6572 }
6573
6574 sub qemu_img_format {
6575     my ($scfg, $volname) = @_;
6576
6577     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6578         return $1;
6579     } else {
6580         return "raw";
6581     }
6582 }
6583
6584 sub qemu_drive_mirror {
6585     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6586
6587     $jobs = {} if !$jobs;
6588
6589     my $qemu_target;
6590     my $format;
6591     $jobs->{"drive-$drive"} = {};
6592
6593     if ($dst_volid =~ /^nbd:/) {
6594         $qemu_target = $dst_volid;
6595         $format = "nbd";
6596     } else {
6597         my $storecfg = PVE::Storage::config();
6598         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6599
6600         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6601
6602         $format = qemu_img_format($dst_scfg, $dst_volname);
6603
6604         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6605
6606         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6607     }
6608
6609     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6610     $opts->{format} = $format if $format;
6611
6612     if (defined($src_bitmap)) {
6613         $opts->{sync} = 'incremental';
6614         $opts->{bitmap} = $src_bitmap;
6615         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6616     }
6617
6618     if (defined($bwlimit)) {
6619         $opts->{speed} = $bwlimit * 1024;
6620         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6621     } else {
6622         print "drive mirror is starting for drive-$drive\n";
6623     }
6624
6625     # if a job already runs for this device we get an error, catch it for cleanup
6626     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6627     if (my $err = $@) {
6628         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6629         warn "$@\n" if $@;
6630         die "mirroring error: $err\n";
6631     }
6632
6633     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6634 }
6635
6636 # $completion can be either
6637 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6638 # 'cancel': wait until all jobs are ready, block-job-cancel them
6639 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6640 sub qemu_drive_mirror_monitor {
6641     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6642
6643     $completion //= 'complete';
6644
6645     eval {
6646         my $err_complete = 0;
6647
6648         while (1) {
6649             die "storage migration timed out\n" if $err_complete > 300;
6650
6651             my $stats = mon_cmd($vmid, "query-block-jobs");
6652
6653             my $running_mirror_jobs = {};
6654             foreach my $stat (@$stats) {
6655                 next if $stat->{type} ne 'mirror';
6656                 $running_mirror_jobs->{$stat->{device}} = $stat;
6657             }
6658
6659             my $readycounter = 0;
6660
6661             foreach my $job (keys %$jobs) {
6662
6663                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6664                     print "$job : finished\n";
6665                     delete $jobs->{$job};
6666                     next;
6667                 }
6668
6669                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6670
6671                 my $busy = $running_mirror_jobs->{$job}->{busy};
6672                 my $ready = $running_mirror_jobs->{$job}->{ready};
6673                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6674                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6675                     my $remaining = $total - $transferred;
6676                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6677
6678                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6679                 }
6680
6681                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6682             }
6683
6684             last if scalar(keys %$jobs) == 0;
6685
6686             if ($readycounter == scalar(keys %$jobs)) {
6687                 print "all mirroring jobs are ready \n";
6688                 last if $completion eq 'skip'; #do the complete later
6689
6690                 if ($vmiddst && $vmiddst != $vmid) {
6691                     my $agent_running = $qga && qga_check_running($vmid);
6692                     if ($agent_running) {
6693                         print "freeze filesystem\n";
6694                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6695                     } else {
6696                         print "suspend vm\n";
6697                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6698                     }
6699
6700                     # if we clone a disk for a new target vm, we don't switch the disk
6701                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6702
6703                     if ($agent_running) {
6704                         print "unfreeze filesystem\n";
6705                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6706                     } else {
6707                         print "resume vm\n";
6708                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6709                     }
6710
6711                     last;
6712                 } else {
6713
6714                     foreach my $job (keys %$jobs) {
6715                         # try to switch the disk if source and destination are on the same guest
6716                         print "$job: Completing block job...\n";
6717
6718                         my $op;
6719                         if ($completion eq 'complete') {
6720                             $op = 'block-job-complete';
6721                         } elsif ($completion eq 'cancel') {
6722                             $op = 'block-job-cancel';
6723                         } else {
6724                             die "invalid completion value: $completion\n";
6725                         }
6726                         eval { mon_cmd($vmid, $op, device => $job) };
6727                         if ($@ =~ m/cannot be completed/) {
6728                             print "$job: Block job cannot be completed, try again.\n";
6729                             $err_complete++;
6730                         }else {
6731                             print "$job: Completed successfully.\n";
6732                             $jobs->{$job}->{complete} = 1;
6733                         }
6734                     }
6735                 }
6736             }
6737             sleep 1;
6738         }
6739     };
6740     my $err = $@;
6741
6742     if ($err) {
6743         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6744         die "mirroring error: $err";
6745     }
6746
6747 }
6748
6749 sub qemu_blockjobs_cancel {
6750     my ($vmid, $jobs) = @_;
6751
6752     foreach my $job (keys %$jobs) {
6753         print "$job: Cancelling block job\n";
6754         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6755         $jobs->{$job}->{cancel} = 1;
6756     }
6757
6758     while (1) {
6759         my $stats = mon_cmd($vmid, "query-block-jobs");
6760
6761         my $running_jobs = {};
6762         foreach my $stat (@$stats) {
6763             $running_jobs->{$stat->{device}} = $stat;
6764         }
6765
6766         foreach my $job (keys %$jobs) {
6767
6768             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6769                 print "$job: Done.\n";
6770                 delete $jobs->{$job};
6771             }
6772         }
6773
6774         last if scalar(keys %$jobs) == 0;
6775
6776         sleep 1;
6777     }
6778 }
6779
6780 sub clone_disk {
6781     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6782         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6783
6784     my $newvolid;
6785
6786     if (!$full) {
6787         print "create linked clone of drive $drivename ($drive->{file})\n";
6788         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6789         push @$newvollist, $newvolid;
6790     } else {
6791
6792         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6793         $storeid = $storage if $storage;
6794
6795         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6796         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6797
6798         print "create full clone of drive $drivename ($drive->{file})\n";
6799         my $name = undef;
6800         if (drive_is_cloudinit($drive)) {
6801             $name = "vm-$newvmid-cloudinit";
6802             $name .= ".$dst_format" if $dst_format ne 'raw';
6803             $snapname = undef;
6804             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6805         } elsif ($drivename eq 'efidisk0') {
6806             $size = get_efivars_size($conf);
6807         }
6808         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6809         push @$newvollist, $newvolid;
6810
6811         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6812
6813         if (drive_is_cloudinit($drive)) {
6814             goto no_data_clone;
6815         }
6816
6817         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6818         if (!$running || $snapname) {
6819             # TODO: handle bwlimits
6820             if ($drivename eq 'efidisk0') {
6821                 # the relevant data on the efidisk may be smaller than the source
6822                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6823                 # that is given by the OVMF_VARS.fd
6824                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6825                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6826                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6827             } else {
6828                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6829             }
6830         } else {
6831
6832             my $kvmver = get_running_qemu_version ($vmid);
6833             if (!min_version($kvmver, 2, 7)) {
6834                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6835                     if $drive->{iothread};
6836             }
6837
6838             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6839         }
6840     }
6841
6842 no_data_clone:
6843     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6844
6845     my $disk = $drive;
6846     $disk->{format} = undef;
6847     $disk->{file} = $newvolid;
6848     $disk->{size} = $size;
6849
6850     return $disk;
6851 }
6852
6853 sub get_running_qemu_version {
6854     my ($vmid) = @_;
6855     my $res = mon_cmd($vmid, "query-version");
6856     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6857 }
6858
6859 sub qemu_use_old_bios_files {
6860     my ($machine_type) = @_;
6861
6862     return if !$machine_type;
6863
6864     my $use_old_bios_files = undef;
6865
6866     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6867         $machine_type = $1;
6868         $use_old_bios_files = 1;
6869     } else {
6870         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6871         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6872         # load new efi bios files on migration. So this hack is required to allow
6873         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6874         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6875         $use_old_bios_files = !min_version($version, 2, 4);
6876     }
6877
6878     return ($use_old_bios_files, $machine_type);
6879 }
6880
6881 sub get_efivars_size {
6882     my ($conf) = @_;
6883     my $arch = get_vm_arch($conf);
6884     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6885     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6886     return -s $ovmf_vars;
6887 }
6888
6889 sub update_efidisk_size {
6890     my ($conf) = @_;
6891
6892     return if !defined($conf->{efidisk0});
6893
6894     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6895     $disk->{size} = get_efivars_size($conf);
6896     $conf->{efidisk0} = print_drive($disk);
6897
6898     return;
6899 }
6900
6901 sub create_efidisk($$$$$) {
6902     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6903
6904     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6905     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6906
6907     my $vars_size_b = -s $ovmf_vars;
6908     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6909     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6910     PVE::Storage::activate_volumes($storecfg, [$volid]);
6911
6912     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6913     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6914
6915     return ($volid, $size/1024);
6916 }
6917
6918 sub vm_iothreads_list {
6919     my ($vmid) = @_;
6920
6921     my $res = mon_cmd($vmid, 'query-iothreads');
6922
6923     my $iothreads = {};
6924     foreach my $iothread (@$res) {
6925         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6926     }
6927
6928     return $iothreads;
6929 }
6930
6931 sub scsihw_infos {
6932     my ($conf, $drive) = @_;
6933
6934     my $maxdev = 0;
6935
6936     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6937         $maxdev = 7;
6938     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6939         $maxdev = 1;
6940     } else {
6941         $maxdev = 256;
6942     }
6943
6944     my $controller = int($drive->{index} / $maxdev);
6945     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6946
6947     return ($maxdev, $controller, $controller_prefix);
6948 }
6949
6950 sub windows_version {
6951     my ($ostype) = @_;
6952
6953     return 0 if !$ostype;
6954
6955     my $winversion = 0;
6956
6957     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6958         $winversion = 5;
6959     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6960         $winversion = 6;
6961     } elsif ($ostype =~ m/^win(\d+)$/) {
6962         $winversion = $1;
6963     }
6964
6965     return $winversion;
6966 }
6967
6968 sub resolve_dst_disk_format {
6969         my ($storecfg, $storeid, $src_volname, $format) = @_;
6970         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6971
6972         if (!$format) {
6973             # if no target format is specified, use the source disk format as hint
6974             if ($src_volname) {
6975                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6976                 $format = qemu_img_format($scfg, $src_volname);
6977             } else {
6978                 return $defFormat;
6979             }
6980         }
6981
6982         # test if requested format is supported - else use default
6983         my $supported = grep { $_ eq $format } @$validFormats;
6984         $format = $defFormat if !$supported;
6985         return $format;
6986 }
6987
6988 # NOTE: if this logic changes, please update docs & possibly gui logic
6989 sub find_vmstate_storage {
6990     my ($conf, $storecfg) = @_;
6991
6992     # first, return storage from conf if set
6993     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
6994
6995     my ($target, $shared, $local);
6996
6997     foreach_storage_used_by_vm($conf, sub {
6998         my ($sid) = @_;
6999         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7000         my $dst = $scfg->{shared} ? \$shared : \$local;
7001         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7002     });
7003
7004     # second, use shared storage where VM has at least one disk
7005     # third, use local storage where VM has at least one disk
7006     # fall back to local storage
7007     $target = $shared // $local // 'local';
7008
7009     return $target;
7010 }
7011
7012 sub generate_uuid {
7013     my ($uuid, $uuid_str);
7014     UUID::generate($uuid);
7015     UUID::unparse($uuid, $uuid_str);
7016     return $uuid_str;
7017 }
7018
7019 sub generate_smbios1_uuid {
7020     return "uuid=".generate_uuid();
7021 }
7022
7023 sub nbd_stop {
7024     my ($vmid) = @_;
7025
7026     mon_cmd($vmid, 'nbd-server-stop');
7027 }
7028
7029 sub create_reboot_request {
7030     my ($vmid) = @_;
7031     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7032         or die "failed to create reboot trigger file: $!\n";
7033     close($fh);
7034 }
7035
7036 sub clear_reboot_request {
7037     my ($vmid) = @_;
7038     my $path = "/run/qemu-server/$vmid.reboot";
7039     my $res = 0;
7040
7041     $res = unlink($path);
7042     die "could not remove reboot request for $vmid: $!"
7043         if !$res && $! != POSIX::ENOENT;
7044
7045     return $res;
7046 }
7047
7048 # bash completion helper
7049
7050 sub complete_backup_archives {
7051     my ($cmdname, $pname, $cvalue) = @_;
7052
7053     my $cfg = PVE::Storage::config();
7054
7055     my $storeid;
7056
7057     if ($cvalue =~ m/^([^:]+):/) {
7058         $storeid = $1;
7059     }
7060
7061     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7062
7063     my $res = [];
7064     foreach my $id (keys %$data) {
7065         foreach my $item (@{$data->{$id}}) {
7066             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7067             push @$res, $item->{volid} if defined($item->{volid});
7068         }
7069     }
7070
7071     return $res;
7072 }
7073
7074 my $complete_vmid_full = sub {
7075     my ($running) = @_;
7076
7077     my $idlist = vmstatus();
7078
7079     my $res = [];
7080
7081     foreach my $id (keys %$idlist) {
7082         my $d = $idlist->{$id};
7083         if (defined($running)) {
7084             next if $d->{template};
7085             next if $running && $d->{status} ne 'running';
7086             next if !$running && $d->{status} eq 'running';
7087         }
7088         push @$res, $id;
7089
7090     }
7091     return $res;
7092 };
7093
7094 sub complete_vmid {
7095     return &$complete_vmid_full();
7096 }
7097
7098 sub complete_vmid_stopped {
7099     return &$complete_vmid_full(0);
7100 }
7101
7102 sub complete_vmid_running {
7103     return &$complete_vmid_full(1);
7104 }
7105
7106 sub complete_storage {
7107
7108     my $cfg = PVE::Storage::config();
7109     my $ids = $cfg->{ids};
7110
7111     my $res = [];
7112     foreach my $sid (keys %$ids) {
7113         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7114         next if !$ids->{$sid}->{content}->{images};
7115         push @$res, $sid;
7116     }
7117
7118     return $res;
7119 }
7120
7121 sub complete_migration_storage {
7122     my ($cmd, $param, $current_value, $all_args) = @_;
7123
7124     my $targetnode = @$all_args[1];
7125
7126     my $cfg = PVE::Storage::config();
7127     my $ids = $cfg->{ids};
7128
7129     my $res = [];
7130     foreach my $sid (keys %$ids) {
7131         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7132         next if !$ids->{$sid}->{content}->{images};
7133         push @$res, $sid;
7134     }
7135
7136     return $res;
7137 }
7138
7139 1;