PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename_cache;
 111 sub nodename {
 112     $nodename_cache //= PVE::INotify::nodename();
 113     return $nodename_cache;
 114 }
 115
 116 my $watchdog_fmt = {
 117     model => {
 118         default_key => 1,
 119         type => 'string',
 120         enum => [qw(i6300esb ib700)],
 121         description => "Watchdog type to emulate.",
 122         default => 'i6300esb',
 123         optional => 1,
 124     },
 125     action => {
 126         type => 'string',
 127         enum => [qw(reset shutdown poweroff pause debug none)],
 128         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 129         optional => 1,
 130     },
 131 };
 132 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 133
 134 my $agent_fmt = {
 135     enabled => {
 136         description => "Enable/disable Qemu GuestAgent.",
 137         type => 'boolean',
 138         default => 0,
 139         default_key => 1,
 140     },
 141     fstrim_cloned_disks => {
 142         description => "Run fstrim after cloning/moving a disk.",
 143         type => 'boolean',
 144         optional => 1,
 145         default => 0
 146     },
 147     type => {
 148         description => "Select the agent type",
 149         type => 'string',
 150         default => 'virtio',
 151         optional => 1,
 152         enum => [qw(virtio isa)],
 153     },
 154 };
 155
 156 my $vga_fmt = {
 157     type => {
 158         description => "Select the VGA type.",
 159         type => 'string',
 160         default => 'std',
 161         optional => 1,
 162         default_key => 1,
 163         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 164     },
 165     memory => {
 166         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 167         type => 'integer',
 168         optional => 1,
 169         minimum => 4,
 170         maximum => 512,
 171     },
 172 };
 173
 174 my $ivshmem_fmt = {
 175     size => {
 176         type => 'integer',
 177         minimum => 1,
 178         description => "The size of the file in MB.",
 179     },
 180     name => {
 181         type => 'string',
 182         pattern => '[a-zA-Z0-9\-]+',
 183         optional => 1,
 184         format_description => 'string',
 185         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 186     },
 187 };
 188
 189 my $audio_fmt = {
 190     device => {
 191         type => 'string',
 192         enum => [qw(ich9-intel-hda intel-hda AC97)],
 193         description =>  "Configure an audio device."
 194     },
 195     driver =>  {
 196         type => 'string',
 197         enum => ['spice'],
 198         default => 'spice',
 199         optional => 1,
 200         description => "Driver backend for the audio device."
 201     },
 202 };
 203
 204 my $spice_enhancements_fmt = {
 205     foldersharing => {
 206         type => 'boolean',
 207         optional => 1,
 208         default => '0',
 209         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 210     },
 211     videostreaming =>  {
 212         type => 'string',
 213         enum => ['off', 'all', 'filter'],
 214         default => 'off',
 215         optional => 1,
 216         description => "Enable video streaming. Uses compression for detected video streams."
 217     },
 218 };
 219
 220 my $rng_fmt = {
 221     source => {
 222         type => 'string',
 223         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 224         default_key => 1,
 225         description => "The file on the host to gather entropy from. In most"
 226                      . " cases /dev/urandom should be preferred over /dev/random"
 227                      . " to avoid entropy-starvation issues on the host. Using"
 228                      . " urandom does *not* decrease security in any meaningful"
 229                      . " way, as it's still seeded from real entropy, and the"
 230                      . " bytes provided will most likely be mixed with real"
 231                      . " entropy on the guest as well. /dev/hwrng can be used"
 232                      . " to pass through a hardware RNG from the host.",
 233     },
 234     max_bytes => {
 235         type => 'integer',
 236         description => "Maximum bytes of entropy injected into the guest every"
 237                      . " 'period' milliseconds. Prefer a lower value when using"
 238                      . " /dev/random as source. Use 0 to disable limiting"
 239                      . " (potentially dangerous!).",
 240         optional => 1,
 241
 242         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 243         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 244         # of overwhelming the host, provided we're reading from /dev/urandom
 245         default => 1024,
 246     },
 247     period => {
 248         type => 'integer',
 249         description => "Every 'period' milliseconds the entropy-injection quota"
 250                      . " is reset, allowing the guest to retrieve another"
 251                      . " 'max_bytes' of entropy.",
 252         optional => 1,
 253         default => 1000,
 254     },
 255 };
 256
 257 my $confdesc = {
 258     onboot => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Specifies whether a VM will be started during system bootup.",
 262         default => 0,
 263     },
 264     autostart => {
 265         optional => 1,
 266         type => 'boolean',
 267         description => "Automatic restart after crash (currently ignored).",
 268         default => 0,
 269     },
 270     hotplug => {
 271         optional => 1,
 272         type => 'string', format => 'pve-hotplug-features',
 273         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 274         default => 'network,disk,usb',
 275     },
 276     reboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 280         default => 1,
 281     },
 282     lock => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Lock/unlock the VM.",
 286         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 287     },
 288     cpulimit => {
 289         optional => 1,
 290         type => 'number',
 291         description => "Limit of CPU usage.",
 292         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 293         minimum => 0,
 294         maximum => 128,
 295         default => 0,
 296     },
 297     cpuunits => {
 298         optional => 1,
 299         type => 'integer',
 300         description => "CPU weight for a VM.",
 301         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 302         minimum => 2,
 303         maximum => 262144,
 304         default => 1024,
 305     },
 306     memory => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 310         minimum => 16,
 311         default => 512,
 312     },
 313     balloon => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 317         minimum => 0,
 318     },
 319     shares => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 323         minimum => 0,
 324         maximum => 50000,
 325         default => 1000,
 326     },
 327     keyboard => {
 328         optional => 1,
 329         type => 'string',
 330         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 331                        "It should not be necessary to set it.",
 332         enum => PVE::Tools::kvmkeymaplist(),
 333         default => undef,
 334     },
 335     name => {
 336         optional => 1,
 337         type => 'string', format => 'dns-name',
 338         description => "Set a name for the VM. Only used on the configuration web interface.",
 339     },
 340     scsihw => {
 341         optional => 1,
 342         type => 'string',
 343         description => "SCSI controller model",
 344         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 345         default => 'lsi',
 346     },
 347     description => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 351     },
 352     ostype => {
 353         optional => 1,
 354         type => 'string',
 355         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 356         description => "Specify guest operating system.",
 357         verbose_description => <<EODESC,
 358 Specify guest operating system. This is used to enable special
 359 optimization/features for specific operating systems:
 360
 361 [horizontal]
 362 other;; unspecified OS
 363 wxp;; Microsoft Windows XP
 364 w2k;; Microsoft Windows 2000
 365 w2k3;; Microsoft Windows 2003
 366 w2k8;; Microsoft Windows 2008
 367 wvista;; Microsoft Windows Vista
 368 win7;; Microsoft Windows 7
 369 win8;; Microsoft Windows 8/2012/2012r2
 370 win10;; Microsoft Windows 10/2016
 371 l24;; Linux 2.4 Kernel
 372 l26;; Linux 2.6 - 5.X Kernel
 373 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 374 EODESC
 375     },
 376     boot => {
 377         optional => 1,
 378         type => 'string',
 379         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 380         pattern => '[acdn]{1,4}',
 381         default => 'cdn',
 382     },
 383     bootdisk => {
 384         optional => 1,
 385         type => 'string', format => 'pve-qm-bootdisk',
 386         description => "Enable booting from specified disk.",
 387         pattern => '(ide|sata|scsi|virtio)\d+',
 388     },
 389     smp => {
 390         optional => 1,
 391         type => 'integer',
 392         description => "The number of CPUs. Please use option -sockets instead.",
 393         minimum => 1,
 394         default => 1,
 395     },
 396     sockets => {
 397         optional => 1,
 398         type => 'integer',
 399         description => "The number of CPU sockets.",
 400         minimum => 1,
 401         default => 1,
 402     },
 403     cores => {
 404         optional => 1,
 405         type => 'integer',
 406         description => "The number of cores per socket.",
 407         minimum => 1,
 408         default => 1,
 409     },
 410     numa => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable NUMA.",
 414         default => 0,
 415     },
 416     hugepages => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Enable/disable hugepages memory.",
 420         enum => [qw(any 2 1024)],
 421     },
 422     vcpus => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "Number of hotplugged vcpus.",
 426         minimum => 1,
 427         default => 0,
 428     },
 429     acpi => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable ACPI.",
 433         default => 1,
 434     },
 435     agent => {
 436         optional => 1,
 437         description => "Enable/disable Qemu GuestAgent and its properties.",
 438         type => 'string',
 439         format => $agent_fmt,
 440     },
 441     kvm => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable KVM hardware virtualization.",
 445         default => 1,
 446     },
 447     tdf => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Enable/disable time drift fix.",
 451         default => 0,
 452     },
 453     localtime => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 457     },
 458     freeze => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 462     },
 463     vga => {
 464         optional => 1,
 465         type => 'string', format => $vga_fmt,
 466         description => "Configure the VGA hardware.",
 467         verbose_description => "Configure the VGA Hardware. If you want to use ".
 468             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 469             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 470             "is 'std' for all OS types besides some Windows versions (XP and " .
 471             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 472             "display server. For win* OS you can select how many independent " .
 473             "displays you want, Linux guests can add displays them self.\n".
 474             "You can also run without any graphic card, using a serial device as terminal.",
 475     },
 476     watchdog => {
 477         optional => 1,
 478         type => 'string', format => 'pve-qm-watchdog',
 479         description => "Create a virtual hardware watchdog device.",
 480         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 481             " (by a guest action), the watchdog must be periodically polled " .
 482             "by an agent inside the guest or else the watchdog will reset " .
 483             "the guest (or execute the respective action specified)",
 484     },
 485     startdate => {
 486         optional => 1,
 487         type => 'string',
 488         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 489         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 490         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 491         default => 'now',
 492     },
 493     startup =>  get_standard_option('pve-startup-order'),
 494     template => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable Template.",
 498         default => 0,
 499     },
 500     args => {
 501         optional => 1,
 502         type => 'string',
 503         description => "Arbitrary arguments passed to kvm.",
 504         verbose_description => <<EODESCR,
 505 Arbitrary arguments passed to kvm, for example:
 506
 507 args: -no-reboot -no-hpet
 508
 509 NOTE: this option is for experts only.
 510 EODESCR
 511     },
 512     tablet => {
 513         optional => 1,
 514         type => 'boolean',
 515         default => 1,
 516         description => "Enable/disable the USB tablet device.",
 517         verbose_description => "Enable/disable the USB tablet device. This device is " .
 518             "usually needed to allow absolute mouse positioning with VNC. " .
 519             "Else the mouse runs out of sync with normal VNC clients. " .
 520             "If you're running lots of console-only guests on one host, " .
 521             "you may consider disabling this to save some context switches. " .
 522             "This is turned off by default if you use spice (-vga=qxl).",
 523     },
 524     migrate_speed => {
 525         optional => 1,
 526         type => 'integer',
 527         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 528         minimum => 0,
 529         default => 0,
 530     },
 531     migrate_downtime => {
 532         optional => 1,
 533         type => 'number',
 534         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 535         minimum => 0,
 536         default => 0.1,
 537     },
 538     cdrom => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-ide',
 541         typetext => '<volume>',
 542         description => "This is an alias for option -ide2",
 543     },
 544     cpu => {
 545         optional => 1,
 546         description => "Emulated CPU type.",
 547         type => 'string',
 548         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 549     },
 550     parent => get_standard_option('pve-snapshot-name', {
 551         optional => 1,
 552         description => "Parent snapshot name. This is used internally, and should not be modified.",
 553     }),
 554     snaptime => {
 555         optional => 1,
 556         description => "Timestamp for snapshots.",
 557         type => 'integer',
 558         minimum => 0,
 559     },
 560     vmstate => {
 561         optional => 1,
 562         type => 'string', format => 'pve-volume-id',
 563         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 564     },
 565     vmstatestorage => get_standard_option('pve-storage-id', {
 566         description => "Default storage for VM state volumes/files.",
 567         optional => 1,
 568     }),
 569     runningmachine => get_standard_option('pve-qemu-machine', {
 570         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 571     }),
 572     machine => get_standard_option('pve-qemu-machine'),
 573     arch => {
 574         description => "Virtual processor architecture. Defaults to the host.",
 575         optional => 1,
 576         type => 'string',
 577         enum => [qw(x86_64 aarch64)],
 578     },
 579     smbios1 => {
 580         description => "Specify SMBIOS type 1 fields.",
 581         type => 'string', format => 'pve-qm-smbios1',
 582         maxLength => 512,
 583         optional => 1,
 584     },
 585     protection => {
 586         optional => 1,
 587         type => 'boolean',
 588         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 589         default => 0,
 590     },
 591     bios => {
 592         optional => 1,
 593         type => 'string',
 594         enum => [ qw(seabios ovmf) ],
 595         description => "Select BIOS implementation.",
 596         default => 'seabios',
 597     },
 598     vmgenid => {
 599         type => 'string',
 600         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 601         format_description => 'UUID',
 602         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 603         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 604             " 128-bit integer value identifier to the guest OS. This allows to".
 605             " notify the guest operating system when the virtual machine is".
 606             " executed with a different configuration (e.g. snapshot execution".
 607             " or creation from a template). The guest operating system notices".
 608             " the change, and is then able to react as appropriate by marking".
 609             " its copies of distributed databases as dirty, re-initializing its".
 610             " random number generator, etc.\n".
 611             "Note that auto-creation only works when done throug API/CLI create".
 612             " or update methods, but not when manually editing the config file.",
 613         default => "1 (autogenerated)",
 614         optional => 1,
 615     },
 616     hookscript => {
 617         type => 'string',
 618         format => 'pve-volume-id',
 619         optional => 1,
 620         description => "Script that will be executed during various steps in the vms lifetime.",
 621     },
 622     ivshmem => {
 623         type => 'string',
 624         format => $ivshmem_fmt,
 625         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 626         optional => 1,
 627     },
 628     audio0 => {
 629         type => 'string',
 630         format => $audio_fmt,
 631         description => "Configure a audio device, useful in combination with QXL/Spice.",
 632         optional => 1
 633     },
 634     spice_enhancements => {
 635         type => 'string',
 636         format => $spice_enhancements_fmt,
 637         description => "Configure additional enhancements for SPICE.",
 638         optional => 1
 639     },
 640     tags => {
 641         type => 'string', format => 'pve-tag-list',
 642         description => 'Tags of the VM. This is only meta information.',
 643         optional => 1,
 644     },
 645     rng0 => {
 646         type => 'string',
 647         format => $rng_fmt,
 648         description => "Configure a VirtIO-based Random Number Generator.",
 649         optional => 1,
 650     },
 651 };
 652
 653 my $cicustom_fmt = {
 654     meta => {
 655         type => 'string',
 656         optional => 1,
 657         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 658         format => 'pve-volume-id',
 659         format_description => 'volume',
 660     },
 661     network => {
 662         type => 'string',
 663         optional => 1,
 664         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 665         format => 'pve-volume-id',
 666         format_description => 'volume',
 667     },
 668     user => {
 669         type => 'string',
 670         optional => 1,
 671         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 672         format => 'pve-volume-id',
 673         format_description => 'volume',
 674     },
 675 };
 676 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 677
 678 my $confdesc_cloudinit = {
 679     citype => {
 680         optional => 1,
 681         type => 'string',
 682         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 683         enum => ['configdrive2', 'nocloud'],
 684     },
 685     ciuser => {
 686         optional => 1,
 687         type => 'string',
 688         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 689     },
 690     cipassword => {
 691         optional => 1,
 692         type => 'string',
 693         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 694     },
 695     cicustom => {
 696         optional => 1,
 697         type => 'string',
 698         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 699         format => 'pve-qm-cicustom',
 700     },
 701     searchdomain => {
 702         optional => 1,
 703         type => 'string',
 704         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 705     },
 706     nameserver => {
 707         optional => 1,
 708         type => 'string', format => 'address-list',
 709         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 710     },
 711     sshkeys => {
 712         optional => 1,
 713         type => 'string',
 714         format => 'urlencoded',
 715         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 716     },
 717 };
 718
 719 # what about other qemu settings ?
 720 #cpu => 'string',
 721 #machine => 'string',
 722 #fda => 'file',
 723 #fdb => 'file',
 724 #mtdblock => 'file',
 725 #sd => 'file',
 726 #pflash => 'file',
 727 #snapshot => 'bool',
 728 #bootp => 'file',
 729 ##tftp => 'dir',
 730 ##smb => 'dir',
 731 #kernel => 'file',
 732 #append => 'string',
 733 #initrd => 'file',
 734 ##soundhw => 'string',
 735
 736 while (my ($k, $v) = each %$confdesc) {
 737     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 738 }
 739
 740 my $MAX_USB_DEVICES = 5;
 741 my $MAX_NETS = 32;
 742 my $MAX_HOSTPCI_DEVICES = 16;
 743 my $MAX_SERIAL_PORTS = 4;
 744 my $MAX_PARALLEL_PORTS = 3;
 745 my $MAX_NUMA = 8;
 746
 747 my $numa_fmt = {
 748     cpus => {
 749         type => "string",
 750         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 751         description => "CPUs accessing this NUMA node.",
 752         format_description => "id[-id];...",
 753     },
 754     memory => {
 755         type => "number",
 756         description => "Amount of memory this NUMA node provides.",
 757         optional => 1,
 758     },
 759     hostnodes => {
 760         type => "string",
 761         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 762         description => "Host NUMA nodes to use.",
 763         format_description => "id[-id];...",
 764         optional => 1,
 765     },
 766     policy => {
 767         type => 'string',
 768         enum => [qw(preferred bind interleave)],
 769         description => "NUMA allocation policy.",
 770         optional => 1,
 771     },
 772 };
 773 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 774 my $numadesc = {
 775     optional => 1,
 776     type => 'string', format => $numa_fmt,
 777     description => "NUMA topology.",
 778 };
 779 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 780
 781 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 782     $confdesc->{"numa$i"} = $numadesc;
 783 }
 784
 785 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 786                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 787                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 788 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 789
 790 my $net_fmt_bridge_descr = <<__EOD__;
 791 Bridge to attach the network device to. The Proxmox VE standard bridge
 792 is called 'vmbr0'.
 793
 794 If you do not specify a bridge, we create a kvm user (NATed) network
 795 device, which provides DHCP and DNS services. The following addresses
 796 are used:
 797
 798  10.0.2.2   Gateway
 799  10.0.2.3   DNS Server
 800  10.0.2.4   SMB Server
 801
 802 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 803 __EOD__
 804
 805 my $net_fmt = {
 806     macaddr  => get_standard_option('mac-addr', {
 807         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 808     }),
 809     model => {
 810         type => 'string',
 811         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 812         enum => $nic_model_list,
 813         default_key => 1,
 814     },
 815     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 816     bridge => {
 817         type => 'string',
 818         description => $net_fmt_bridge_descr,
 819         format_description => 'bridge',
 820         optional => 1,
 821     },
 822     queues => {
 823         type => 'integer',
 824         minimum => 0, maximum => 16,
 825         description => 'Number of packet queues to be used on the device.',
 826         optional => 1,
 827     },
 828     rate => {
 829         type => 'number',
 830         minimum => 0,
 831         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 832         optional => 1,
 833     },
 834     tag => {
 835         type => 'integer',
 836         minimum => 1, maximum => 4094,
 837         description => 'VLAN tag to apply to packets on this interface.',
 838         optional => 1,
 839     },
 840     trunks => {
 841         type => 'string',
 842         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 843         description => 'VLAN trunks to pass through this interface.',
 844         format_description => 'vlanid[;vlanid...]',
 845         optional => 1,
 846     },
 847     firewall => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be protected by the firewall.',
 850         optional => 1,
 851     },
 852     link_down => {
 853         type => 'boolean',
 854         description => 'Whether this interface should be disconnected (like pulling the plug).',
 855         optional => 1,
 856     },
 857 };
 858
 859 my $netdesc = {
 860     optional => 1,
 861     type => 'string', format => $net_fmt,
 862     description => "Specify network devices.",
 863 };
 864
 865 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 866
 867 my $ipconfig_fmt = {
 868     ip => {
 869         type => 'string',
 870         format => 'pve-ipv4-config',
 871         format_description => 'IPv4Format/CIDR',
 872         description => 'IPv4 address in CIDR format.',
 873         optional => 1,
 874         default => 'dhcp',
 875     },
 876     gw => {
 877         type => 'string',
 878         format => 'ipv4',
 879         format_description => 'GatewayIPv4',
 880         description => 'Default gateway for IPv4 traffic.',
 881         optional => 1,
 882         requires => 'ip',
 883     },
 884     ip6 => {
 885         type => 'string',
 886         format => 'pve-ipv6-config',
 887         format_description => 'IPv6Format/CIDR',
 888         description => 'IPv6 address in CIDR format.',
 889         optional => 1,
 890         default => 'dhcp',
 891     },
 892     gw6 => {
 893         type => 'string',
 894         format => 'ipv6',
 895         format_description => 'GatewayIPv6',
 896         description => 'Default gateway for IPv6 traffic.',
 897         optional => 1,
 898         requires => 'ip6',
 899     },
 900 };
 901 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 902 my $ipconfigdesc = {
 903     optional => 1,
 904     type => 'string', format => 'pve-qm-ipconfig',
 905     description => <<'EODESCR',
 906 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 907
 908 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 909
 910 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 911 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 912
 913 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 914 EODESCR
 915 };
 916 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 917
 918 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 919     $confdesc->{"net$i"} = $netdesc;
 920     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 921 }
 922
 923 foreach my $key (keys %$confdesc_cloudinit) {
 924     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 925 }
 926
 927 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 928 sub verify_volume_id_or_qm_path {
 929     my ($volid, $noerr) = @_;
 930
 931     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 932         return $volid;
 933     }
 934
 935     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 936     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 937     if ($@) {
 938         return undef if $noerr;
 939         die $@;
 940     }
 941     return $volid;
 942 }
 943
 944 my $usb_fmt = {
 945     host => {
 946         default_key => 1,
 947         type => 'string', format => 'pve-qm-usb-device',
 948         format_description => 'HOSTUSBDEVICE|spice',
 949         description => <<EODESCR,
 950 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 951
 952  'bus-port(.port)*' (decimal numbers) or
 953  'vendor_id:product_id' (hexadeciaml numbers) or
 954  'spice'
 955
 956 You can use the 'lsusb -t' command to list existing usb devices.
 957
 958 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 959
 960 The value 'spice' can be used to add a usb redirection devices for spice.
 961 EODESCR
 962     },
 963     usb3 => {
 964         optional => 1,
 965         type => 'boolean',
 966         description => "Specifies whether if given host option is a USB3 device or port.",
 967         default => 0,
 968     },
 969 };
 970
 971 my $usbdesc = {
 972     optional => 1,
 973     type => 'string', format => $usb_fmt,
 974     description => "Configure an USB device (n is 0 to 4).",
 975 };
 976 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 977
 978 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 979 my $hostpci_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string',
 983         pattern => qr/$PCIRE(;$PCIRE)*/,
 984         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 985         description => <<EODESCR,
 986 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 987 of PCI virtual functions of the host. HOSTPCIID syntax is:
 988
 989 'bus:dev.func' (hexadecimal numbers)
 990
 991 You can us the 'lspci' command to list existing PCI devices.
 992 EODESCR
 993     },
 994     rombar => {
 995         type => 'boolean',
 996         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 997         optional => 1,
 998         default => 1,
 999     },
1000     romfile => {
1001         type => 'string',
1002         pattern => '[^,;]+',
1003         format_description => 'string',
1004         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1005         optional => 1,
1006     },
1007     pcie => {
1008         type => 'boolean',
1009         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1010         optional => 1,
1011         default => 0,
1012     },
1013     'x-vga' => {
1014         type => 'boolean',
1015         description =>  "Enable vfio-vga device support.",
1016         optional => 1,
1017         default => 0,
1018     },
1019     'mdev' => {
1020         type => 'string',
1021         format_description => 'string',
1022         pattern => '[^/\.:]+',
1023         optional => 1,
1024         description => <<EODESCR
1025 The type of mediated device to use.
1026 An instance of this type will be created on startup of the VM and
1027 will be cleaned up when the VM stops.
1028 EODESCR
1029     }
1030 };
1031 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1032
1033 my $hostpcidesc = {
1034         optional => 1,
1035         type => 'string', format => 'pve-qm-hostpci',
1036         description => "Map host PCI devices into guest.",
1037         verbose_description =>  <<EODESCR,
1038 Map host PCI devices into guest.
1039
1040 NOTE: This option allows direct access to host hardware. So it is no longer
1041 possible to migrate such machines - use with special care.
1042
1043 CAUTION: Experimental! User reported problems with this option.
1044 EODESCR
1045 };
1046 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1047
1048 my $serialdesc = {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '(/dev/.+|socket)',
1052         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1053         verbose_description =>  <<EODESCR,
1054 Create a serial device inside the VM (n is 0 to 3), and pass through a
1055 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1056 host side (use 'qm terminal' to open a terminal connection).
1057
1058 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1059
1060 CAUTION: Experimental! User reported problems with this option.
1061 EODESCR
1062 };
1063
1064 my $paralleldesc= {
1065         optional => 1,
1066         type => 'string',
1067         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1068         description =>  "Map host parallel devices (n is 0 to 2).",
1069         verbose_description =>  <<EODESCR,
1070 Map host parallel devices (n is 0 to 2).
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1073
1074 CAUTION: Experimental! User reported problems with this option.
1075 EODESCR
1076 };
1077
1078 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1079     $confdesc->{"parallel$i"} = $paralleldesc;
1080 }
1081
1082 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1083     $confdesc->{"serial$i"} = $serialdesc;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1087     $confdesc->{"hostpci$i"} = $hostpcidesc;
1088 }
1089
1090 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1091     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1092 }
1093
1094 for (my $i = 0; $i < $PVE::QemuServer::Drive::MAX_UNUSED_DISKS; $i++) {
1095     $confdesc->{"unused$i"} = $PVE::QemuServer::Drive::unuseddesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1099     $confdesc->{"usb$i"} = $usbdesc;
1100 }
1101
1102 my $kvm_api_version = 0;
1103
1104 sub kvm_version {
1105     return $kvm_api_version if $kvm_api_version;
1106
1107     open my $fh, '<', '/dev/kvm'
1108         or return undef;
1109
1110     # 0xae00 => KVM_GET_API_VERSION
1111     $kvm_api_version = ioctl($fh, 0xae00, 0);
1112
1113     return $kvm_api_version;
1114 }
1115
1116 my $kvm_user_version = {};
1117 my $kvm_mtime = {};
1118
1119 sub kvm_user_version {
1120     my ($binary) = @_;
1121
1122     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1123     my $st = stat($binary);
1124
1125     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1126     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1127         $cachedmtime == $st->mtime;
1128
1129     $kvm_user_version->{$binary} = 'unknown';
1130     $kvm_mtime->{$binary} = $st->mtime;
1131
1132     my $code = sub {
1133         my $line = shift;
1134         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1135             $kvm_user_version->{$binary} = $2;
1136         }
1137     };
1138
1139     eval { run_command([$binary, '--version'], outfunc => $code); };
1140     warn $@ if $@;
1141
1142     return $kvm_user_version->{$binary};
1143
1144 }
1145
1146 sub kernel_has_vhost_net {
1147     return -c '/dev/vhost-net';
1148 }
1149
1150 sub option_exists {
1151     my $key = shift;
1152     return defined($confdesc->{$key});
1153 }
1154
1155 my $cdrom_path;
1156 sub get_cdrom_path {
1157
1158     return  $cdrom_path if $cdrom_path;
1159
1160     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1161     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1162     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1163 }
1164
1165 sub get_iso_path {
1166     my ($storecfg, $vmid, $cdrom) = @_;
1167
1168     if ($cdrom eq 'cdrom') {
1169         return get_cdrom_path();
1170     } elsif ($cdrom eq 'none') {
1171         return '';
1172     } elsif ($cdrom =~ m|^/|) {
1173         return $cdrom;
1174     } else {
1175         return PVE::Storage::path($storecfg, $cdrom);
1176     }
1177 }
1178
1179 # try to convert old style file names to volume IDs
1180 sub filename_to_volume_id {
1181     my ($vmid, $file, $media) = @_;
1182
1183      if (!($file eq 'none' || $file eq 'cdrom' ||
1184           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1185
1186         return undef if $file =~ m|/|;
1187
1188         if ($media && $media eq 'cdrom') {
1189             $file = "local:iso/$file";
1190         } else {
1191             $file = "local:$vmid/$file";
1192         }
1193     }
1194
1195     return $file;
1196 }
1197
1198 sub verify_media_type {
1199     my ($opt, $vtype, $media) = @_;
1200
1201     return if !$media;
1202
1203     my $etype;
1204     if ($media eq 'disk') {
1205         $etype = 'images';
1206     } elsif ($media eq 'cdrom') {
1207         $etype = 'iso';
1208     } else {
1209         die "internal error";
1210     }
1211
1212     return if ($vtype eq $etype);
1213
1214     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1215 }
1216
1217 sub cleanup_drive_path {
1218     my ($opt, $storecfg, $drive) = @_;
1219
1220     # try to convert filesystem paths to volume IDs
1221
1222     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1223         ($drive->{file} !~ m|^/dev/.+|) &&
1224         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1225         ($drive->{file} !~ m/^\d+$/)) {
1226         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1227         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1228         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1229         verify_media_type($opt, $vtype, $drive->{media});
1230         $drive->{file} = $volid;
1231     }
1232
1233     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1234 }
1235
1236 sub parse_hotplug_features {
1237     my ($data) = @_;
1238
1239     my $res = {};
1240
1241     return $res if $data eq '0';
1242
1243     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1244
1245     foreach my $feature (PVE::Tools::split_list($data)) {
1246         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1247             $res->{$1} = 1;
1248         } else {
1249             die "invalid hotplug feature '$feature'\n";
1250         }
1251     }
1252     return $res;
1253 }
1254
1255 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1256 sub pve_verify_hotplug_features {
1257     my ($value, $noerr) = @_;
1258
1259     return $value if parse_hotplug_features($value);
1260
1261     return undef if $noerr;
1262
1263     die "unable to parse hotplug option\n";
1264 }
1265
1266 sub scsi_inquiry {
1267     my($fh, $noerr) = @_;
1268
1269     my $SG_IO = 0x2285;
1270     my $SG_GET_VERSION_NUM = 0x2282;
1271
1272     my $versionbuf = "\x00" x 8;
1273     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1274     if (!$ret) {
1275         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1276         return undef;
1277     }
1278     my $version = unpack("I", $versionbuf);
1279     if ($version < 30000) {
1280         die "scsi generic interface too old\n"  if !$noerr;
1281         return undef;
1282     }
1283
1284     my $buf = "\x00" x 36;
1285     my $sensebuf = "\x00" x 8;
1286     my $cmd = pack("C x3 C x1", 0x12, 36);
1287
1288     # see /usr/include/scsi/sg.h
1289     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1290
1291     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1292                       length($sensebuf), 0, length($buf), $buf,
1293                       $cmd, $sensebuf, 6000);
1294
1295     $ret = ioctl($fh, $SG_IO, $packet);
1296     if (!$ret) {
1297         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1298         return undef;
1299     }
1300
1301     my @res = unpack($sg_io_hdr_t, $packet);
1302     if ($res[17] || $res[18]) {
1303         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1304         return undef;
1305     }
1306
1307     my $res = {};
1308     (my $byte0, my $byte1, $res->{vendor},
1309      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1310
1311     $res->{removable} = $byte1 & 128 ? 1 : 0;
1312     $res->{type} = $byte0 & 31;
1313
1314     return $res;
1315 }
1316
1317 sub path_is_scsi {
1318     my ($path) = @_;
1319
1320     my $fh = IO::File->new("+<$path") || return undef;
1321     my $res = scsi_inquiry($fh, 1);
1322     close($fh);
1323
1324     return $res;
1325 }
1326
1327 sub print_tabletdevice_full {
1328     my ($conf, $arch) = @_;
1329
1330     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1331
1332     # we use uhci for old VMs because tablet driver was buggy in older qemu
1333     my $usbbus;
1334     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1335         $usbbus = 'ehci';
1336     } else {
1337         $usbbus = 'uhci';
1338     }
1339
1340     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1341 }
1342
1343 sub print_keyboarddevice_full {
1344     my ($conf, $arch, $machine) = @_;
1345
1346     return undef if $arch ne 'aarch64';
1347
1348     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1349 }
1350
1351 sub print_drivedevice_full {
1352     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1353
1354     my $device = '';
1355     my $maxdev = 0;
1356
1357     if ($drive->{interface} eq 'virtio') {
1358         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1359         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1360         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1361     } elsif ($drive->{interface} eq 'scsi') {
1362
1363         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1364         my $unit = $drive->{index} % $maxdev;
1365         my $devicetype = 'hd';
1366         my $path = '';
1367         if (drive_is_cdrom($drive)) {
1368             $devicetype = 'cd';
1369         } else {
1370             if ($drive->{file} =~ m|^/|) {
1371                 $path = $drive->{file};
1372                 if (my $info = path_is_scsi($path)) {
1373                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1374                         $devicetype = 'block';
1375                     } elsif ($info->{type} == 1) { # tape
1376                         $devicetype = 'generic';
1377                     }
1378                 }
1379             } else {
1380                  $path = PVE::Storage::path($storecfg, $drive->{file});
1381             }
1382
1383             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1384             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1385             if ($path =~ m/^iscsi\:\/\// &&
1386                !min_version($version, 4, 1)) {
1387                 $devicetype = 'generic';
1388             }
1389         }
1390
1391         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1392             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1393         } else {
1394             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1395         }
1396
1397         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1398             $device .= ",rotation_rate=1";
1399         }
1400         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1401
1402     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1403         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1404         my $controller = int($drive->{index} / $maxdev);
1405         my $unit = $drive->{index} % $maxdev;
1406         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1407
1408         $device = "ide-$devicetype";
1409         if ($drive->{interface} eq 'ide') {
1410             $device .= ",bus=ide.$controller,unit=$unit";
1411         } else {
1412             $device .= ",bus=ahci$controller.$unit";
1413         }
1414         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1415
1416         if ($devicetype eq 'hd') {
1417             if (my $model = $drive->{model}) {
1418                 $model = URI::Escape::uri_unescape($model);
1419                 $device .= ",model=$model";
1420             }
1421             if ($drive->{ssd}) {
1422                 $device .= ",rotation_rate=1";
1423             }
1424         }
1425         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1426     } elsif ($drive->{interface} eq 'usb') {
1427         die "implement me";
1428         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1429     } else {
1430         die "unsupported interface type";
1431     }
1432
1433     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1434
1435     if (my $serial = $drive->{serial}) {
1436         $serial = URI::Escape::uri_unescape($serial);
1437         $device .= ",serial=$serial";
1438     }
1439
1440
1441     return $device;
1442 }
1443
1444 sub get_initiator_name {
1445     my $initiator;
1446
1447     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1448     while (defined(my $line = <$fh>)) {
1449         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1450         $initiator = $1;
1451         last;
1452     }
1453     $fh->close();
1454
1455     return $initiator;
1456 }
1457
1458 sub print_drive_commandline_full {
1459     my ($storecfg, $vmid, $drive) = @_;
1460
1461     my $path;
1462     my $volid = $drive->{file};
1463     my $format;
1464
1465     if (drive_is_cdrom($drive)) {
1466         $path = get_iso_path($storecfg, $vmid, $volid);
1467     } else {
1468         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1469         if ($storeid) {
1470             $path = PVE::Storage::path($storecfg, $volid);
1471             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1472             $format = qemu_img_format($scfg, $volname);
1473         } else {
1474             $path = $volid;
1475             $format = "raw";
1476         }
1477    }
1478
1479     my $opts = '';
1480     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1481     foreach my $o (@qemu_drive_options) {
1482         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1483     }
1484
1485     # snapshot only accepts on|off
1486     if (defined($drive->{snapshot})) {
1487         my $v = $drive->{snapshot} ? 'on' : 'off';
1488         $opts .= ",snapshot=$v";
1489     }
1490
1491     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1492         my ($dir, $qmpname) = @$type;
1493         if (my $v = $drive->{"mbps$dir"}) {
1494             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1495         }
1496         if (my $v = $drive->{"mbps${dir}_max"}) {
1497             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1498         }
1499         if (my $v = $drive->{"bps${dir}_max_length"}) {
1500             $opts .= ",throttling.bps$qmpname-max-length=$v";
1501         }
1502         if (my $v = $drive->{"iops${dir}"}) {
1503             $opts .= ",throttling.iops$qmpname=$v";
1504         }
1505         if (my $v = $drive->{"iops${dir}_max"}) {
1506             $opts .= ",throttling.iops$qmpname-max=$v";
1507         }
1508         if (my $v = $drive->{"iops${dir}_max_length"}) {
1509             $opts .= ",throttling.iops$qmpname-max-length=$v";
1510         }
1511     }
1512
1513     $opts .= ",format=$format" if $format && !$drive->{format};
1514
1515     my $cache_direct = 0;
1516
1517     if (my $cache = $drive->{cache}) {
1518         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1519     } elsif (!drive_is_cdrom($drive)) {
1520         $opts .= ",cache=none";
1521         $cache_direct = 1;
1522     }
1523
1524     # aio native works only with O_DIRECT
1525     if (!$drive->{aio}) {
1526         if($cache_direct) {
1527             $opts .= ",aio=native";
1528         } else {
1529             $opts .= ",aio=threads";
1530         }
1531     }
1532
1533     if (!drive_is_cdrom($drive)) {
1534         my $detectzeroes;
1535         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1536             $detectzeroes = 'off';
1537         } elsif ($drive->{discard}) {
1538             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1539         } else {
1540             # This used to be our default with discard not being specified:
1541             $detectzeroes = 'on';
1542         }
1543         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1544     }
1545
1546     my $pathinfo = $path ? "file=$path," : '';
1547
1548     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1549 }
1550
1551 sub print_netdevice_full {
1552     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1553
1554     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1555
1556     my $device = $net->{model};
1557     if ($net->{model} eq 'virtio') {
1558          $device = 'virtio-net-pci';
1559      };
1560
1561     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1562     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1563     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1564         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1565         my $vectors = $net->{queues} * 2 + 2;
1566         $tmpstr .= ",vectors=$vectors,mq=on";
1567     }
1568     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1569
1570     if ($use_old_bios_files) {
1571         my $romfile;
1572         if ($device eq 'virtio-net-pci') {
1573             $romfile = 'pxe-virtio.rom';
1574         } elsif ($device eq 'e1000') {
1575             $romfile = 'pxe-e1000.rom';
1576         } elsif ($device eq 'ne2k') {
1577             $romfile = 'pxe-ne2k_pci.rom';
1578         } elsif ($device eq 'pcnet') {
1579             $romfile = 'pxe-pcnet.rom';
1580         } elsif ($device eq 'rtl8139') {
1581             $romfile = 'pxe-rtl8139.rom';
1582         }
1583         $tmpstr .= ",romfile=$romfile" if $romfile;
1584     }
1585
1586     return $tmpstr;
1587 }
1588
1589 sub print_netdev_full {
1590     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1591
1592     my $i = '';
1593     if ($netid =~ m/^net(\d+)$/) {
1594         $i = int($1);
1595     }
1596
1597     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1598
1599     my $ifname = "tap${vmid}i$i";
1600
1601     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1602     die "interface name '$ifname' is too long (max 15 character)\n"
1603         if length($ifname) >= 16;
1604
1605     my $vhostparam = '';
1606     if (is_native($arch)) {
1607         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1608     }
1609
1610     my $vmname = $conf->{name} || "vm$vmid";
1611
1612     my $netdev = "";
1613     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1614
1615     if ($net->{bridge}) {
1616         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1617     } else {
1618         $netdev = "type=user,id=$netid,hostname=$vmname";
1619     }
1620
1621     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1622
1623     return $netdev;
1624 }
1625
1626 my $vga_map = {
1627     'cirrus' => 'cirrus-vga',
1628     'std' => 'VGA',
1629     'vmware' => 'vmware-svga',
1630     'virtio' => 'virtio-vga',
1631 };
1632
1633 sub print_vga_device {
1634     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1635
1636     my $type = $vga_map->{$vga->{type}};
1637     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1638         $type = 'virtio-gpu';
1639     }
1640     my $vgamem_mb = $vga->{memory};
1641
1642     my $max_outputs = '';
1643     if ($qxlnum) {
1644         $type = $id ? 'qxl' : 'qxl-vga';
1645
1646         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1647             # set max outputs so linux can have up to 4 qxl displays with one device
1648             if (min_version($machine_version, 4, 1)) {
1649                 $max_outputs = ",max_outputs=4";
1650             }
1651         }
1652     }
1653
1654     die "no devicetype for $vga->{type}\n" if !$type;
1655
1656     my $memory = "";
1657     if ($vgamem_mb) {
1658         if ($vga->{type} eq 'virtio') {
1659             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1660             $memory = ",max_hostmem=$bytes";
1661         } elsif ($qxlnum) {
1662             # from https://www.spice-space.org/multiple-monitors.html
1663             $memory = ",vgamem_mb=$vga->{memory}";
1664             my $ram = $vgamem_mb * 4;
1665             my $vram = $vgamem_mb * 2;
1666             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1667         } else {
1668             $memory = ",vgamem_mb=$vga->{memory}";
1669         }
1670     } elsif ($qxlnum && $id) {
1671         $memory = ",ram_size=67108864,vram_size=33554432";
1672     }
1673
1674     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1675     my $vgaid = "vga" . ($id // '');
1676     my $pciaddr;
1677
1678     if ($q35 && $vgaid eq 'vga') {
1679         # the first display uses pcie.0 bus on q35 machines
1680         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1681     } else {
1682         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1683     }
1684
1685     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1686 }
1687
1688 sub parse_number_sets {
1689     my ($set) = @_;
1690     my $res = [];
1691     foreach my $part (split(/;/, $set)) {
1692         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1693             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1694             push @$res, [ $1, $2 ];
1695         } else {
1696             die "invalid range: $part\n";
1697         }
1698     }
1699     return $res;
1700 }
1701
1702 sub parse_numa {
1703     my ($data) = @_;
1704
1705     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1706     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1707     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1708     return $res;
1709 }
1710
1711 sub parse_hostpci {
1712     my ($value) = @_;
1713
1714     return undef if !$value;
1715
1716     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1717
1718     my @idlist = split(/;/, $res->{host});
1719     delete $res->{host};
1720     foreach my $id (@idlist) {
1721         my $devs = PVE::SysFSTools::lspci($id);
1722         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1723         push @{$res->{pciid}}, @$devs;
1724     }
1725     return $res;
1726 }
1727
1728 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1729 sub parse_net {
1730     my ($data) = @_;
1731
1732     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1733     if ($@) {
1734         warn $@;
1735         return undef;
1736     }
1737     if (!defined($res->{macaddr})) {
1738         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1739         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1740     }
1741     return $res;
1742 }
1743
1744 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1745 sub parse_ipconfig {
1746     my ($data) = @_;
1747
1748     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1749     if ($@) {
1750         warn $@;
1751         return undef;
1752     }
1753
1754     if ($res->{gw} && !$res->{ip}) {
1755         warn 'gateway specified without specifying an IP address';
1756         return undef;
1757     }
1758     if ($res->{gw6} && !$res->{ip6}) {
1759         warn 'IPv6 gateway specified without specifying an IPv6 address';
1760         return undef;
1761     }
1762     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1763         warn 'gateway specified together with DHCP';
1764         return undef;
1765     }
1766     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1767         # gw6 + auto/dhcp
1768         warn "IPv6 gateway specified together with $res->{ip6} address";
1769         return undef;
1770     }
1771
1772     if (!$res->{ip} && !$res->{ip6}) {
1773         return { ip => 'dhcp', ip6 => 'dhcp' };
1774     }
1775
1776     return $res;
1777 }
1778
1779 sub print_net {
1780     my $net = shift;
1781
1782     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1783 }
1784
1785 sub add_random_macs {
1786     my ($settings) = @_;
1787
1788     foreach my $opt (keys %$settings) {
1789         next if $opt !~ m/^net(\d+)$/;
1790         my $net = parse_net($settings->{$opt});
1791         next if !$net;
1792         $settings->{$opt} = print_net($net);
1793     }
1794 }
1795
1796 sub vm_is_volid_owner {
1797     my ($storecfg, $vmid, $volid) = @_;
1798
1799     if ($volid !~  m|^/|) {
1800         my ($path, $owner);
1801         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1802         if ($owner && ($owner == $vmid)) {
1803             return 1;
1804         }
1805     }
1806
1807     return undef;
1808 }
1809
1810 sub vmconfig_register_unused_drive {
1811     my ($storecfg, $vmid, $conf, $drive) = @_;
1812
1813     if (drive_is_cloudinit($drive)) {
1814         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1815         warn $@ if $@;
1816     } elsif (!drive_is_cdrom($drive)) {
1817         my $volid = $drive->{file};
1818         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1819             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1820         }
1821     }
1822 }
1823
1824 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1825 my $smbios1_fmt = {
1826     uuid => {
1827         type => 'string',
1828         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1829         format_description => 'UUID',
1830         description => "Set SMBIOS1 UUID.",
1831         optional => 1,
1832     },
1833     version => {
1834         type => 'string',
1835         pattern => '[A-Za-z0-9+\/]+={0,2}',
1836         format_description => 'Base64 encoded string',
1837         description => "Set SMBIOS1 version.",
1838         optional => 1,
1839     },
1840     serial => {
1841         type => 'string',
1842         pattern => '[A-Za-z0-9+\/]+={0,2}',
1843         format_description => 'Base64 encoded string',
1844         description => "Set SMBIOS1 serial number.",
1845         optional => 1,
1846     },
1847     manufacturer => {
1848         type => 'string',
1849         pattern => '[A-Za-z0-9+\/]+={0,2}',
1850         format_description => 'Base64 encoded string',
1851         description => "Set SMBIOS1 manufacturer.",
1852         optional => 1,
1853     },
1854     product => {
1855         type => 'string',
1856         pattern => '[A-Za-z0-9+\/]+={0,2}',
1857         format_description => 'Base64 encoded string',
1858         description => "Set SMBIOS1 product ID.",
1859         optional => 1,
1860     },
1861     sku => {
1862         type => 'string',
1863         pattern => '[A-Za-z0-9+\/]+={0,2}',
1864         format_description => 'Base64 encoded string',
1865         description => "Set SMBIOS1 SKU string.",
1866         optional => 1,
1867     },
1868     family => {
1869         type => 'string',
1870         pattern => '[A-Za-z0-9+\/]+={0,2}',
1871         format_description => 'Base64 encoded string',
1872         description => "Set SMBIOS1 family string.",
1873         optional => 1,
1874     },
1875     base64 => {
1876         type => 'boolean',
1877         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1878         optional => 1,
1879     },
1880 };
1881
1882 sub parse_smbios1 {
1883     my ($data) = @_;
1884
1885     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1886     warn $@ if $@;
1887     return $res;
1888 }
1889
1890 sub print_smbios1 {
1891     my ($smbios1) = @_;
1892     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1893 }
1894
1895 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1896
1897 sub parse_watchdog {
1898     my ($value) = @_;
1899
1900     return undef if !$value;
1901
1902     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1903     warn $@ if $@;
1904     return $res;
1905 }
1906
1907 sub parse_guest_agent {
1908     my ($value) = @_;
1909
1910     return {} if !defined($value->{agent});
1911
1912     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1913     warn $@ if $@;
1914
1915     # if the agent is disabled ignore the other potentially set properties
1916     return {} if !$res->{enabled};
1917     return $res;
1918 }
1919
1920 sub parse_vga {
1921     my ($value) = @_;
1922
1923     return {} if !$value;
1924     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1925     warn $@ if $@;
1926     return $res;
1927 }
1928
1929 sub parse_rng {
1930     my ($value) = @_;
1931
1932     return undef if !$value;
1933
1934     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1935     warn $@ if $@;
1936     return $res;
1937 }
1938
1939 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1940 sub verify_usb_device {
1941     my ($value, $noerr) = @_;
1942
1943     return $value if parse_usb_device($value);
1944
1945     return undef if $noerr;
1946
1947     die "unable to parse usb device\n";
1948 }
1949
1950 # add JSON properties for create and set function
1951 sub json_config_properties {
1952     my $prop = shift;
1953
1954     foreach my $opt (keys %$confdesc) {
1955         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1956         $prop->{$opt} = $confdesc->{$opt};
1957     }
1958
1959     return $prop;
1960 }
1961
1962 # return copy of $confdesc_cloudinit to generate documentation
1963 sub cloudinit_config_properties {
1964
1965     return dclone($confdesc_cloudinit);
1966 }
1967
1968 sub check_type {
1969     my ($key, $value) = @_;
1970
1971     die "unknown setting '$key'\n" if !$confdesc->{$key};
1972
1973     my $type = $confdesc->{$key}->{type};
1974
1975     if (!defined($value)) {
1976         die "got undefined value\n";
1977     }
1978
1979     if ($value =~ m/[\n\r]/) {
1980         die "property contains a line feed\n";
1981     }
1982
1983     if ($type eq 'boolean') {
1984         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1985         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1986         die "type check ('boolean') failed - got '$value'\n";
1987     } elsif ($type eq 'integer') {
1988         return int($1) if $value =~ m/^(\d+)$/;
1989         die "type check ('integer') failed - got '$value'\n";
1990     } elsif ($type eq 'number') {
1991         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1992         die "type check ('number') failed - got '$value'\n";
1993     } elsif ($type eq 'string') {
1994         if (my $fmt = $confdesc->{$key}->{format}) {
1995             PVE::JSONSchema::check_format($fmt, $value);
1996             return $value;
1997         }
1998         $value =~ s/^\"(.*)\"$/$1/;
1999         return $value;
2000     } else {
2001         die "internal error"
2002     }
2003 }
2004
2005 sub destroy_vm {
2006     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2007
2008     my $conf = PVE::QemuConfig->load_config($vmid);
2009
2010     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2011
2012     if ($conf->{template}) {
2013         # check if any base image is still used by a linked clone
2014         foreach_drive($conf, sub {
2015                 my ($ds, $drive) = @_;
2016                 return if drive_is_cdrom($drive);
2017
2018                 my $volid = $drive->{file};
2019                 return if !$volid || $volid =~ m|^/|;
2020
2021                 die "base volume '$volid' is still in use by linked cloned\n"
2022                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2023
2024         });
2025     }
2026
2027     # only remove disks owned by this VM
2028     foreach_drive($conf, sub {
2029         my ($ds, $drive) = @_;
2030         return if drive_is_cdrom($drive, 1);
2031
2032         my $volid = $drive->{file};
2033         return if !$volid || $volid =~ m|^/|;
2034
2035         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2036         return if !$path || !$owner || ($owner != $vmid);
2037
2038         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2039         warn "Could not remove disk '$volid', check manually: $@" if $@;
2040     });
2041
2042     # also remove unused disk
2043     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2044     PVE::Storage::foreach_volid($vmdisks, sub {
2045         my ($volid, $sid, $volname, $d) = @_;
2046         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2047         warn $@ if $@;
2048     });
2049
2050     if (defined $replacement_conf) {
2051         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2052     } else {
2053         PVE::QemuConfig->destroy_config($vmid);
2054     }
2055 }
2056
2057 sub parse_vm_config {
2058     my ($filename, $raw) = @_;
2059
2060     return undef if !defined($raw);
2061
2062     my $res = {
2063         digest => Digest::SHA::sha1_hex($raw),
2064         snapshots => {},
2065         pending => {},
2066     };
2067
2068     $filename =~ m|/qemu-server/(\d+)\.conf$|
2069         || die "got strange filename '$filename'";
2070
2071     my $vmid = $1;
2072
2073     my $conf = $res;
2074     my $descr;
2075     my $section = '';
2076
2077     my @lines = split(/\n/, $raw);
2078     foreach my $line (@lines) {
2079         next if $line =~ m/^\s*$/;
2080
2081         if ($line =~ m/^\[PENDING\]\s*$/i) {
2082             $section = 'pending';
2083             if (defined($descr)) {
2084                 $descr =~ s/\s+$//;
2085                 $conf->{description} = $descr;
2086             }
2087             $descr = undef;
2088             $conf = $res->{$section} = {};
2089             next;
2090
2091         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2092             $section = $1;
2093             if (defined($descr)) {
2094                 $descr =~ s/\s+$//;
2095                 $conf->{description} = $descr;
2096             }
2097             $descr = undef;
2098             $conf = $res->{snapshots}->{$section} = {};
2099             next;
2100         }
2101
2102         if ($line =~ m/^\#(.*)\s*$/) {
2103             $descr = '' if !defined($descr);
2104             $descr .= PVE::Tools::decode_text($1) . "\n";
2105             next;
2106         }
2107
2108         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2109             $descr = '' if !defined($descr);
2110             $descr .= PVE::Tools::decode_text($2);
2111         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2112             $conf->{snapstate} = $1;
2113         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2114             my $key = $1;
2115             my $value = $2;
2116             $conf->{$key} = $value;
2117         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2118             my $value = $1;
2119             if ($section eq 'pending') {
2120                 $conf->{delete} = $value; # we parse this later
2121             } else {
2122                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2123             }
2124         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2125             my $key = $1;
2126             my $value = $2;
2127             eval { $value = check_type($key, $value); };
2128             if ($@) {
2129                 warn "vm $vmid - unable to parse value of '$key' - $@";
2130             } else {
2131                 $key = 'ide2' if $key eq 'cdrom';
2132                 my $fmt = $confdesc->{$key}->{format};
2133                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2134                     my $v = parse_drive($key, $value);
2135                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2136                         $v->{file} = $volid;
2137                         $value = print_drive($v);
2138                     } else {
2139                         warn "vm $vmid - unable to parse value of '$key'\n";
2140                         next;
2141                     }
2142                 }
2143
2144                 $conf->{$key} = $value;
2145             }
2146         }
2147     }
2148
2149     if (defined($descr)) {
2150         $descr =~ s/\s+$//;
2151         $conf->{description} = $descr;
2152     }
2153     delete $res->{snapstate}; # just to be sure
2154
2155     return $res;
2156 }
2157
2158 sub write_vm_config {
2159     my ($filename, $conf) = @_;
2160
2161     delete $conf->{snapstate}; # just to be sure
2162
2163     if ($conf->{cdrom}) {
2164         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2165         $conf->{ide2} = $conf->{cdrom};
2166         delete $conf->{cdrom};
2167     }
2168
2169     # we do not use 'smp' any longer
2170     if ($conf->{sockets}) {
2171         delete $conf->{smp};
2172     } elsif ($conf->{smp}) {
2173         $conf->{sockets} = $conf->{smp};
2174         delete $conf->{cores};
2175         delete $conf->{smp};
2176     }
2177
2178     my $used_volids = {};
2179
2180     my $cleanup_config = sub {
2181         my ($cref, $pending, $snapname) = @_;
2182
2183         foreach my $key (keys %$cref) {
2184             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2185                 $key eq 'snapstate' || $key eq 'pending';
2186             my $value = $cref->{$key};
2187             if ($key eq 'delete') {
2188                 die "propertry 'delete' is only allowed in [PENDING]\n"
2189                     if !$pending;
2190                 # fixme: check syntax?
2191                 next;
2192             }
2193             eval { $value = check_type($key, $value); };
2194             die "unable to parse value of '$key' - $@" if $@;
2195
2196             $cref->{$key} = $value;
2197
2198             if (!$snapname && is_valid_drivename($key)) {
2199                 my $drive = parse_drive($key, $value);
2200                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2201             }
2202         }
2203     };
2204
2205     &$cleanup_config($conf);
2206
2207     &$cleanup_config($conf->{pending}, 1);
2208
2209     foreach my $snapname (keys %{$conf->{snapshots}}) {
2210         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2211         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2212     }
2213
2214     # remove 'unusedX' settings if we re-add a volume
2215     foreach my $key (keys %$conf) {
2216         my $value = $conf->{$key};
2217         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2218             delete $conf->{$key};
2219         }
2220     }
2221
2222     my $generate_raw_config = sub {
2223         my ($conf, $pending) = @_;
2224
2225         my $raw = '';
2226
2227         # add description as comment to top of file
2228         if (defined(my $descr = $conf->{description})) {
2229             if ($descr) {
2230                 foreach my $cl (split(/\n/, $descr)) {
2231                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2232                 }
2233             } else {
2234                 $raw .= "#\n" if $pending;
2235             }
2236         }
2237
2238         foreach my $key (sort keys %$conf) {
2239             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2240             $raw .= "$key: $conf->{$key}\n";
2241         }
2242         return $raw;
2243     };
2244
2245     my $raw = &$generate_raw_config($conf);
2246
2247     if (scalar(keys %{$conf->{pending}})){
2248         $raw .= "\n[PENDING]\n";
2249         $raw .= &$generate_raw_config($conf->{pending}, 1);
2250     }
2251
2252     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2253         $raw .= "\n[$snapname]\n";
2254         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2255     }
2256
2257     return $raw;
2258 }
2259
2260 sub load_defaults {
2261
2262     my $res = {};
2263
2264     # we use static defaults from our JSON schema configuration
2265     foreach my $key (keys %$confdesc) {
2266         if (defined(my $default = $confdesc->{$key}->{default})) {
2267             $res->{$key} = $default;
2268         }
2269     }
2270
2271     return $res;
2272 }
2273
2274 sub config_list {
2275     my $vmlist = PVE::Cluster::get_vmlist();
2276     my $res = {};
2277     return $res if !$vmlist || !$vmlist->{ids};
2278     my $ids = $vmlist->{ids};
2279     my $nodename = nodename();
2280
2281     foreach my $vmid (keys %$ids) {
2282         my $d = $ids->{$vmid};
2283         next if !$d->{node} || $d->{node} ne $nodename;
2284         next if !$d->{type} || $d->{type} ne 'qemu';
2285         $res->{$vmid}->{exists} = 1;
2286     }
2287     return $res;
2288 }
2289
2290 # test if VM uses local resources (to prevent migration)
2291 sub check_local_resources {
2292     my ($conf, $noerr) = @_;
2293
2294     my @loc_res = ();
2295
2296     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2297     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2298
2299     push @loc_res, "ivshmem" if $conf->{ivshmem};
2300
2301     foreach my $k (keys %$conf) {
2302         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2303         # sockets are safe: they will recreated be on the target side post-migrate
2304         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2305         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2306     }
2307
2308     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2309
2310     return \@loc_res;
2311 }
2312
2313 # check if used storages are available on all nodes (use by migrate)
2314 sub check_storage_availability {
2315     my ($storecfg, $conf, $node) = @_;
2316
2317     foreach_drive($conf, sub {
2318         my ($ds, $drive) = @_;
2319
2320         my $volid = $drive->{file};
2321         return if !$volid;
2322
2323         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2324         return if !$sid;
2325
2326         # check if storage is available on both nodes
2327         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2328         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2329    });
2330 }
2331
2332 # list nodes where all VM images are available (used by has_feature API)
2333 sub shared_nodes {
2334     my ($conf, $storecfg) = @_;
2335
2336     my $nodelist = PVE::Cluster::get_nodelist();
2337     my $nodehash = { map { $_ => 1 } @$nodelist };
2338     my $nodename = nodename();
2339
2340     foreach_drive($conf, sub {
2341         my ($ds, $drive) = @_;
2342
2343         my $volid = $drive->{file};
2344         return if !$volid;
2345
2346         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2347         if ($storeid) {
2348             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2349             if ($scfg->{disable}) {
2350                 $nodehash = {};
2351             } elsif (my $avail = $scfg->{nodes}) {
2352                 foreach my $node (keys %$nodehash) {
2353                     delete $nodehash->{$node} if !$avail->{$node};
2354                 }
2355             } elsif (!$scfg->{shared}) {
2356                 foreach my $node (keys %$nodehash) {
2357                     delete $nodehash->{$node} if $node ne $nodename
2358                 }
2359             }
2360         }
2361     });
2362
2363     return $nodehash
2364 }
2365
2366 sub check_local_storage_availability {
2367     my ($conf, $storecfg) = @_;
2368
2369     my $nodelist = PVE::Cluster::get_nodelist();
2370     my $nodehash = { map { $_ => {} } @$nodelist };
2371
2372     foreach_drive($conf, sub {
2373         my ($ds, $drive) = @_;
2374
2375         my $volid = $drive->{file};
2376         return if !$volid;
2377
2378         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2379         if ($storeid) {
2380             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2381
2382             if ($scfg->{disable}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2385                 }
2386             } elsif (my $avail = $scfg->{nodes}) {
2387                 foreach my $node (keys %$nodehash) {
2388                     if (!$avail->{$node}) {
2389                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2390                     }
2391                 }
2392             }
2393         }
2394     });
2395
2396     foreach my $node (values %$nodehash) {
2397         if (my $unavail = $node->{unavailable_storages}) {
2398             $node->{unavailable_storages} = [ sort keys %$unavail ];
2399         }
2400     }
2401
2402     return $nodehash
2403 }
2404
2405 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2406 sub check_running {
2407     my ($vmid, $nocheck, $node) = @_;
2408
2409     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2410     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2411 }
2412
2413 sub vzlist {
2414
2415     my $vzlist = config_list();
2416
2417     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2418
2419     while (defined(my $de = $fd->read)) {
2420         next if $de !~ m/^(\d+)\.pid$/;
2421         my $vmid = $1;
2422         next if !defined($vzlist->{$vmid});
2423         if (my $pid = check_running($vmid)) {
2424             $vzlist->{$vmid}->{pid} = $pid;
2425         }
2426     }
2427
2428     return $vzlist;
2429 }
2430
2431 our $vmstatus_return_properties = {
2432     vmid => get_standard_option('pve-vmid'),
2433     status => {
2434         description => "Qemu process status.",
2435         type => 'string',
2436         enum => ['stopped', 'running'],
2437     },
2438     maxmem => {
2439         description => "Maximum memory in bytes.",
2440         type => 'integer',
2441         optional => 1,
2442         renderer => 'bytes',
2443     },
2444     maxdisk => {
2445         description => "Root disk size in bytes.",
2446         type => 'integer',
2447         optional => 1,
2448         renderer => 'bytes',
2449     },
2450     name => {
2451         description => "VM name.",
2452         type => 'string',
2453         optional => 1,
2454     },
2455     qmpstatus => {
2456         description => "Qemu QMP agent status.",
2457         type => 'string',
2458         optional => 1,
2459     },
2460     pid => {
2461         description => "PID of running qemu process.",
2462         type => 'integer',
2463         optional => 1,
2464     },
2465     uptime => {
2466         description => "Uptime.",
2467         type => 'integer',
2468         optional => 1,
2469         renderer => 'duration',
2470     },
2471     cpus => {
2472         description => "Maximum usable CPUs.",
2473         type => 'number',
2474         optional => 1,
2475     },
2476     lock => {
2477         description => "The current config lock, if any.",
2478         type => 'string',
2479         optional => 1,
2480     },
2481     tags => {
2482         description  => "The current configured tags, if any",
2483         type => 'string',
2484         optional => 1,
2485     },
2486 };
2487
2488 my $last_proc_pid_stat;
2489
2490 # get VM status information
2491 # This must be fast and should not block ($full == false)
2492 # We only query KVM using QMP if $full == true (this can be slow)
2493 sub vmstatus {
2494     my ($opt_vmid, $full) = @_;
2495
2496     my $res = {};
2497
2498     my $storecfg = PVE::Storage::config();
2499
2500     my $list = vzlist();
2501     my $defaults = load_defaults();
2502
2503     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2504
2505     my $cpucount = $cpuinfo->{cpus} || 1;
2506
2507     foreach my $vmid (keys %$list) {
2508         next if $opt_vmid && ($vmid ne $opt_vmid);
2509
2510         my $conf = PVE::QemuConfig->load_config($vmid);
2511
2512         my $d = { vmid => $vmid };
2513         $d->{pid} = $list->{$vmid}->{pid};
2514
2515         # fixme: better status?
2516         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2517
2518         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2519         if (defined($size)) {
2520             $d->{disk} = 0; # no info available
2521             $d->{maxdisk} = $size;
2522         } else {
2523             $d->{disk} = 0;
2524             $d->{maxdisk} = 0;
2525         }
2526
2527         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2528             * ($conf->{cores} || $defaults->{cores});
2529         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2530         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2531
2532         $d->{name} = $conf->{name} || "VM $vmid";
2533         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2534             : $defaults->{memory}*(1024*1024);
2535
2536         if ($conf->{balloon}) {
2537             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2538             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2539                 : $defaults->{shares};
2540         }
2541
2542         $d->{uptime} = 0;
2543         $d->{cpu} = 0;
2544         $d->{mem} = 0;
2545
2546         $d->{netout} = 0;
2547         $d->{netin} = 0;
2548
2549         $d->{diskread} = 0;
2550         $d->{diskwrite} = 0;
2551
2552         $d->{template} = PVE::QemuConfig->is_template($conf);
2553
2554         $d->{serial} = 1 if conf_has_serial($conf);
2555         $d->{lock} = $conf->{lock} if $conf->{lock};
2556         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2557
2558         $res->{$vmid} = $d;
2559     }
2560
2561     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2562     foreach my $dev (keys %$netdev) {
2563         next if $dev !~ m/^tap([1-9]\d*)i/;
2564         my $vmid = $1;
2565         my $d = $res->{$vmid};
2566         next if !$d;
2567
2568         $d->{netout} += $netdev->{$dev}->{receive};
2569         $d->{netin} += $netdev->{$dev}->{transmit};
2570
2571         if ($full) {
2572             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2573             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2574         }
2575
2576     }
2577
2578     my $ctime = gettimeofday;
2579
2580     foreach my $vmid (keys %$list) {
2581
2582         my $d = $res->{$vmid};
2583         my $pid = $d->{pid};
2584         next if !$pid;
2585
2586         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2587         next if !$pstat; # not running
2588
2589         my $used = $pstat->{utime} + $pstat->{stime};
2590
2591         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2592
2593         if ($pstat->{vsize}) {
2594             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2595         }
2596
2597         my $old = $last_proc_pid_stat->{$pid};
2598         if (!$old) {
2599             $last_proc_pid_stat->{$pid} = {
2600                 time => $ctime,
2601                 used => $used,
2602                 cpu => 0,
2603             };
2604             next;
2605         }
2606
2607         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2608
2609         if ($dtime > 1000) {
2610             my $dutime = $used -  $old->{used};
2611
2612             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2613             $last_proc_pid_stat->{$pid} = {
2614                 time => $ctime,
2615                 used => $used,
2616                 cpu => $d->{cpu},
2617             };
2618         } else {
2619             $d->{cpu} = $old->{cpu};
2620         }
2621     }
2622
2623     return $res if !$full;
2624
2625     my $qmpclient = PVE::QMPClient->new();
2626
2627     my $ballooncb = sub {
2628         my ($vmid, $resp) = @_;
2629
2630         my $info = $resp->{'return'};
2631         return if !$info->{max_mem};
2632
2633         my $d = $res->{$vmid};
2634
2635         # use memory assigned to VM
2636         $d->{maxmem} = $info->{max_mem};
2637         $d->{balloon} = $info->{actual};
2638
2639         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2640             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2641             $d->{freemem} = $info->{free_mem};
2642         }
2643
2644         $d->{ballooninfo} = $info;
2645     };
2646
2647     my $blockstatscb = sub {
2648         my ($vmid, $resp) = @_;
2649         my $data = $resp->{'return'} || [];
2650         my $totalrdbytes = 0;
2651         my $totalwrbytes = 0;
2652
2653         for my $blockstat (@$data) {
2654             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2655             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2656
2657             $blockstat->{device} =~ s/drive-//;
2658             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2659         }
2660         $res->{$vmid}->{diskread} = $totalrdbytes;
2661         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2662     };
2663
2664     my $statuscb = sub {
2665         my ($vmid, $resp) = @_;
2666
2667         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2668         # this fails if ballon driver is not loaded, so this must be
2669         # the last commnand (following command are aborted if this fails).
2670         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2671
2672         my $status = 'unknown';
2673         if (!defined($status = $resp->{'return'}->{status})) {
2674             warn "unable to get VM status\n";
2675             return;
2676         }
2677
2678         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2679     };
2680
2681     foreach my $vmid (keys %$list) {
2682         next if $opt_vmid && ($vmid ne $opt_vmid);
2683         next if !$res->{$vmid}->{pid}; # not running
2684         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2685     }
2686
2687     $qmpclient->queue_execute(undef, 2);
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2692     }
2693
2694     return $res;
2695 }
2696
2697 sub conf_has_serial {
2698     my ($conf) = @_;
2699
2700     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2701         if ($conf->{"serial$i"}) {
2702             return 1;
2703         }
2704     }
2705
2706     return 0;
2707 }
2708
2709 sub conf_has_audio {
2710     my ($conf, $id) = @_;
2711
2712     $id //= 0;
2713     my $audio = $conf->{"audio$id"};
2714     return undef if !defined($audio);
2715
2716     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2717     my $audiodriver = $audioproperties->{driver} // 'spice';
2718
2719     return {
2720         dev => $audioproperties->{device},
2721         dev_id => "audiodev$id",
2722         backend => $audiodriver,
2723         backend_id => "$audiodriver-backend${id}",
2724     };
2725 }
2726
2727 sub vga_conf_has_spice {
2728     my ($vga) = @_;
2729
2730     my $vgaconf = parse_vga($vga);
2731     my $vgatype = $vgaconf->{type};
2732     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2733
2734     return $1 || 1;
2735 }
2736
2737 sub is_native($) {
2738     my ($arch) = @_;
2739     return get_host_arch() eq $arch;
2740 }
2741
2742 sub get_vm_arch {
2743     my ($conf) = @_;
2744     return $conf->{arch} // get_host_arch();
2745 }
2746
2747 my $default_machines = {
2748     x86_64 => 'pc',
2749     aarch64 => 'virt',
2750 };
2751
2752 sub get_vm_machine {
2753     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2754
2755     my $machine = $forcemachine || $conf->{machine};
2756
2757     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2758         $arch //= 'x86_64';
2759         $machine ||= $default_machines->{$arch};
2760         if ($add_pve_version) {
2761             $kvmversion //= kvm_user_version();
2762             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2763             $machine .= "+pve$pvever";
2764         }
2765     }
2766
2767     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2768         # for version-pinned machines that do not include a pve-version (e.g.
2769         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2770         $machine .= '+pve0';
2771     }
2772
2773     return $machine;
2774 }
2775
2776 sub get_ovmf_files($) {
2777     my ($arch) = @_;
2778
2779     my $ovmf = $OVMF->{$arch}
2780         or die "no OVMF images known for architecture '$arch'\n";
2781
2782     return @$ovmf;
2783 }
2784
2785 my $Arch2Qemu = {
2786     aarch64 => '/usr/bin/qemu-system-aarch64',
2787     x86_64 => '/usr/bin/qemu-system-x86_64',
2788 };
2789 sub get_command_for_arch($) {
2790     my ($arch) = @_;
2791     return '/usr/bin/kvm' if is_native($arch);
2792
2793     my $cmd = $Arch2Qemu->{$arch}
2794         or die "don't know how to emulate architecture '$arch'\n";
2795     return $cmd;
2796 }
2797
2798 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2799 # to use in a QEMU command line (-cpu element), first array_intersect the result
2800 # of query_supported_ with query_understood_. This is necessary because:
2801 #
2802 # a) query_understood_ returns flags the host cannot use and
2803 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2804 #    flags, but CPU settings - with most of them being flags. Those settings
2805 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2806 #
2807 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2808 # expensive. If you need the value returned from this, you can get it much
2809 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2810 # $accel being 'kvm' or 'tcg'.
2811 #
2812 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2813 # changes, automatically populating pmxcfs.
2814 #
2815 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2816 # since kvm and tcg machines support different flags
2817 #
2818 sub query_supported_cpu_flags {
2819     my ($arch) = @_;
2820
2821     $arch //= get_host_arch();
2822     my $default_machine = $default_machines->{$arch};
2823
2824     my $flags = {};
2825
2826     # FIXME: Once this is merged, the code below should work for ARM as well:
2827     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2828     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2829         $arch eq "aarch64";
2830
2831     my $kvm_supported = defined(kvm_version());
2832     my $qemu_cmd = get_command_for_arch($arch);
2833     my $fakevmid = -1;
2834     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2835
2836     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2837     my $query_supported_run_qemu = sub {
2838         my ($kvm) = @_;
2839
2840         my $flags = {};
2841         my $cmd = [
2842             $qemu_cmd,
2843             '-machine', $default_machine,
2844             '-display', 'none',
2845             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2846             '-mon', 'chardev=qmp,mode=control',
2847             '-pidfile', $pidfile,
2848             '-S', '-daemonize'
2849         ];
2850
2851         if (!$kvm) {
2852             push @$cmd, '-accel', 'tcg';
2853         }
2854
2855         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2856         die "QEMU flag querying VM exited with code " . $rc if $rc;
2857
2858         eval {
2859             my $cmd_result = mon_cmd(
2860                 $fakevmid,
2861                 'query-cpu-model-expansion',
2862                 type => 'full',
2863                 model => { name => 'host' }
2864             );
2865
2866             my $props = $cmd_result->{model}->{props};
2867             foreach my $prop (keys %$props) {
2868                 next if $props->{$prop} ne '1';
2869                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2870                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2871                 # We only keep those with underscores, to match /proc/cpuinfo
2872                 $prop =~ s/\.|-/_/g;
2873                 $flags->{$prop} = 1;
2874             }
2875         };
2876         my $err = $@;
2877
2878         # force stop with 10 sec timeout and 'nocheck'
2879         # always stop, even if QMP failed
2880         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2881
2882         die $err if $err;
2883
2884         return [ sort keys %$flags ];
2885     };
2886
2887     # We need to query QEMU twice, since KVM and TCG have different supported flags
2888     PVE::QemuConfig->lock_config($fakevmid, sub {
2889         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2890         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2891
2892         if ($kvm_supported) {
2893             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2894             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2895         }
2896     });
2897
2898     return $flags;
2899 }
2900
2901 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2902 my $understood_cpu_flag_dir = "/usr/share/kvm";
2903 sub query_understood_cpu_flags {
2904     my $arch = get_host_arch();
2905     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2906
2907     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2908         if ! -e $filepath;
2909
2910     my $raw = file_get_contents($filepath);
2911     $raw =~ s/^\s+|\s+$//g;
2912     my @flags = split(/\s+/, $raw);
2913
2914     return \@flags;
2915 }
2916
2917 sub config_to_command {
2918     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2919
2920     my $cmd = [];
2921     my $globalFlags = [];
2922     my $machineFlags = [];
2923     my $rtcFlags = [];
2924     my $devices = [];
2925     my $pciaddr = '';
2926     my $bridges = {};
2927     my $ostype = $conf->{ostype};
2928     my $winversion = windows_version($ostype);
2929     my $kvm = $conf->{kvm};
2930     my $nodename = nodename();
2931
2932     my $arch = get_vm_arch($conf);
2933     my $kvm_binary = get_command_for_arch($arch);
2934     my $kvmver = kvm_user_version($kvm_binary);
2935
2936     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2937         $kvmver //= "undefined";
2938         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2939     }
2940
2941     my $add_pve_version = min_version($kvmver, 4, 1);
2942
2943     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2944     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2945     $kvm //= 1 if is_native($arch);
2946
2947     $machine_version =~ m/(\d+)\.(\d+)/;
2948     my ($machine_major, $machine_minor) = ($1, $2);
2949     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2950         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2951
2952     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2953         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2954         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2955           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2956     }
2957
2958     # if a specific +pve version is required for a feature, use $version_guard
2959     # instead of min_version to allow machines to be run with the minimum
2960     # required version
2961     my $required_pve_version = 0;
2962     my $version_guard = sub {
2963         my ($major, $minor, $pve) = @_;
2964         return 0 if !min_version($machine_version, $major, $minor, $pve);
2965         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2966         return 1;
2967     };
2968
2969     if ($kvm) {
2970         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2971             if !defined kvm_version();
2972     }
2973
2974     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2975     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2976     my $use_old_bios_files = undef;
2977     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2978
2979     my $cpuunits = defined($conf->{cpuunits}) ?
2980             $conf->{cpuunits} : $defaults->{cpuunits};
2981
2982     push @$cmd, $kvm_binary;
2983
2984     push @$cmd, '-id', $vmid;
2985
2986     my $vmname = $conf->{name} || "vm$vmid";
2987
2988     push @$cmd, '-name', $vmname;
2989
2990     my $use_virtio = 0;
2991
2992     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2993     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2994     push @$cmd, '-mon', "chardev=qmp,mode=control";
2995
2996     if (min_version($machine_version, 2, 12)) {
2997         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2998         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2999     }
3000
3001     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3002
3003     push @$cmd, '-daemonize';
3004
3005     if ($conf->{smbios1}) {
3006         my $smbios_conf = parse_smbios1($conf->{smbios1});
3007         if ($smbios_conf->{base64}) {
3008             # Do not pass base64 flag to qemu
3009             delete $smbios_conf->{base64};
3010             my $smbios_string = "";
3011             foreach my $key (keys %$smbios_conf) {
3012                 my $value;
3013                 if ($key eq "uuid") {
3014                     $value = $smbios_conf->{uuid}
3015                 } else {
3016                     $value = decode_base64($smbios_conf->{$key});
3017                 }
3018                 # qemu accepts any binary data, only commas need escaping by double comma
3019                 $value =~ s/,/,,/g;
3020                 $smbios_string .= "," . $key . "=" . $value if $value;
3021             }
3022             push @$cmd, '-smbios', "type=1" . $smbios_string;
3023         } else {
3024             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3025         }
3026     }
3027
3028     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3029     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3030         die "uefi base image not found\n" if ! -f $ovmf_code;
3031
3032         my $path;
3033         my $format;
3034         if (my $efidisk = $conf->{efidisk0}) {
3035             my $d = parse_drive('efidisk0', $efidisk);
3036             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3037             $format = $d->{format};
3038             if ($storeid) {
3039                 $path = PVE::Storage::path($storecfg, $d->{file});
3040                 if (!defined($format)) {
3041                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3042                     $format = qemu_img_format($scfg, $volname);
3043                 }
3044             } else {
3045                 $path = $d->{file};
3046                 die "efidisk format must be specified\n"
3047                     if !defined($format);
3048             }
3049         } else {
3050             warn "no efidisk configured! Using temporary efivars disk.\n";
3051             $path = "/tmp/$vmid-ovmf.fd";
3052             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3053             $format = 'raw';
3054         }
3055
3056         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3057         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3058     }
3059
3060     # load q35 config
3061     if ($q35) {
3062         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3063         if (min_version($machine_version, 4, 0)) {
3064             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3065         } else {
3066             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3067         }
3068     }
3069
3070     if ($conf->{vmgenid}) {
3071         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3072     }
3073
3074     # add usb controllers
3075     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3076     push @$devices, @usbcontrollers if @usbcontrollers;
3077     my $vga = parse_vga($conf->{vga});
3078
3079     my $qxlnum = vga_conf_has_spice($conf->{vga});
3080     $vga->{type} = 'qxl' if $qxlnum;
3081
3082     if (!$vga->{type}) {
3083         if ($arch eq 'aarch64') {
3084             $vga->{type} = 'virtio';
3085         } elsif (min_version($machine_version, 2, 9)) {
3086             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3087         } else {
3088             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3089         }
3090     }
3091
3092     # enable absolute mouse coordinates (needed by vnc)
3093     my $tablet;
3094     if (defined($conf->{tablet})) {
3095         $tablet = $conf->{tablet};
3096     } else {
3097         $tablet = $defaults->{tablet};
3098         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3099         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3100     }
3101
3102     if ($tablet) {
3103         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3104         my $kbd = print_keyboarddevice_full($conf, $arch);
3105         push @$devices, '-device', $kbd if defined($kbd);
3106     }
3107
3108     my $kvm_off = 0;
3109     my $gpu_passthrough;
3110
3111     # host pci devices
3112     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3113         my $id = "hostpci$i";
3114         my $d = parse_hostpci($conf->{$id});
3115         next if !$d;
3116
3117         if (my $pcie = $d->{pcie}) {
3118             die "q35 machine model is not enabled" if !$q35;
3119             # win7 wants to have the pcie devices directly on the pcie bus
3120             # instead of in the root port
3121             if ($winversion == 7) {
3122                 $pciaddr = print_pcie_addr("${id}bus0");
3123             } else {
3124                 # add more root ports if needed, 4 are present by default
3125                 # by pve-q35 cfgs, rest added here on demand.
3126                 if ($i > 3) {
3127                     push @$devices, '-device', print_pcie_root_port($i);
3128                 }
3129                 $pciaddr = print_pcie_addr($id);
3130             }
3131         } else {
3132             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3133         }
3134
3135         my $xvga = '';
3136         if ($d->{'x-vga'}) {
3137             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3138             $kvm_off = 1;
3139             $vga->{type} = 'none' if !defined($conf->{vga});
3140             $gpu_passthrough = 1;
3141         }
3142
3143         my $pcidevices = $d->{pciid};
3144         my $multifunction = 1 if @$pcidevices > 1;
3145
3146         my $sysfspath;
3147         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3148             my $pci_id = $pcidevices->[0]->{id};
3149             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3150             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3151         } elsif ($d->{mdev}) {
3152             warn "ignoring mediated device '$id' with multifunction device\n";
3153         }
3154
3155         my $j=0;
3156         foreach my $pcidevice (@$pcidevices) {
3157             my $devicestr = "vfio-pci";
3158
3159             if ($sysfspath) {
3160                 $devicestr .= ",sysfsdev=$sysfspath";
3161             } else {
3162                 $devicestr .= ",host=$pcidevice->{id}";
3163             }
3164
3165             my $mf_addr = $multifunction ? ".$j" : '';
3166             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3167
3168             if ($j == 0) {
3169                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3170                 $devicestr .= "$xvga";
3171                 $devicestr .= ",multifunction=on" if $multifunction;
3172                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3173             }
3174
3175             push @$devices, '-device', $devicestr;
3176             $j++;
3177         }
3178     }
3179
3180     # usb devices
3181     my $usb_dev_features = {};
3182     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3183
3184     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3185     push @$devices, @usbdevices if @usbdevices;
3186     # serial devices
3187     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3188         if (my $path = $conf->{"serial$i"}) {
3189             if ($path eq 'socket') {
3190                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3191                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3192                 # On aarch64, serial0 is the UART device. Qemu only allows
3193                 # connecting UART devices via the '-serial' command line, as
3194                 # the device has a fixed slot on the hardware...
3195                 if ($arch eq 'aarch64' && $i == 0) {
3196                     push @$devices, '-serial', "chardev:serial$i";
3197                 } else {
3198                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3199                 }
3200             } else {
3201                 die "no such serial device\n" if ! -c $path;
3202                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3203                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3204             }
3205         }
3206     }
3207
3208     # parallel devices
3209     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3210         if (my $path = $conf->{"parallel$i"}) {
3211             die "no such parallel device\n" if ! -c $path;
3212             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3213             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3214             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3215         }
3216     }
3217
3218     if (my $audio = conf_has_audio($conf)) {
3219
3220         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3221
3222         my $id = $audio->{dev_id};
3223         if ($audio->{dev} eq 'AC97') {
3224             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3225         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3226             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3227             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3228             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3229         } else {
3230             die "unkown audio device '$audio->{dev}', implement me!";
3231         }
3232
3233         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3234     }
3235
3236     my $sockets = 1;
3237     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3238     $sockets = $conf->{sockets} if  $conf->{sockets};
3239
3240     my $cores = $conf->{cores} || 1;
3241
3242     my $maxcpus = $sockets * $cores;
3243
3244     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3245
3246     my $allowed_vcpus = $cpuinfo->{cpus};
3247
3248     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3249         if ($allowed_vcpus < $maxcpus);
3250
3251     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3252
3253         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3254         for (my $i = 2; $i <= $vcpus; $i++)  {
3255             my $cpustr = print_cpu_device($conf,$i);
3256             push @$cmd, '-device', $cpustr;
3257         }
3258
3259     } else {
3260
3261         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3262     }
3263     push @$cmd, '-nodefaults';
3264
3265     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3266
3267     my $bootindex_hash = {};
3268     my $i = 1;
3269     foreach my $o (split(//, $bootorder)) {
3270         $bootindex_hash->{$o} = $i*100;
3271         $i++;
3272     }
3273
3274     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3275
3276     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3277
3278     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3279
3280     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3281         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3282         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3283         push @$cmd,  '-vnc', "unix:$socket,password";
3284     } else {
3285         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3286         push @$cmd, '-nographic';
3287     }
3288
3289     # time drift fix
3290     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3291
3292     my $useLocaltime = $conf->{localtime};
3293
3294     if ($winversion >= 5) { # windows
3295         $useLocaltime = 1 if !defined($conf->{localtime});
3296
3297         # use time drift fix when acpi is enabled
3298         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3299             $tdf = 1 if !defined($conf->{tdf});
3300         }
3301     }
3302
3303     if ($winversion >= 6) {
3304         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3305         push @$cmd, '-no-hpet';
3306     }
3307
3308     push @$rtcFlags, 'driftfix=slew' if $tdf;
3309
3310     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3311         push @$rtcFlags, "base=$conf->{startdate}";
3312     } elsif ($useLocaltime) {
3313         push @$rtcFlags, 'base=localtime';
3314     }
3315
3316     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3317
3318     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3319
3320     push @$cmd, '-S' if $conf->{freeze};
3321
3322     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3323
3324     my $guest_agent = parse_guest_agent($conf);
3325
3326     if ($guest_agent->{enabled}) {
3327         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3328         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3329
3330         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3331             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3332             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3333             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3334         } elsif ($guest_agent->{type} eq 'isa') {
3335             push @$devices, '-device', "isa-serial,chardev=qga0";
3336         }
3337     }
3338
3339     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3340     if ($rng && &$version_guard(4, 1, 2)) {
3341         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3342         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3343
3344         my $limiter_str = "";
3345         if ($max_bytes) {
3346             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3347         }
3348
3349         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3350         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3351             if ! -e $rng->{source};
3352
3353         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3354
3355         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3356         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3357     }
3358
3359     my $spice_port;
3360
3361     if ($qxlnum) {
3362         if ($qxlnum > 1) {
3363             if ($winversion){
3364                 for(my $i = 1; $i < $qxlnum; $i++){
3365                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3366                 }
3367             } else {
3368                 # assume other OS works like Linux
3369                 my ($ram, $vram) = ("134217728", "67108864");
3370                 if ($vga->{memory}) {
3371                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3372                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3373                 }
3374                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3375                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3376             }
3377         }
3378
3379         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3380
3381         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3382         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3383         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3384
3385         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3386         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3387         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3388
3389         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3390         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3391
3392         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3393         if ($spice_enhancement->{foldersharing}) {
3394             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3395             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3396         }
3397
3398         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3399         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3400         push @$devices, '-spice', "$spice_opts";
3401     }
3402
3403     # enable balloon by default, unless explicitly disabled
3404     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3405         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3406         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3407     }
3408
3409     if ($conf->{watchdog}) {
3410         my $wdopts = parse_watchdog($conf->{watchdog});
3411         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3412         my $watchdog = $wdopts->{model} || 'i6300esb';
3413         push @$devices, '-device', "$watchdog$pciaddr";
3414         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3415     }
3416
3417     my $vollist = [];
3418     my $scsicontroller = {};
3419     my $ahcicontroller = {};
3420     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3421
3422     # Add iscsi initiator name if available
3423     if (my $initiator = get_initiator_name()) {
3424         push @$devices, '-iscsi', "initiator-name=$initiator";
3425     }
3426
3427     foreach_drive($conf, sub {
3428         my ($ds, $drive) = @_;
3429
3430         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3431             push @$vollist, $drive->{file};
3432         }
3433
3434         # ignore efidisk here, already added in bios/fw handling code above
3435         return if $drive->{interface} eq 'efidisk';
3436
3437         $use_virtio = 1 if $ds =~ m/^virtio/;
3438
3439         if (drive_is_cdrom ($drive)) {
3440             if ($bootindex_hash->{d}) {
3441                 $drive->{bootindex} = $bootindex_hash->{d};
3442                 $bootindex_hash->{d} += 1;
3443             }
3444         } else {
3445             if ($bootindex_hash->{c}) {
3446                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3447                 $bootindex_hash->{c} += 1;
3448             }
3449         }
3450
3451         if($drive->{interface} eq 'virtio'){
3452            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3453         }
3454
3455         if ($drive->{interface} eq 'scsi') {
3456
3457             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3458
3459             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3460                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3461
3462             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3463             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3464
3465             my $iothread = '';
3466             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3467                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3468                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3469             } elsif ($drive->{iothread}) {
3470                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3471             }
3472
3473             my $queues = '';
3474             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3475                 $queues = ",num_queues=$drive->{queues}";
3476             }
3477
3478             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3479             $scsicontroller->{$controller}=1;
3480         }
3481
3482         if ($drive->{interface} eq 'sata') {
3483            my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3484            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3485            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3486            $ahcicontroller->{$controller}=1;
3487         }
3488
3489         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3490         push @$devices, '-drive',$drive_cmd;
3491         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3492     });
3493
3494     for (my $i = 0; $i < $MAX_NETS; $i++) {
3495          next if !$conf->{"net$i"};
3496          my $d = parse_net($conf->{"net$i"});
3497          next if !$d;
3498
3499          $use_virtio = 1 if $d->{model} eq 'virtio';
3500
3501          if ($bootindex_hash->{n}) {
3502             $d->{bootindex} = $bootindex_hash->{n};
3503             $bootindex_hash->{n} += 1;
3504          }
3505
3506          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3507          push @$devices, '-netdev', $netdevfull;
3508
3509          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3510          push @$devices, '-device', $netdevicefull;
3511     }
3512
3513     if ($conf->{ivshmem}) {
3514         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3515
3516         my $bus;
3517         if ($q35) {
3518             $bus = print_pcie_addr("ivshmem");
3519         } else {
3520             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3521         }
3522
3523         my $ivshmem_name = $ivshmem->{name} // $vmid;
3524         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3525
3526         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3527         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3528     }
3529
3530     # pci.4 is nested in pci.1
3531     $bridges->{1} = 1 if $bridges->{4};
3532
3533     if (!$q35) {
3534         # add pci bridges
3535         if (min_version($machine_version, 2, 3)) {
3536            $bridges->{1} = 1;
3537            $bridges->{2} = 1;
3538         }
3539
3540         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3541
3542     }
3543
3544     for my $k (sort {$b cmp $a} keys %$bridges) {
3545         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3546         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3547         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3548         if ($q35) {
3549             # add after -readconfig pve-q35.cfg
3550             splice @$devices, 2, 0, '-device', $devstr;
3551         } else {
3552             unshift @$devices, '-device', $devstr if $k > 0;
3553         }
3554     }
3555
3556     if (!$kvm) {
3557         push @$machineFlags, 'accel=tcg';
3558     }
3559
3560     my $machine_type_min = $machine_type;
3561     if ($add_pve_version) {
3562         $machine_type_min =~ s/\+pve\d+$//;
3563         $machine_type_min .= "+pve$required_pve_version";
3564     }
3565     push @$machineFlags, "type=${machine_type_min}";
3566
3567     push @$cmd, @$devices;
3568     push @$cmd, '-rtc', join(',', @$rtcFlags)
3569         if scalar(@$rtcFlags);
3570     push @$cmd, '-machine', join(',', @$machineFlags)
3571         if scalar(@$machineFlags);
3572     push @$cmd, '-global', join(',', @$globalFlags)
3573         if scalar(@$globalFlags);
3574
3575     if (my $vmstate = $conf->{vmstate}) {
3576         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3577         push @$vollist, $vmstate;
3578         push @$cmd, '-loadstate', $statepath;
3579         print "activating and using '$vmstate' as vmstate\n";
3580     }
3581
3582     # add custom args
3583     if ($conf->{args}) {
3584         my $aa = PVE::Tools::split_args($conf->{args});
3585         push @$cmd, @$aa;
3586     }
3587
3588     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3589 }
3590
3591 sub spice_port {
3592     my ($vmid) = @_;
3593
3594     my $res = mon_cmd($vmid, 'query-spice');
3595
3596     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3597 }
3598
3599 sub vm_devices_list {
3600     my ($vmid) = @_;
3601
3602     my $res = mon_cmd($vmid, 'query-pci');
3603     my $devices_to_check = [];
3604     my $devices = {};
3605     foreach my $pcibus (@$res) {
3606         push @$devices_to_check, @{$pcibus->{devices}},
3607     }
3608
3609     while (@$devices_to_check) {
3610         my $to_check = [];
3611         for my $d (@$devices_to_check) {
3612             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3613             next if !$d->{'pci_bridge'};
3614
3615             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3616             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3617         }
3618         $devices_to_check = $to_check;
3619     }
3620
3621     my $resblock = mon_cmd($vmid, 'query-block');
3622     foreach my $block (@$resblock) {
3623         if($block->{device} =~ m/^drive-(\S+)/){
3624                 $devices->{$1} = 1;
3625         }
3626     }
3627
3628     my $resmice = mon_cmd($vmid, 'query-mice');
3629     foreach my $mice (@$resmice) {
3630         if ($mice->{name} eq 'QEMU HID Tablet') {
3631             $devices->{tablet} = 1;
3632             last;
3633         }
3634     }
3635
3636     # for usb devices there is no query-usb
3637     # but we can iterate over the entries in
3638     # qom-list path=/machine/peripheral
3639     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3640     foreach my $per (@$resperipheral) {
3641         if ($per->{name} =~ m/^usb\d+$/) {
3642             $devices->{$per->{name}} = 1;
3643         }
3644     }
3645
3646     return $devices;
3647 }
3648
3649 sub vm_deviceplug {
3650     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3651
3652     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3653
3654     my $devices_list = vm_devices_list($vmid);
3655     return 1 if defined($devices_list->{$deviceid});
3656
3657     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3658
3659     if ($deviceid eq 'tablet') {
3660
3661         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3662
3663     } elsif ($deviceid eq 'keyboard') {
3664
3665         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3666
3667     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3668
3669         die "usb hotplug currently not reliable\n";
3670         # since we can't reliably hot unplug all added usb devices
3671         # and usb passthrough disables live migration
3672         # we disable usb hotplugging for now
3673         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3674
3675     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3676
3677         qemu_iothread_add($vmid, $deviceid, $device);
3678
3679         qemu_driveadd($storecfg, $vmid, $device);
3680         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3681
3682         qemu_deviceadd($vmid, $devicefull);
3683         eval { qemu_deviceaddverify($vmid, $deviceid); };
3684         if (my $err = $@) {
3685             eval { qemu_drivedel($vmid, $deviceid); };
3686             warn $@ if $@;
3687             die $err;
3688         }
3689
3690     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3691
3692
3693         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3694         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3695         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3696
3697         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3698
3699         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3700             qemu_iothread_add($vmid, $deviceid, $device);
3701             $devicefull .= ",iothread=iothread-$deviceid";
3702         }
3703
3704         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3705             $devicefull .= ",num_queues=$device->{queues}";
3706         }
3707
3708         qemu_deviceadd($vmid, $devicefull);
3709         qemu_deviceaddverify($vmid, $deviceid);
3710
3711     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3712
3713         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3714         qemu_driveadd($storecfg, $vmid, $device);
3715
3716         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3717         eval { qemu_deviceadd($vmid, $devicefull); };
3718         if (my $err = $@) {
3719             eval { qemu_drivedel($vmid, $deviceid); };
3720             warn $@ if $@;
3721             die $err;
3722         }
3723
3724     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3725
3726         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3727
3728         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3729         my $use_old_bios_files = undef;
3730         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3731
3732         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3733         qemu_deviceadd($vmid, $netdevicefull);
3734         eval {
3735             qemu_deviceaddverify($vmid, $deviceid);
3736             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3737         };
3738         if (my $err = $@) {
3739             eval { qemu_netdevdel($vmid, $deviceid); };
3740             warn $@ if $@;
3741             die $err;
3742         }
3743
3744     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3745
3746         my $bridgeid = $2;
3747         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3748         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3749
3750         qemu_deviceadd($vmid, $devicefull);
3751         qemu_deviceaddverify($vmid, $deviceid);
3752
3753     } else {
3754         die "can't hotplug device '$deviceid'\n";
3755     }
3756
3757     return 1;
3758 }
3759
3760 # fixme: this should raise exceptions on error!
3761 sub vm_deviceunplug {
3762     my ($vmid, $conf, $deviceid) = @_;
3763
3764     my $devices_list = vm_devices_list($vmid);
3765     return 1 if !defined($devices_list->{$deviceid});
3766
3767     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3768
3769     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3770
3771         qemu_devicedel($vmid, $deviceid);
3772
3773     } elsif ($deviceid =~ m/^usb\d+$/) {
3774
3775         die "usb hotplug currently not reliable\n";
3776         # when unplugging usb devices this way,
3777         # there may be remaining usb controllers/hubs
3778         # so we disable it for now
3779         qemu_devicedel($vmid, $deviceid);
3780         qemu_devicedelverify($vmid, $deviceid);
3781
3782     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3783
3784         qemu_devicedel($vmid, $deviceid);
3785         qemu_devicedelverify($vmid, $deviceid);
3786         qemu_drivedel($vmid, $deviceid);
3787         qemu_iothread_del($conf, $vmid, $deviceid);
3788
3789     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3790
3791         qemu_devicedel($vmid, $deviceid);
3792         qemu_devicedelverify($vmid, $deviceid);
3793         qemu_iothread_del($conf, $vmid, $deviceid);
3794
3795     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3796
3797         qemu_devicedel($vmid, $deviceid);
3798         qemu_drivedel($vmid, $deviceid);
3799         qemu_deletescsihw($conf, $vmid, $deviceid);
3800
3801     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3802
3803         qemu_devicedel($vmid, $deviceid);
3804         qemu_devicedelverify($vmid, $deviceid);
3805         qemu_netdevdel($vmid, $deviceid);
3806
3807     } else {
3808         die "can't unplug device '$deviceid'\n";
3809     }
3810
3811     return 1;
3812 }
3813
3814 sub qemu_deviceadd {
3815     my ($vmid, $devicefull) = @_;
3816
3817     $devicefull = "driver=".$devicefull;
3818     my %options =  split(/[=,]/, $devicefull);
3819
3820     mon_cmd($vmid, "device_add" , %options);
3821 }
3822
3823 sub qemu_devicedel {
3824     my ($vmid, $deviceid) = @_;
3825
3826     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3827 }
3828
3829 sub qemu_iothread_add {
3830     my($vmid, $deviceid, $device) = @_;
3831
3832     if ($device->{iothread}) {
3833         my $iothreads = vm_iothreads_list($vmid);
3834         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3835     }
3836 }
3837
3838 sub qemu_iothread_del {
3839     my($conf, $vmid, $deviceid) = @_;
3840
3841     my $confid = $deviceid;
3842     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3843         $confid = 'scsi' . $1;
3844     }
3845     my $device = parse_drive($confid, $conf->{$confid});
3846     if ($device->{iothread}) {
3847         my $iothreads = vm_iothreads_list($vmid);
3848         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3849     }
3850 }
3851
3852 sub qemu_objectadd {
3853     my($vmid, $objectid, $qomtype) = @_;
3854
3855     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3856
3857     return 1;
3858 }
3859
3860 sub qemu_objectdel {
3861     my($vmid, $objectid) = @_;
3862
3863     mon_cmd($vmid, "object-del", id => $objectid);
3864
3865     return 1;
3866 }
3867
3868 sub qemu_driveadd {
3869     my ($storecfg, $vmid, $device) = @_;
3870
3871     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3872     $drive =~ s/\\/\\\\/g;
3873     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3874
3875     # If the command succeeds qemu prints: "OK"
3876     return 1 if $ret =~ m/OK/s;
3877
3878     die "adding drive failed: $ret\n";
3879 }
3880
3881 sub qemu_drivedel {
3882     my($vmid, $deviceid) = @_;
3883
3884     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3885     $ret =~ s/^\s+//;
3886
3887     return 1 if $ret eq "";
3888
3889     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3890     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3891
3892     die "deleting drive $deviceid failed : $ret\n";
3893 }
3894
3895 sub qemu_deviceaddverify {
3896     my ($vmid, $deviceid) = @_;
3897
3898     for (my $i = 0; $i <= 5; $i++) {
3899          my $devices_list = vm_devices_list($vmid);
3900          return 1 if defined($devices_list->{$deviceid});
3901          sleep 1;
3902     }
3903
3904     die "error on hotplug device '$deviceid'\n";
3905 }
3906
3907
3908 sub qemu_devicedelverify {
3909     my ($vmid, $deviceid) = @_;
3910
3911     # need to verify that the device is correctly removed as device_del
3912     # is async and empty return is not reliable
3913
3914     for (my $i = 0; $i <= 5; $i++) {
3915          my $devices_list = vm_devices_list($vmid);
3916          return 1 if !defined($devices_list->{$deviceid});
3917          sleep 1;
3918     }
3919
3920     die "error on hot-unplugging device '$deviceid'\n";
3921 }
3922
3923 sub qemu_findorcreatescsihw {
3924     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3925
3926     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3927
3928     my $scsihwid="$controller_prefix$controller";
3929     my $devices_list = vm_devices_list($vmid);
3930
3931     if(!defined($devices_list->{$scsihwid})) {
3932         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3933     }
3934
3935     return 1;
3936 }
3937
3938 sub qemu_deletescsihw {
3939     my ($conf, $vmid, $opt) = @_;
3940
3941     my $device = parse_drive($opt, $conf->{$opt});
3942
3943     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3944         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3945         return 1;
3946     }
3947
3948     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3949
3950     my $devices_list = vm_devices_list($vmid);
3951     foreach my $opt (keys %{$devices_list}) {
3952         if (is_valid_drivename($opt)) {
3953             my $drive = parse_drive($opt, $conf->{$opt});
3954             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3955                 return 1;
3956             }
3957         }
3958     }
3959
3960     my $scsihwid="scsihw$controller";
3961
3962     vm_deviceunplug($vmid, $conf, $scsihwid);
3963
3964     return 1;
3965 }
3966
3967 sub qemu_add_pci_bridge {
3968     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3969
3970     my $bridges = {};
3971
3972     my $bridgeid;
3973
3974     print_pci_addr($device, $bridges, $arch, $machine_type);
3975
3976     while (my ($k, $v) = each %$bridges) {
3977         $bridgeid = $k;
3978     }
3979     return 1 if !defined($bridgeid) || $bridgeid < 1;
3980
3981     my $bridge = "pci.$bridgeid";
3982     my $devices_list = vm_devices_list($vmid);
3983
3984     if (!defined($devices_list->{$bridge})) {
3985         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3986     }
3987
3988     return 1;
3989 }
3990
3991 sub qemu_set_link_status {
3992     my ($vmid, $device, $up) = @_;
3993
3994     mon_cmd($vmid, "set_link", name => $device,
3995                up => $up ? JSON::true : JSON::false);
3996 }
3997
3998 sub qemu_netdevadd {
3999     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4000
4001     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4002     my %options =  split(/[=,]/, $netdev);
4003
4004     mon_cmd($vmid, "netdev_add",  %options);
4005     return 1;
4006 }
4007
4008 sub qemu_netdevdel {
4009     my ($vmid, $deviceid) = @_;
4010
4011     mon_cmd($vmid, "netdev_del", id => $deviceid);
4012 }
4013
4014 sub qemu_usb_hotplug {
4015     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4016
4017     return if !$device;
4018
4019     # remove the old one first
4020     vm_deviceunplug($vmid, $conf, $deviceid);
4021
4022     # check if xhci controller is necessary and available
4023     if ($device->{usb3}) {
4024
4025         my $devicelist = vm_devices_list($vmid);
4026
4027         if (!$devicelist->{xhci}) {
4028             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4029             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4030         }
4031     }
4032     my $d = parse_usb_device($device->{host});
4033     $d->{usb3} = $device->{usb3};
4034
4035     # add the new one
4036     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4037 }
4038
4039 sub qemu_cpu_hotplug {
4040     my ($vmid, $conf, $vcpus) = @_;
4041
4042     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4043
4044     my $sockets = 1;
4045     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4046     $sockets = $conf->{sockets} if  $conf->{sockets};
4047     my $cores = $conf->{cores} || 1;
4048     my $maxcpus = $sockets * $cores;
4049
4050     $vcpus = $maxcpus if !$vcpus;
4051
4052     die "you can't add more vcpus than maxcpus\n"
4053         if $vcpus > $maxcpus;
4054
4055     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4056
4057     if ($vcpus < $currentvcpus) {
4058
4059         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4060
4061             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4062                 qemu_devicedel($vmid, "cpu$i");
4063                 my $retry = 0;
4064                 my $currentrunningvcpus = undef;
4065                 while (1) {
4066                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4067                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4068                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4069                     $retry++;
4070                     sleep 1;
4071                 }
4072                 #update conf after each succesfull cpu unplug
4073                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4074                 PVE::QemuConfig->write_config($vmid, $conf);
4075             }
4076         } else {
4077             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4078         }
4079
4080         return;
4081     }
4082
4083     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4084     die "vcpus in running vm does not match its configuration\n"
4085         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4086
4087     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4088
4089         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4090             my $cpustr = print_cpu_device($conf, $i);
4091             qemu_deviceadd($vmid, $cpustr);
4092
4093             my $retry = 0;
4094             my $currentrunningvcpus = undef;
4095             while (1) {
4096                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4097                 last if scalar(@{$currentrunningvcpus}) == $i;
4098                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4099                 sleep 1;
4100                 $retry++;
4101             }
4102             #update conf after each succesfull cpu hotplug
4103             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4104             PVE::QemuConfig->write_config($vmid, $conf);
4105         }
4106     } else {
4107
4108         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4109             mon_cmd($vmid, "cpu-add", id => int($i));
4110         }
4111     }
4112 }
4113
4114 sub qemu_block_set_io_throttle {
4115     my ($vmid, $deviceid,
4116         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4117         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4118         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4119         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4120
4121     return if !check_running($vmid) ;
4122
4123     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4124         bps => int($bps),
4125         bps_rd => int($bps_rd),
4126         bps_wr => int($bps_wr),
4127         iops => int($iops),
4128         iops_rd => int($iops_rd),
4129         iops_wr => int($iops_wr),
4130         bps_max => int($bps_max),
4131         bps_rd_max => int($bps_rd_max),
4132         bps_wr_max => int($bps_wr_max),
4133         iops_max => int($iops_max),
4134         iops_rd_max => int($iops_rd_max),
4135         iops_wr_max => int($iops_wr_max),
4136         bps_max_length => int($bps_max_length),
4137         bps_rd_max_length => int($bps_rd_max_length),
4138         bps_wr_max_length => int($bps_wr_max_length),
4139         iops_max_length => int($iops_max_length),
4140         iops_rd_max_length => int($iops_rd_max_length),
4141         iops_wr_max_length => int($iops_wr_max_length),
4142     );
4143
4144 }
4145
4146 # old code, only used to shutdown old VM after update
4147 sub __read_avail {
4148     my ($fh, $timeout) = @_;
4149
4150     my $sel = new IO::Select;
4151     $sel->add($fh);
4152
4153     my $res = '';
4154     my $buf;
4155
4156     my @ready;
4157     while (scalar (@ready = $sel->can_read($timeout))) {
4158         my $count;
4159         if ($count = $fh->sysread($buf, 8192)) {
4160             if ($buf =~ /^(.*)\(qemu\) $/s) {
4161                 $res .= $1;
4162                 last;
4163             } else {
4164                 $res .= $buf;
4165             }
4166         } else {
4167             if (!defined($count)) {
4168                 die "$!\n";
4169             }
4170             last;
4171         }
4172     }
4173
4174     die "monitor read timeout\n" if !scalar(@ready);
4175
4176     return $res;
4177 }
4178
4179 sub qemu_block_resize {
4180     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4181
4182     my $running = check_running($vmid);
4183
4184     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4185
4186     return if !$running;
4187
4188     my $padding = (1024 - $size % 1024) % 1024;
4189     $size = $size + $padding;
4190
4191     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4192
4193 }
4194
4195 sub qemu_volume_snapshot {
4196     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4197
4198     my $running = check_running($vmid);
4199
4200     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4201         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4202     } else {
4203         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4204     }
4205 }
4206
4207 sub qemu_volume_snapshot_delete {
4208     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4209
4210     my $running = check_running($vmid);
4211
4212     if($running) {
4213
4214         $running = undef;
4215         my $conf = PVE::QemuConfig->load_config($vmid);
4216         foreach_drive($conf, sub {
4217             my ($ds, $drive) = @_;
4218             $running = 1 if $drive->{file} eq $volid;
4219         });
4220     }
4221
4222     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4223         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4224     } else {
4225         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4226     }
4227 }
4228
4229 sub set_migration_caps {
4230     my ($vmid) = @_;
4231
4232     my $cap_ref = [];
4233
4234     my $enabled_cap = {
4235         "auto-converge" => 1,
4236         "xbzrle" => 1,
4237         "x-rdma-pin-all" => 0,
4238         "zero-blocks" => 0,
4239         "compress" => 0
4240     };
4241
4242     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4243
4244     for my $supported_capability (@$supported_capabilities) {
4245         push @$cap_ref, {
4246             capability => $supported_capability->{capability},
4247             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4248         };
4249     }
4250
4251     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4252 }
4253
4254 my $fast_plug_option = {
4255     'lock' => 1,
4256     'name' => 1,
4257     'onboot' => 1,
4258     'shares' => 1,
4259     'startup' => 1,
4260     'description' => 1,
4261     'protection' => 1,
4262     'vmstatestorage' => 1,
4263     'hookscript' => 1,
4264     'tags' => 1,
4265 };
4266
4267 # hotplug changes in [PENDING]
4268 # $selection hash can be used to only apply specified options, for
4269 # example: { cores => 1 } (only apply changed 'cores')
4270 # $errors ref is used to return error messages
4271 sub vmconfig_hotplug_pending {
4272     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4273
4274     my $defaults = load_defaults();
4275     my $arch = get_vm_arch($conf);
4276     my $machine_type = get_vm_machine($conf, undef, $arch);
4277
4278     # commit values which do not have any impact on running VM first
4279     # Note: those option cannot raise errors, we we do not care about
4280     # $selection and always apply them.
4281
4282     my $add_error = sub {
4283         my ($opt, $msg) = @_;
4284         $errors->{$opt} = "hotplug problem - $msg";
4285     };
4286
4287     my $changes = 0;
4288     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4289         if ($fast_plug_option->{$opt}) {
4290             $conf->{$opt} = $conf->{pending}->{$opt};
4291             delete $conf->{pending}->{$opt};
4292             $changes = 1;
4293         }
4294     }
4295
4296     if ($changes) {
4297         PVE::QemuConfig->write_config($vmid, $conf);
4298     }
4299
4300     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4301
4302     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4303     foreach my $opt (sort keys %$pending_delete_hash) {
4304         next if $selection && !$selection->{$opt};
4305         my $force = $pending_delete_hash->{$opt}->{force};
4306         eval {
4307             if ($opt eq 'hotplug') {
4308                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4309             } elsif ($opt eq 'tablet') {
4310                 die "skip\n" if !$hotplug_features->{usb};
4311                 if ($defaults->{tablet}) {
4312                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4313                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4314                         if $arch eq 'aarch64';
4315                 } else {
4316                     vm_deviceunplug($vmid, $conf, 'tablet');
4317                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4318                 }
4319             } elsif ($opt =~ m/^usb\d+/) {
4320                 die "skip\n";
4321                 # since we cannot reliably hot unplug usb devices
4322                 # we are disabling it
4323                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4324                 vm_deviceunplug($vmid, $conf, $opt);
4325             } elsif ($opt eq 'vcpus') {
4326                 die "skip\n" if !$hotplug_features->{cpu};
4327                 qemu_cpu_hotplug($vmid, $conf, undef);
4328             } elsif ($opt eq 'balloon') {
4329                 # enable balloon device is not hotpluggable
4330                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4331                 # here we reset the ballooning value to memory
4332                 my $balloon = $conf->{memory} || $defaults->{memory};
4333                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4334             } elsif ($fast_plug_option->{$opt}) {
4335                 # do nothing
4336             } elsif ($opt =~ m/^net(\d+)$/) {
4337                 die "skip\n" if !$hotplug_features->{network};
4338                 vm_deviceunplug($vmid, $conf, $opt);
4339             } elsif (is_valid_drivename($opt)) {
4340                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4341                 vm_deviceunplug($vmid, $conf, $opt);
4342                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4343             } elsif ($opt =~ m/^memory$/) {
4344                 die "skip\n" if !$hotplug_features->{memory};
4345                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4346             } elsif ($opt eq 'cpuunits') {
4347                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4348             } elsif ($opt eq 'cpulimit') {
4349                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4350             } else {
4351                 die "skip\n";
4352             }
4353         };
4354         if (my $err = $@) {
4355             &$add_error($opt, $err) if $err ne "skip\n";
4356         } else {
4357             delete $conf->{$opt};
4358             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4359         }
4360     }
4361
4362     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4363     $apply_pending_cloudinit = sub {
4364         return if $apply_pending_cloudinit_done; # once is enough
4365         $apply_pending_cloudinit_done = 1; # once is enough
4366
4367         my ($key, $value) = @_;
4368
4369         my @cloudinit_opts = keys %$confdesc_cloudinit;
4370         foreach my $opt (keys %{$conf->{pending}}) {
4371             next if !grep { $_ eq $opt } @cloudinit_opts;
4372             $conf->{$opt} = delete $conf->{pending}->{$opt};
4373         }
4374
4375         my $new_conf = { %$conf };
4376         $new_conf->{$key} = $value;
4377         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4378     };
4379
4380     foreach my $opt (keys %{$conf->{pending}}) {
4381         next if $selection && !$selection->{$opt};
4382         my $value = $conf->{pending}->{$opt};
4383         eval {
4384             if ($opt eq 'hotplug') {
4385                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4386             } elsif ($opt eq 'tablet') {
4387                 die "skip\n" if !$hotplug_features->{usb};
4388                 if ($value == 1) {
4389                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4390                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4391                         if $arch eq 'aarch64';
4392                 } elsif ($value == 0) {
4393                     vm_deviceunplug($vmid, $conf, 'tablet');
4394                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4395                 }
4396             } elsif ($opt =~ m/^usb\d+$/) {
4397                 die "skip\n";
4398                 # since we cannot reliably hot unplug usb devices
4399                 # we are disabling it
4400                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4401                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4402                 die "skip\n" if !$d;
4403                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4404             } elsif ($opt eq 'vcpus') {
4405                 die "skip\n" if !$hotplug_features->{cpu};
4406                 qemu_cpu_hotplug($vmid, $conf, $value);
4407             } elsif ($opt eq 'balloon') {
4408                 # enable/disable balloning device is not hotpluggable
4409                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4410                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4411                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4412
4413                 # allow manual ballooning if shares is set to zero
4414                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4415                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4416                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4417                 }
4418             } elsif ($opt =~ m/^net(\d+)$/) {
4419                 # some changes can be done without hotplug
4420                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4421                                     $vmid, $opt, $value, $arch, $machine_type);
4422             } elsif (is_valid_drivename($opt)) {
4423                 die "skip\n" if $opt eq 'efidisk0';
4424                 # some changes can be done without hotplug
4425                 my $drive = parse_drive($opt, $value);
4426                 if (drive_is_cloudinit($drive)) {
4427                     &$apply_pending_cloudinit($opt, $value);
4428                 }
4429                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4430                                      $vmid, $opt, $value, $arch, $machine_type);
4431             } elsif ($opt =~ m/^memory$/) { #dimms
4432                 die "skip\n" if !$hotplug_features->{memory};
4433                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4434             } elsif ($opt eq 'cpuunits') {
4435                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4436             } elsif ($opt eq 'cpulimit') {
4437                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4438                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4439             } else {
4440                 die "skip\n";  # skip non-hot-pluggable options
4441             }
4442         };
4443         if (my $err = $@) {
4444             &$add_error($opt, $err) if $err ne "skip\n";
4445         } else {
4446             $conf->{$opt} = $value;
4447             delete $conf->{pending}->{$opt};
4448         }
4449     }
4450
4451     PVE::QemuConfig->write_config($vmid, $conf);
4452 }
4453
4454 sub try_deallocate_drive {
4455     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4456
4457     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4458         my $volid = $drive->{file};
4459         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4460             my $sid = PVE::Storage::parse_volume_id($volid);
4461             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4462
4463             # check if the disk is really unused
4464             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4465                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4466             PVE::Storage::vdisk_free($storecfg, $volid);
4467             return 1;
4468         } else {
4469             # If vm is not owner of this disk remove from config
4470             return 1;
4471         }
4472     }
4473
4474     return undef;
4475 }
4476
4477 sub vmconfig_delete_or_detach_drive {
4478     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4479
4480     my $drive = parse_drive($opt, $conf->{$opt});
4481
4482     my $rpcenv = PVE::RPCEnvironment::get();
4483     my $authuser = $rpcenv->get_user();
4484
4485     if ($force) {
4486         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4487         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4488     } else {
4489         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4490     }
4491 }
4492
4493
4494
4495 sub vmconfig_apply_pending {
4496     my ($vmid, $conf, $storecfg, $errors) = @_;
4497
4498     my $add_apply_error = sub {
4499         my ($opt, $msg) = @_;
4500         my $err_msg = "unable to apply pending change $opt : $msg";
4501         $errors->{$opt} = $err_msg;
4502         warn $err_msg;
4503     };
4504
4505     # cold plug
4506
4507     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4508     foreach my $opt (sort keys %$pending_delete_hash) {
4509         my $force = $pending_delete_hash->{$opt}->{force};
4510         eval {
4511             if ($opt =~ m/^unused/) {
4512                 die "internal error";
4513             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4514                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4515             }
4516         };
4517         if (my $err = $@) {
4518             $add_apply_error->($opt, $err);
4519         } else {
4520             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4521             delete $conf->{$opt};
4522         }
4523     }
4524
4525     PVE::QemuConfig->cleanup_pending($conf);
4526
4527     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4528         next if $opt eq 'delete'; # just to be sure
4529         eval {
4530             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4531                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4532             }
4533         };
4534         if (my $err = $@) {
4535             $add_apply_error->($opt, $err);
4536         } else {
4537             $conf->{$opt} = delete $conf->{pending}->{$opt};
4538         }
4539     }
4540
4541     # write all changes at once to avoid unnecessary i/o
4542     PVE::QemuConfig->write_config($vmid, $conf);
4543 }
4544
4545 sub vmconfig_update_net {
4546     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4547
4548     my $newnet = parse_net($value);
4549
4550     if ($conf->{$opt}) {
4551         my $oldnet = parse_net($conf->{$opt});
4552
4553         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4554             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4555             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4556             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4557
4558             # for non online change, we try to hot-unplug
4559             die "skip\n" if !$hotplug;
4560             vm_deviceunplug($vmid, $conf, $opt);
4561         } else {
4562
4563             die "internal error" if $opt !~ m/net(\d+)/;
4564             my $iface = "tap${vmid}i$1";
4565
4566             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4567                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4568                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4569                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4570                 PVE::Network::tap_unplug($iface);
4571
4572                 if ($have_sdn) {
4573                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4574                 } else {
4575                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4576                 }
4577             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4578                 # Rate can be applied on its own but any change above needs to
4579                 # include the rate in tap_plug since OVS resets everything.
4580                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4581             }
4582
4583             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4584                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4585             }
4586
4587             return 1;
4588         }
4589     }
4590
4591     if ($hotplug) {
4592         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4593     } else {
4594         die "skip\n";
4595     }
4596 }
4597
4598 sub vmconfig_update_disk {
4599     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4600
4601     my $drive = parse_drive($opt, $value);
4602
4603     if ($conf->{$opt}) {
4604
4605         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4606
4607             my $media = $drive->{media} || 'disk';
4608             my $oldmedia = $old_drive->{media} || 'disk';
4609             die "unable to change media type\n" if $media ne $oldmedia;
4610
4611             if (!drive_is_cdrom($old_drive)) {
4612
4613                 if ($drive->{file} ne $old_drive->{file}) {
4614
4615                     die "skip\n" if !$hotplug;
4616
4617                     # unplug and register as unused
4618                     vm_deviceunplug($vmid, $conf, $opt);
4619                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4620
4621                 } else {
4622                     # update existing disk
4623
4624                     # skip non hotpluggable value
4625                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4626                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4627                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4628                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4629                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4630                         die "skip\n";
4631                     }
4632
4633                     # apply throttle
4634                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4635                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4636                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4637                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4638                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4639                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4640                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4641                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4642                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4643                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4644                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4645                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4646                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4647                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4648                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4649                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4650                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4651                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4652
4653                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4654                                                    ($drive->{mbps} || 0)*1024*1024,
4655                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4656                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4657                                                    $drive->{iops} || 0,
4658                                                    $drive->{iops_rd} || 0,
4659                                                    $drive->{iops_wr} || 0,
4660                                                    ($drive->{mbps_max} || 0)*1024*1024,
4661                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4662                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4663                                                    $drive->{iops_max} || 0,
4664                                                    $drive->{iops_rd_max} || 0,
4665                                                    $drive->{iops_wr_max} || 0,
4666                                                    $drive->{bps_max_length} || 1,
4667                                                    $drive->{bps_rd_max_length} || 1,
4668                                                    $drive->{bps_wr_max_length} || 1,
4669                                                    $drive->{iops_max_length} || 1,
4670                                                    $drive->{iops_rd_max_length} || 1,
4671                                                    $drive->{iops_wr_max_length} || 1);
4672
4673                     }
4674
4675                     return 1;
4676                 }
4677
4678             } else { # cdrom
4679
4680                 if ($drive->{file} eq 'none') {
4681                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4682                     if (drive_is_cloudinit($old_drive)) {
4683                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4684                     }
4685                 } else {
4686                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4687
4688                     # force eject if locked
4689                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4690
4691                     if ($path) {
4692                         mon_cmd($vmid, "blockdev-change-medium",
4693                             id => "$opt", filename => "$path");
4694                     }
4695                 }
4696
4697                 return 1;
4698             }
4699         }
4700     }
4701
4702     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4703     # hotplug new disks
4704     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4705     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4706 }
4707
4708 sub vm_start {
4709     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4710         $forcemachine, $spice_ticket, $migration_network, $migration_type,
4711         $targetstorage, $timeout, $nbd_protocol_version) = @_;
4712
4713     PVE::QemuConfig->lock_config($vmid, sub {
4714         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4715
4716         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4717
4718         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
4719
4720         PVE::QemuConfig->check_lock($conf)
4721             if !($skiplock || $is_suspended);
4722
4723         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4724
4725         # clean up leftover reboot request files
4726         eval { clear_reboot_request($vmid); };
4727         warn $@ if $@;
4728
4729         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4730             vmconfig_apply_pending($vmid, $conf, $storecfg);
4731             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4732         }
4733
4734         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4735
4736         my $defaults = load_defaults();
4737
4738         # set environment variable useful inside network script
4739         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4740
4741         my $local_volumes = {};
4742
4743         if ($targetstorage) {
4744             foreach_drive($conf, sub {
4745                 my ($ds, $drive) = @_;
4746
4747                 return if drive_is_cdrom($drive);
4748
4749                 my $volid = $drive->{file};
4750
4751                 return if !$volid;
4752
4753                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4754
4755                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4756                 return if $scfg->{shared};
4757                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4758             });
4759
4760             my $format = undef;
4761
4762             foreach my $opt (sort keys %$local_volumes) {
4763
4764                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4765                 my $drive = parse_drive($opt, $conf->{$opt});
4766
4767                 # If a remote storage is specified and the format of the original
4768                 # volume is not available there, fall back to the default format.
4769                 # Otherwise use the same format as the original.
4770                 if ($targetstorage && $targetstorage ne "1") {
4771                     $storeid = $targetstorage;
4772                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4773                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4774                     my $fileFormat = qemu_img_format($scfg, $volname);
4775                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4776                 } else {
4777                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4778                     $format = qemu_img_format($scfg, $volname);
4779                 }
4780
4781                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4782                 my $newdrive = $drive;
4783                 $newdrive->{format} = $format;
4784                 $newdrive->{file} = $newvolid;
4785                 my $drivestr = print_drive($newdrive);
4786                 $local_volumes->{$opt} = $drivestr;
4787                 #pass drive to conf for command line
4788                 $conf->{$opt} = $drivestr;
4789             }
4790         }
4791
4792         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4793
4794         if ($is_suspended) {
4795             # enforce machine type on suspended vm to ensure HW compatibility
4796             $forcemachine = $conf->{runningmachine};
4797             print "Resuming suspended VM\n";
4798         }
4799
4800         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4801
4802         my $migration_ip;
4803         my $get_migration_ip = sub {
4804             my ($cidr, $nodename) = @_;
4805
4806             return $migration_ip if defined($migration_ip);
4807
4808             if (!defined($cidr)) {
4809                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4810                 $cidr = $dc_conf->{migration}->{network};
4811             }
4812
4813             if (defined($cidr)) {
4814                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4815
4816                 die "could not get IP: no address configured on local " .
4817                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
4818
4819                 die "could not get IP: multiple addresses configured on local " .
4820                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
4821
4822                 $migration_ip = @$ips[0];
4823             }
4824
4825             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4826                 if !defined($migration_ip);
4827
4828             return $migration_ip;
4829         };
4830
4831         my $migrate_uri;
4832         if ($statefile) {
4833             if ($statefile eq 'tcp') {
4834                 my $localip = "localhost";
4835                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4836                 my $nodename = nodename();
4837
4838                 if (!defined($migration_type)) {
4839                     if (defined($datacenterconf->{migration}->{type})) {
4840                         $migration_type = $datacenterconf->{migration}->{type};
4841                     } else {
4842                         $migration_type = 'secure';
4843                     }
4844                 }
4845
4846                 if ($migration_type eq 'insecure') {
4847                     $localip = $get_migration_ip->($migration_network, $nodename);
4848                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4849                 }
4850
4851                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4852                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4853                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4854                 push @$cmd, '-incoming', $migrate_uri;
4855                 push @$cmd, '-S';
4856
4857             } elsif ($statefile eq 'unix') {
4858                 # should be default for secure migrations as a ssh TCP forward
4859                 # tunnel is not deterministic reliable ready and fails regurarly
4860                 # to set up in time, so use UNIX socket forwards
4861                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4862                 unlink $socket_addr;
4863
4864                 $migrate_uri = "unix:$socket_addr";
4865
4866                 push @$cmd, '-incoming', $migrate_uri;
4867                 push @$cmd, '-S';
4868
4869             } elsif (-e $statefile) {
4870                 push @$cmd, '-loadstate', $statefile;
4871             } else {
4872                 my $statepath = PVE::Storage::path($storecfg, $statefile);
4873                 push @$vollist, $statefile;
4874                 push @$cmd, '-loadstate', $statepath;
4875             }
4876         } elsif ($paused) {
4877             push @$cmd, '-S';
4878         }
4879
4880         # host pci devices
4881         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4882           my $d = parse_hostpci($conf->{"hostpci$i"});
4883           next if !$d;
4884           my $pcidevices = $d->{pciid};
4885           foreach my $pcidevice (@$pcidevices) {
4886                 my $pciid = $pcidevice->{id};
4887
4888                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4889                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4890                 die "no pci device info for device '$pciid'\n" if !$info;
4891
4892                 if ($d->{mdev}) {
4893                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4894                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4895                 } else {
4896                     die "can't unbind/bind pci group to vfio '$pciid'\n"
4897                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4898                     die "can't reset pci device '$pciid'\n"
4899                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4900                 }
4901           }
4902         }
4903
4904         PVE::Storage::activate_volumes($storecfg, $vollist);
4905
4906         eval {
4907             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4908                 outfunc => sub {}, errfunc => sub {});
4909         };
4910         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4911         # timeout should be more than enough here...
4912         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4913
4914         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4915                                                   : $defaults->{cpuunits};
4916
4917         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
4918         my %run_params = (
4919             timeout => $statefile ? undef : $start_timeout,
4920             umask => 0077,
4921             noerr => 1,
4922         );
4923
4924         # when migrating, prefix QEMU output so other side can pick up any
4925         # errors that might occur and show the user
4926         if ($migratedfrom) {
4927             $run_params{quiet} = 1;
4928             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4929         }
4930
4931         my %properties = (
4932             Slice => 'qemu.slice',
4933             KillMode => 'none',
4934             CPUShares => $cpuunits
4935         );
4936
4937         if (my $cpulimit = $conf->{cpulimit}) {
4938             $properties{CPUQuota} = int($cpulimit * 100);
4939         }
4940         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4941
4942         my $run_qemu = sub {
4943             PVE::Tools::run_fork sub {
4944                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4945
4946                 my $exitcode = run_command($cmd, %run_params);
4947                 die "QEMU exited with code $exitcode\n" if $exitcode;
4948             };
4949         };
4950
4951         if ($conf->{hugepages}) {
4952
4953             my $code = sub {
4954                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4955                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4956
4957                 PVE::QemuServer::Memory::hugepages_mount();
4958                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4959
4960                 eval { $run_qemu->() };
4961                 if (my $err = $@) {
4962                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4963                     die $err;
4964                 }
4965
4966                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4967             };
4968             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4969
4970         } else {
4971             eval { $run_qemu->() };
4972         }
4973
4974         if (my $err = $@) {
4975             # deactivate volumes if start fails
4976             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4977             die "start failed: $err";
4978         }
4979
4980         print "migration listens on $migrate_uri\n" if $migrate_uri;
4981
4982         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4983             eval { mon_cmd($vmid, "cont"); };
4984             warn $@ if $@;
4985         }
4986
4987         #start nbd server for storage migration
4988         if ($targetstorage) {
4989             $nbd_protocol_version //= 0;
4990
4991             my $migrate_storage_uri;
4992             # nbd_protocol_version > 0 for unix socket support
4993             if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
4994                 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
4995                 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
4996                 $migrate_storage_uri = "nbd:unix:$socket_path";
4997             } else {
4998                 my $nodename = nodename();
4999                 my $localip = $get_migration_ip->($migration_network, $nodename);
5000                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5001                 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5002
5003                 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5004                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5005                 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5006             }
5007
5008             foreach my $opt (sort keys %$local_volumes) {
5009                 my $drivestr = $local_volumes->{$opt};
5010                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5011                 print "storage migration listens on $migrate_storage_uri:exportname=drive-$opt volume:$drivestr\n";
5012             }
5013         }
5014
5015         if ($migratedfrom) {
5016             eval {
5017                 set_migration_caps($vmid);
5018             };
5019             warn $@ if $@;
5020
5021             if ($spice_port) {
5022                 print "spice listens on port $spice_port\n";
5023                 if ($spice_ticket) {
5024                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5025                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5026                 }
5027             }
5028
5029         } else {
5030             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5031                 if !$statefile && $conf->{balloon};
5032
5033             foreach my $opt (keys %$conf) {
5034                 next if $opt !~  m/^net\d+$/;
5035                 my $nicconf = parse_net($conf->{$opt});
5036                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5037             }
5038         }
5039
5040         mon_cmd($vmid, 'qom-set',
5041                     path => "machine/peripheral/balloon0",
5042                     property => "guest-stats-polling-interval",
5043                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5044
5045         if ($is_suspended) {
5046             print "Resumed VM, removing state\n";
5047             if (my $vmstate = $conf->{vmstate}) {
5048                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5049                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5050             }
5051             delete $conf->@{qw(lock vmstate runningmachine)};
5052             PVE::QemuConfig->write_config($vmid, $conf);
5053         }
5054
5055         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5056     });
5057 }
5058
5059 sub vm_commandline {
5060     my ($storecfg, $vmid, $snapname) = @_;
5061
5062     my $conf = PVE::QemuConfig->load_config($vmid);
5063     my $forcemachine;
5064
5065     if ($snapname) {
5066         my $snapshot = $conf->{snapshots}->{$snapname};
5067         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5068
5069         # check for a 'runningmachine' in snapshot
5070         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5071
5072         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5073
5074         $conf = $snapshot;
5075     }
5076
5077     my $defaults = load_defaults();
5078
5079     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5080
5081     return PVE::Tools::cmd2string($cmd);
5082 }
5083
5084 sub vm_reset {
5085     my ($vmid, $skiplock) = @_;
5086
5087     PVE::QemuConfig->lock_config($vmid, sub {
5088
5089         my $conf = PVE::QemuConfig->load_config($vmid);
5090
5091         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5092
5093         mon_cmd($vmid, "system_reset");
5094     });
5095 }
5096
5097 sub get_vm_volumes {
5098     my ($conf) = @_;
5099
5100     my $vollist = [];
5101     foreach_volid($conf, sub {
5102         my ($volid, $attr) = @_;
5103
5104         return if $volid =~ m|^/|;
5105
5106         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5107         return if !$sid;
5108
5109         push @$vollist, $volid;
5110     });
5111
5112     return $vollist;
5113 }
5114
5115 sub vm_stop_cleanup {
5116     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5117
5118     eval {
5119
5120         if (!$keepActive) {
5121             my $vollist = get_vm_volumes($conf);
5122             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5123         }
5124
5125         foreach my $ext (qw(mon qmp pid vnc qga)) {
5126             unlink "/var/run/qemu-server/${vmid}.$ext";
5127         }
5128
5129         if ($conf->{ivshmem}) {
5130             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5131             # just delete it for now, VMs which have this already open do not
5132             # are affected, but new VMs will get a separated one. If this
5133             # becomes an issue we either add some sort of ref-counting or just
5134             # add a "don't delete on stop" flag to the ivshmem format.
5135             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5136         }
5137
5138         foreach my $key (keys %$conf) {
5139             next if $key !~ m/^hostpci(\d+)$/;
5140             my $hostpciindex = $1;
5141             my $d = parse_hostpci($conf->{$key});
5142             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5143
5144             foreach my $pci (@{$d->{pciid}}) {
5145                 my $pciid = $pci->{id};
5146                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5147             }
5148         }
5149
5150         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5151     };
5152     warn $@ if $@; # avoid errors - just warn
5153 }
5154
5155 # call only in locked context
5156 sub _do_vm_stop {
5157     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5158
5159     my $pid = check_running($vmid, $nocheck);
5160     return if !$pid;
5161
5162     my $conf;
5163     if (!$nocheck) {
5164         $conf = PVE::QemuConfig->load_config($vmid);
5165         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5166         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5167             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5168             $timeout = $opts->{down} if $opts->{down};
5169         }
5170         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5171     }
5172
5173     eval {
5174         if ($shutdown) {
5175             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5176                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5177             } else {
5178                 mon_cmd($vmid, "system_powerdown");
5179             }
5180         } else {
5181             mon_cmd($vmid, "quit");
5182         }
5183     };
5184     my $err = $@;
5185
5186     if (!$err) {
5187         $timeout = 60 if !defined($timeout);
5188
5189         my $count = 0;
5190         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5191             $count++;
5192             sleep 1;
5193         }
5194
5195         if ($count >= $timeout) {
5196             if ($force) {
5197                 warn "VM still running - terminating now with SIGTERM\n";
5198                 kill 15, $pid;
5199             } else {
5200                 die "VM quit/powerdown failed - got timeout\n";
5201             }
5202         } else {
5203             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5204             return;
5205         }
5206     } else {
5207         if ($force) {
5208             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5209             kill 15, $pid;
5210         } else {
5211             die "VM quit/powerdown failed\n";
5212         }
5213     }
5214
5215     # wait again
5216     $timeout = 10;
5217
5218     my $count = 0;
5219     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5220         $count++;
5221         sleep 1;
5222     }
5223
5224     if ($count >= $timeout) {
5225         warn "VM still running - terminating now with SIGKILL\n";
5226         kill 9, $pid;
5227         sleep 1;
5228     }
5229
5230     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5231 }
5232
5233 # Note: use $nocheck to skip tests if VM configuration file exists.
5234 # We need that when migration VMs to other nodes (files already moved)
5235 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5236 sub vm_stop {
5237     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5238
5239     $force = 1 if !defined($force) && !$shutdown;
5240
5241     if ($migratedfrom){
5242         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5243         kill 15, $pid if $pid;
5244         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5245         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5246         return;
5247     }
5248
5249     PVE::QemuConfig->lock_config($vmid, sub {
5250         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5251    });
5252 }
5253
5254 sub vm_reboot {
5255     my ($vmid, $timeout) = @_;
5256
5257     PVE::QemuConfig->lock_config($vmid, sub {
5258         eval {
5259
5260             # only reboot if running, as qmeventd starts it again on a stop event
5261             return if !check_running($vmid);
5262
5263             create_reboot_request($vmid);
5264
5265             my $storecfg = PVE::Storage::config();
5266             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5267
5268         };
5269         if (my $err = $@) {
5270             # avoid that the next normal shutdown will be confused for a reboot
5271             clear_reboot_request($vmid);
5272             die $err;
5273         }
5274    });
5275 }
5276
5277 # note: if using the statestorage parameter, the caller has to check privileges
5278 sub vm_suspend {
5279     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5280
5281     my $conf;
5282     my $path;
5283     my $storecfg;
5284     my $vmstate;
5285
5286     PVE::QemuConfig->lock_config($vmid, sub {
5287
5288         $conf = PVE::QemuConfig->load_config($vmid);
5289
5290         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5291         PVE::QemuConfig->check_lock($conf)
5292             if !($skiplock || $is_backing_up);
5293
5294         die "cannot suspend to disk during backup\n"
5295             if $is_backing_up && $includestate;
5296
5297         if ($includestate) {
5298             $conf->{lock} = 'suspending';
5299             my $date = strftime("%Y-%m-%d", localtime(time()));
5300             $storecfg = PVE::Storage::config();
5301             if (!$statestorage) {
5302                 $statestorage = find_vmstate_storage($conf, $storecfg);
5303                 # check permissions for the storage
5304                 my $rpcenv = PVE::RPCEnvironment::get();
5305                 if ($rpcenv->{type} ne 'cli') {
5306                     my $authuser = $rpcenv->get_user();
5307                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5308                 }
5309             }
5310
5311
5312             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5313             $path = PVE::Storage::path($storecfg, $vmstate);
5314             PVE::QemuConfig->write_config($vmid, $conf);
5315         } else {
5316             mon_cmd($vmid, "stop");
5317         }
5318     });
5319
5320     if ($includestate) {
5321         # save vm state
5322         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5323
5324         eval {
5325             mon_cmd($vmid, "savevm-start", statefile => $path);
5326             for(;;) {
5327                 my $state = mon_cmd($vmid, "query-savevm");
5328                 if (!$state->{status}) {
5329                     die "savevm not active\n";
5330                 } elsif ($state->{status} eq 'active') {
5331                     sleep(1);
5332                     next;
5333                 } elsif ($state->{status} eq 'completed') {
5334                     print "State saved, quitting\n";
5335                     last;
5336                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5337                     die "query-savevm failed with error '$state->{error}'\n"
5338                 } else {
5339                     die "query-savevm returned status '$state->{status}'\n";
5340                 }
5341             }
5342         };
5343         my $err = $@;
5344
5345         PVE::QemuConfig->lock_config($vmid, sub {
5346             $conf = PVE::QemuConfig->load_config($vmid);
5347             if ($err) {
5348                 # cleanup, but leave suspending lock, to indicate something went wrong
5349                 eval {
5350                     mon_cmd($vmid, "savevm-end");
5351                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5352                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5353                     delete $conf->@{qw(vmstate runningmachine)};
5354                     PVE::QemuConfig->write_config($vmid, $conf);
5355                 };
5356                 warn $@ if $@;
5357                 die $err;
5358             }
5359
5360             die "lock changed unexpectedly\n"
5361                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5362
5363             mon_cmd($vmid, "quit");
5364             $conf->{lock} = 'suspended';
5365             PVE::QemuConfig->write_config($vmid, $conf);
5366         });
5367     }
5368 }
5369
5370 sub vm_resume {
5371     my ($vmid, $skiplock, $nocheck) = @_;
5372
5373     PVE::QemuConfig->lock_config($vmid, sub {
5374         my $res = mon_cmd($vmid, 'query-status');
5375         my $resume_cmd = 'cont';
5376
5377         if ($res->{status} && $res->{status} eq 'suspended') {
5378             $resume_cmd = 'system_wakeup';
5379         }
5380
5381         if (!$nocheck) {
5382
5383             my $conf = PVE::QemuConfig->load_config($vmid);
5384
5385             PVE::QemuConfig->check_lock($conf)
5386                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5387         }
5388
5389         mon_cmd($vmid, $resume_cmd);
5390     });
5391 }
5392
5393 sub vm_sendkey {
5394     my ($vmid, $skiplock, $key) = @_;
5395
5396     PVE::QemuConfig->lock_config($vmid, sub {
5397
5398         my $conf = PVE::QemuConfig->load_config($vmid);
5399
5400         # there is no qmp command, so we use the human monitor command
5401         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5402         die $res if $res ne '';
5403     });
5404 }
5405
5406 # vzdump restore implementaion
5407
5408 sub tar_archive_read_firstfile {
5409     my $archive = shift;
5410
5411     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5412
5413     # try to detect archive type first
5414     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5415         die "unable to open file '$archive'\n";
5416     my $firstfile = <$fh>;
5417     kill 15, $pid;
5418     close $fh;
5419
5420     die "ERROR: archive contaions no data\n" if !$firstfile;
5421     chomp $firstfile;
5422
5423     return $firstfile;
5424 }
5425
5426 sub tar_restore_cleanup {
5427     my ($storecfg, $statfile) = @_;
5428
5429     print STDERR "starting cleanup\n";
5430
5431     if (my $fd = IO::File->new($statfile, "r")) {
5432         while (defined(my $line = <$fd>)) {
5433             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5434                 my $volid = $2;
5435                 eval {
5436                     if ($volid =~ m|^/|) {
5437                         unlink $volid || die 'unlink failed\n';
5438                     } else {
5439                         PVE::Storage::vdisk_free($storecfg, $volid);
5440                     }
5441                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5442                 };
5443                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5444             } else {
5445                 print STDERR "unable to parse line in statfile - $line";
5446             }
5447         }
5448         $fd->close();
5449     }
5450 }
5451
5452 sub restore_file_archive {
5453     my ($archive, $vmid, $user, $opts) = @_;
5454
5455     my $format = $opts->{format};
5456     my $comp;
5457
5458     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5459         $format = 'tar' if !$format;
5460         $comp = 'gzip';
5461     } elsif ($archive =~ m/\.tar$/) {
5462         $format = 'tar' if !$format;
5463     } elsif ($archive =~ m/.tar.lzo$/) {
5464         $format = 'tar' if !$format;
5465         $comp = 'lzop';
5466     } elsif ($archive =~ m/\.vma$/) {
5467         $format = 'vma' if !$format;
5468     } elsif ($archive =~ m/\.vma\.gz$/) {
5469         $format = 'vma' if !$format;
5470         $comp = 'gzip';
5471     } elsif ($archive =~ m/\.vma\.lzo$/) {
5472         $format = 'vma' if !$format;
5473         $comp = 'lzop';
5474     } else {
5475         $format = 'vma' if !$format; # default
5476     }
5477
5478     # try to detect archive format
5479     if ($format eq 'tar') {
5480         return restore_tar_archive($archive, $vmid, $user, $opts);
5481     } else {
5482         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5483     }
5484 }
5485
5486 # hepler to remove disks that will not be used after restore
5487 my $restore_cleanup_oldconf = sub {
5488     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5489
5490     foreach_drive($oldconf, sub {
5491         my ($ds, $drive) = @_;
5492
5493         return if drive_is_cdrom($drive, 1);
5494
5495         my $volid = $drive->{file};
5496         return if !$volid || $volid =~ m|^/|;
5497
5498         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5499         return if !$path || !$owner || ($owner != $vmid);
5500
5501         # Note: only delete disk we want to restore
5502         # other volumes will become unused
5503         if ($virtdev_hash->{$ds}) {
5504             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5505             if (my $err = $@) {
5506                 warn $err;
5507             }
5508         }
5509     });
5510
5511     # delete vmstate files, after the restore we have no snapshots anymore
5512     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5513         my $snap = $oldconf->{snapshots}->{$snapname};
5514         if ($snap->{vmstate}) {
5515             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5516             if (my $err = $@) {
5517                 warn $err;
5518             }
5519         }
5520     }
5521 };
5522
5523 # Helper to parse vzdump backup device hints
5524 #
5525 # $rpcenv: Environment, used to ckeck storage permissions
5526 # $user: User ID, to check storage permissions
5527 # $storecfg: Storage configuration
5528 # $fh: the file handle for reading the configuration
5529 # $devinfo: should contain device sizes for all backu-up'ed devices
5530 # $options: backup options (pool, default storage)
5531 #
5532 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5533 my $parse_backup_hints = sub {
5534     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5535
5536     my $virtdev_hash = {};
5537
5538     while (defined(my $line = <$fh>)) {
5539         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5540             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5541             die "archive does not contain data for drive '$virtdev'\n"
5542                 if !$devinfo->{$devname};
5543
5544             if (defined($options->{storage})) {
5545                 $storeid = $options->{storage} || 'local';
5546             } elsif (!$storeid) {
5547                 $storeid = 'local';
5548             }
5549             $format = 'raw' if !$format;
5550             $devinfo->{$devname}->{devname} = $devname;
5551             $devinfo->{$devname}->{virtdev} = $virtdev;
5552             $devinfo->{$devname}->{format} = $format;
5553             $devinfo->{$devname}->{storeid} = $storeid;
5554
5555             # check permission on storage
5556             my $pool = $options->{pool}; # todo: do we need that?
5557             if ($user ne 'root@pam') {
5558                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5559             }
5560
5561             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5562         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5563             my $virtdev = $1;
5564             my $drive = parse_drive($virtdev, $2);
5565             if (drive_is_cloudinit($drive)) {
5566                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5567                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5568                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5569
5570                 $virtdev_hash->{$virtdev} = {
5571                     format => $format,
5572                     storeid => $options->{storage} // $storeid,
5573                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5574                     is_cloudinit => 1,
5575                 };
5576             }
5577         }
5578     }
5579
5580     return $virtdev_hash;
5581 };
5582
5583 # Helper to allocate and activate all volumes required for a restore
5584 #
5585 # $storecfg: Storage configuration
5586 # $virtdev_hash: as returned by parse_backup_hints()
5587 #
5588 # Returns: { $virtdev => $volid }
5589 my $restore_allocate_devices = sub {
5590     my ($storecfg, $virtdev_hash, $vmid) = @_;
5591
5592     my $map = {};
5593     foreach my $virtdev (sort keys %$virtdev_hash) {
5594         my $d = $virtdev_hash->{$virtdev};
5595         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5596         my $storeid = $d->{storeid};
5597         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5598
5599         # test if requested format is supported
5600         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5601         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5602         $d->{format} = $defFormat if !$supported;
5603
5604         my $name;
5605         if ($d->{is_cloudinit}) {
5606             $name = "vm-$vmid-cloudinit";
5607             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5608         }
5609
5610         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5611
5612         print STDERR "new volume ID is '$volid'\n";
5613         $d->{volid} = $volid;
5614
5615         PVE::Storage::activate_volumes($storecfg, [$volid]);
5616
5617         $map->{$virtdev} = $volid;
5618     }
5619
5620     return $map;
5621 };
5622
5623 my $restore_update_config_line = sub {
5624     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5625
5626     return if $line =~ m/^\#qmdump\#/;
5627     return if $line =~ m/^\#vzdump\#/;
5628     return if $line =~ m/^lock:/;
5629     return if $line =~ m/^unused\d+:/;
5630     return if $line =~ m/^parent:/;
5631
5632     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5633     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5634         # try to convert old 1.X settings
5635         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5636         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5637             my ($model, $macaddr) = split(/\=/, $devconfig);
5638             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5639             my $net = {
5640                 model => $model,
5641                 bridge => "vmbr$ind",
5642                 macaddr => $macaddr,
5643             };
5644             my $netstr = print_net($net);
5645
5646             print $outfd "net$cookie->{netcount}: $netstr\n";
5647             $cookie->{netcount}++;
5648         }
5649     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5650         my ($id, $netstr) = ($1, $2);
5651         my $net = parse_net($netstr);
5652         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5653         $netstr = print_net($net);
5654         print $outfd "$id: $netstr\n";
5655     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5656         my $virtdev = $1;
5657         my $value = $3;
5658         my $di = parse_drive($virtdev, $value);
5659         if (defined($di->{backup}) && !$di->{backup}) {
5660             print $outfd "#$line";
5661         } elsif ($map->{$virtdev}) {
5662             delete $di->{format}; # format can change on restore
5663             $di->{file} = $map->{$virtdev};
5664             $value = print_drive($di);
5665             print $outfd "$virtdev: $value\n";
5666         } else {
5667             print $outfd $line;
5668         }
5669     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5670         my $vmgenid = $1;
5671         if ($vmgenid ne '0') {
5672             # always generate a new vmgenid if there was a valid one setup
5673             $vmgenid = generate_uuid();
5674         }
5675         print $outfd "vmgenid: $vmgenid\n";
5676     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5677         my ($uuid, $uuid_str);
5678         UUID::generate($uuid);
5679         UUID::unparse($uuid, $uuid_str);
5680         my $smbios1 = parse_smbios1($2);
5681         $smbios1->{uuid} = $uuid_str;
5682         print $outfd $1.print_smbios1($smbios1)."\n";
5683     } else {
5684         print $outfd $line;
5685     }
5686 };
5687
5688 my $restore_deactivate_volumes = sub {
5689     my ($storecfg, $devinfo) = @_;
5690
5691     my $vollist = [];
5692     foreach my $devname (keys %$devinfo) {
5693         my $volid = $devinfo->{$devname}->{volid};
5694         push @$vollist, $volid if $volid;
5695     }
5696
5697     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5698 };
5699
5700 my $restore_destroy_volumes = sub {
5701     my ($storecfg, $devinfo) = @_;
5702
5703     foreach my $devname (keys %$devinfo) {
5704         my $volid = $devinfo->{$devname}->{volid};
5705         next if !$volid;
5706         eval {
5707             if ($volid =~ m|^/|) {
5708                 unlink $volid || die 'unlink failed\n';
5709             } else {
5710                 PVE::Storage::vdisk_free($storecfg, $volid);
5711             }
5712             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5713         };
5714         print STDERR "unable to cleanup '$volid' - $@" if $@;
5715     }
5716 };
5717
5718 sub scan_volids {
5719     my ($cfg, $vmid) = @_;
5720
5721     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5722
5723     my $volid_hash = {};
5724     foreach my $storeid (keys %$info) {
5725         foreach my $item (@{$info->{$storeid}}) {
5726             next if !($item->{volid} && $item->{size});
5727             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5728             $volid_hash->{$item->{volid}} = $item;
5729         }
5730     }
5731
5732     return $volid_hash;
5733 }
5734
5735 sub update_disk_config {
5736     my ($vmid, $conf, $volid_hash) = @_;
5737
5738     my $changes;
5739     my $prefix = "VM $vmid:";
5740
5741     # used and unused disks
5742     my $referenced = {};
5743
5744     # Note: it is allowed to define multiple storages with same path (alias), so
5745     # we need to check both 'volid' and real 'path' (two different volid can point
5746     # to the same path).
5747
5748     my $referencedpath = {};
5749
5750     # update size info
5751     foreach my $opt (keys %$conf) {
5752         if (is_valid_drivename($opt)) {
5753             my $drive = parse_drive($opt, $conf->{$opt});
5754             my $volid = $drive->{file};
5755             next if !$volid;
5756
5757             # mark volid as "in-use" for next step
5758             $referenced->{$volid} = 1;
5759             if ($volid_hash->{$volid} &&
5760                 (my $path = $volid_hash->{$volid}->{path})) {
5761                 $referencedpath->{$path} = 1;
5762             }
5763
5764             next if drive_is_cdrom($drive);
5765             next if !$volid_hash->{$volid};
5766
5767             my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5768             if (defined($updated)) {
5769                 $changes = 1;
5770                 $conf->{$opt} = print_drive($updated);
5771                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5772             }
5773         }
5774     }
5775
5776     # remove 'unusedX' entry if volume is used
5777     foreach my $opt (keys %$conf) {
5778         next if $opt !~ m/^unused\d+$/;
5779         my $volid = $conf->{$opt};
5780         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5781         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5782             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5783             $changes = 1;
5784             delete $conf->{$opt};
5785         }
5786
5787         $referenced->{$volid} = 1;
5788         $referencedpath->{$path} = 1 if $path;
5789     }
5790
5791     foreach my $volid (sort keys %$volid_hash) {
5792         next if $volid =~ m/vm-$vmid-state-/;
5793         next if $referenced->{$volid};
5794         my $path = $volid_hash->{$volid}->{path};
5795         next if !$path; # just to be sure
5796         next if $referencedpath->{$path};
5797         $changes = 1;
5798         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5799         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5800         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5801     }
5802
5803     return $changes;
5804 }
5805
5806 sub rescan {
5807     my ($vmid, $nolock, $dryrun) = @_;
5808
5809     my $cfg = PVE::Storage::config();
5810
5811     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5812     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5813     foreach my $stor (keys %{$cfg->{ids}}) {
5814         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5815     }
5816
5817     print "rescan volumes...\n";
5818     my $volid_hash = scan_volids($cfg, $vmid);
5819
5820     my $updatefn =  sub {
5821         my ($vmid) = @_;
5822
5823         my $conf = PVE::QemuConfig->load_config($vmid);
5824
5825         PVE::QemuConfig->check_lock($conf);
5826
5827         my $vm_volids = {};
5828         foreach my $volid (keys %$volid_hash) {
5829             my $info = $volid_hash->{$volid};
5830             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5831         }
5832
5833         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5834
5835         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5836     };
5837
5838     if (defined($vmid)) {
5839         if ($nolock) {
5840             &$updatefn($vmid);
5841         } else {
5842             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5843         }
5844     } else {
5845         my $vmlist = config_list();
5846         foreach my $vmid (keys %$vmlist) {
5847             if ($nolock) {
5848                 &$updatefn($vmid);
5849             } else {
5850                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5851             }
5852         }
5853     }
5854 }
5855
5856 sub restore_proxmox_backup_archive {
5857     my ($archive, $vmid, $user, $options) = @_;
5858
5859     my $storecfg = PVE::Storage::config();
5860
5861     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5862     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5863
5864     my $server = $scfg->{server};
5865     my $datastore = $scfg->{datastore};
5866     my $username = $scfg->{username} // 'root@pam';
5867     my $fingerprint = $scfg->{fingerprint};
5868
5869     my $repo = "$username\@$server:$datastore";
5870     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5871     local $ENV{PBS_PASSWORD} = $password;
5872     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5873
5874     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5875         PVE::Storage::parse_volname($storecfg, $archive);
5876
5877     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5878
5879     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5880
5881     my $tmpdir = "/var/tmp/vzdumptmp$$";
5882     rmtree $tmpdir;
5883     mkpath $tmpdir;
5884
5885     my $conffile = PVE::QemuConfig->config_file($vmid);
5886     my $tmpfn = "$conffile.$$.tmp";
5887      # disable interrupts (always do cleanups)
5888     local $SIG{INT} =
5889         local $SIG{TERM} =
5890         local $SIG{QUIT} =
5891         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5892
5893     # Note: $oldconf is undef if VM does not exists
5894     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5895     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5896
5897     my $rpcenv = PVE::RPCEnvironment::get();
5898     my $devinfo = {};
5899
5900     eval {
5901         # enable interrupts
5902         local $SIG{INT} =
5903             local $SIG{TERM} =
5904             local $SIG{QUIT} =
5905             local $SIG{HUP} =
5906             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5907
5908         my $cfgfn = "$tmpdir/qemu-server.conf";
5909         my $firewall_config_fn = "$tmpdir/fw.conf";
5910         my $index_fn = "$tmpdir/index.json";
5911
5912         my $cmd = "restore";
5913
5914         my $param = [$pbs_backup_name, "index.json", $index_fn];
5915         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5916         my $index = PVE::Tools::file_get_contents($index_fn);
5917         $index = decode_json($index);
5918
5919         # print Dumper($index);
5920         foreach my $info (@{$index->{files}}) {
5921             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
5922                 my $devname = $1;
5923                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
5924                     $devinfo->{$devname}->{size} = $1;
5925                 } else {
5926                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
5927                 }
5928             }
5929         }
5930
5931         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
5932         if (!$is_qemu_server_backup) {
5933             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
5934         }
5935         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
5936
5937         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
5938         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5939
5940         if ($has_firewall_config) {
5941             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
5942             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5943
5944             my $pve_firewall_dir = '/etc/pve/firewall';
5945             mkdir $pve_firewall_dir; # make sure the dir exists
5946             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
5947         }
5948
5949         my $fh = IO::File->new($cfgfn, "r") ||
5950             "unable to read qemu-server.conf - $!\n";
5951
5952         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
5953
5954         # fixme: rate limit?
5955
5956         # create empty/temp config
5957         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
5958
5959         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
5960
5961         # allocate volumes
5962         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
5963
5964         foreach my $virtdev (sort keys %$virtdev_hash) {
5965             my $d = $virtdev_hash->{$virtdev};
5966             next if $d->{is_cloudinit}; # no need to restore cloudinit
5967
5968             my $volid = $d->{volid};
5969
5970             my $path = PVE::Storage::path($storecfg, $volid);
5971
5972             my $pbs_restore_cmd = [
5973                 '/usr/bin/pbs-restore',
5974                 '--repository', $repo,
5975                 $pbs_backup_name,
5976                 "$d->{devname}.img.fidx",
5977                 $path,
5978                 '--verbose',
5979                 ];
5980
5981             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
5982                 push @$pbs_restore_cmd, '--skip-zero';
5983             }
5984
5985             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
5986             print "restore proxmox backup image: $dbg_cmdstring\n";
5987             run_command($pbs_restore_cmd);
5988         }
5989
5990         $fh->seek(0, 0) || die "seek failed - $!\n";
5991
5992         my $outfd = new IO::File ($tmpfn, "w") ||
5993             die "unable to write config for VM $vmid\n";
5994
5995         my $cookie = { netcount => 0 };
5996         while (defined(my $line = <$fh>)) {
5997             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
5998         }
5999
6000         $fh->close();
6001         $outfd->close();
6002     };
6003     my $err = $@;
6004
6005     $restore_deactivate_volumes->($storecfg, $devinfo);
6006
6007     rmtree $tmpdir;
6008
6009     if ($err) {
6010         unlink $tmpfn;
6011         $restore_destroy_volumes->($storecfg, $devinfo);
6012         die $err;
6013     }
6014
6015     rename($tmpfn, $conffile) ||
6016         die "unable to commit configuration file '$conffile'\n";
6017
6018     PVE::Cluster::cfs_update(); # make sure we read new file
6019
6020     eval { rescan($vmid, 1); };
6021     warn $@ if $@;
6022 }
6023
6024 sub restore_vma_archive {
6025     my ($archive, $vmid, $user, $opts, $comp) = @_;
6026
6027     my $readfrom = $archive;
6028
6029     my $cfg = PVE::Storage::config();
6030     my $commands = [];
6031     my $bwlimit = $opts->{bwlimit};
6032
6033     my $dbg_cmdstring = '';
6034     my $add_pipe = sub {
6035         my ($cmd) = @_;
6036         push @$commands, $cmd;
6037         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6038         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6039         $readfrom = '-';
6040     };
6041
6042     my $input = undef;
6043     if ($archive eq '-') {
6044         $input = '<&STDIN';
6045     } else {
6046         # If we use a backup from a PVE defined storage we also consider that
6047         # storage's rate limit:
6048         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6049         if (defined($volid)) {
6050             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6051             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6052             if ($readlimit) {
6053                 print STDERR "applying read rate limit: $readlimit\n";
6054                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6055                 $add_pipe->($cstream);
6056             }
6057         }
6058     }
6059
6060     if ($comp) {
6061         my $cmd;
6062         if ($comp eq 'gzip') {
6063             $cmd = ['zcat', $readfrom];
6064         } elsif ($comp eq 'lzop') {
6065             $cmd = ['lzop', '-d', '-c', $readfrom];
6066         } else {
6067             die "unknown compression method '$comp'\n";
6068         }
6069         $add_pipe->($cmd);
6070     }
6071
6072     my $tmpdir = "/var/tmp/vzdumptmp$$";
6073     rmtree $tmpdir;
6074
6075     # disable interrupts (always do cleanups)
6076     local $SIG{INT} =
6077         local $SIG{TERM} =
6078         local $SIG{QUIT} =
6079         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6080
6081     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6082     POSIX::mkfifo($mapfifo, 0600);
6083     my $fifofh;
6084
6085     my $openfifo = sub {
6086         open($fifofh, '>', $mapfifo) || die $!;
6087     };
6088
6089     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6090
6091     my $oldtimeout;
6092     my $timeout = 5;
6093
6094     my $devinfo = {};
6095
6096     my $rpcenv = PVE::RPCEnvironment::get();
6097
6098     my $conffile = PVE::QemuConfig->config_file($vmid);
6099     my $tmpfn = "$conffile.$$.tmp";
6100
6101     # Note: $oldconf is undef if VM does not exist
6102     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6103     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6104
6105     my %storage_limits;
6106
6107     my $print_devmap = sub {
6108         my $cfgfn = "$tmpdir/qemu-server.conf";
6109
6110         # we can read the config - that is already extracted
6111         my $fh = IO::File->new($cfgfn, "r") ||
6112             "unable to read qemu-server.conf - $!\n";
6113
6114         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6115         if (-f $fwcfgfn) {
6116             my $pve_firewall_dir = '/etc/pve/firewall';
6117             mkdir $pve_firewall_dir; # make sure the dir exists
6118             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6119         }
6120
6121         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6122
6123         foreach my $key (keys %storage_limits) {
6124             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6125             next if !$limit;
6126             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6127             $storage_limits{$key} = $limit * 1024;
6128         }
6129
6130         foreach my $devname (keys %$devinfo) {
6131             die "found no device mapping information for device '$devname'\n"
6132                 if !$devinfo->{$devname}->{virtdev};
6133         }
6134
6135         # create empty/temp config
6136         if ($oldconf) {
6137             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6138             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6139         }
6140
6141         # allocate volumes
6142         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6143
6144         # print restore information to $fifofh
6145         foreach my $virtdev (sort keys %$virtdev_hash) {
6146             my $d = $virtdev_hash->{$virtdev};
6147             next if $d->{is_cloudinit}; # no need to restore cloudinit
6148
6149             my $storeid = $d->{storeid};
6150             my $volid = $d->{volid};
6151
6152             my $map_opts = '';
6153             if (my $limit = $storage_limits{$storeid}) {
6154                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6155             }
6156
6157             my $write_zeros = 1;
6158             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6159                 $write_zeros = 0;
6160             }
6161
6162             my $path = PVE::Storage::path($cfg, $volid);
6163
6164             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6165
6166             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6167         }
6168
6169         $fh->seek(0, 0) || die "seek failed - $!\n";
6170
6171         my $outfd = new IO::File ($tmpfn, "w") ||
6172             die "unable to write config for VM $vmid\n";
6173
6174         my $cookie = { netcount => 0 };
6175         while (defined(my $line = <$fh>)) {
6176             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6177         }
6178
6179         $fh->close();
6180         $outfd->close();
6181     };
6182
6183     eval {
6184         # enable interrupts
6185         local $SIG{INT} =
6186             local $SIG{TERM} =
6187             local $SIG{QUIT} =
6188             local $SIG{HUP} =
6189             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6190         local $SIG{ALRM} = sub { die "got timeout\n"; };
6191
6192         $oldtimeout = alarm($timeout);
6193
6194         my $parser = sub {
6195             my $line = shift;
6196
6197             print "$line\n";
6198
6199             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6200                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6201                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6202             } elsif ($line =~ m/^CTIME: /) {
6203                 # we correctly received the vma config, so we can disable
6204                 # the timeout now for disk allocation (set to 10 minutes, so
6205                 # that we always timeout if something goes wrong)
6206                 alarm(600);
6207                 &$print_devmap();
6208                 print $fifofh "done\n";
6209                 my $tmp = $oldtimeout || 0;
6210                 $oldtimeout = undef;
6211                 alarm($tmp);
6212                 close($fifofh);
6213             }
6214         };
6215
6216         print "restore vma archive: $dbg_cmdstring\n";
6217         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6218     };
6219     my $err = $@;
6220
6221     alarm($oldtimeout) if $oldtimeout;
6222
6223     $restore_deactivate_volumes->($cfg, $devinfo);
6224
6225     unlink $mapfifo;
6226     rmtree $tmpdir;
6227
6228     if ($err) {
6229         unlink $tmpfn;
6230         $restore_destroy_volumes->($cfg, $devinfo);
6231         die $err;
6232     }
6233
6234     rename($tmpfn, $conffile) ||
6235         die "unable to commit configuration file '$conffile'\n";
6236
6237     PVE::Cluster::cfs_update(); # make sure we read new file
6238
6239     eval { rescan($vmid, 1); };
6240     warn $@ if $@;
6241 }
6242
6243 sub restore_tar_archive {
6244     my ($archive, $vmid, $user, $opts) = @_;
6245
6246     if ($archive ne '-') {
6247         my $firstfile = tar_archive_read_firstfile($archive);
6248         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6249             if $firstfile ne 'qemu-server.conf';
6250     }
6251
6252     my $storecfg = PVE::Storage::config();
6253
6254     # avoid zombie disks when restoring over an existing VM -> cleanup first
6255     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6256     # skiplock=1 because qmrestore has set the 'create' lock itself already
6257     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6258     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6259
6260     my $tocmd = "/usr/lib/qemu-server/qmextract";
6261
6262     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6263     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6264     $tocmd .= ' --prealloc' if $opts->{prealloc};
6265     $tocmd .= ' --info' if $opts->{info};
6266
6267     # tar option "xf" does not autodetect compression when read from STDIN,
6268     # so we pipe to zcat
6269     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6270         PVE::Tools::shellquote("--to-command=$tocmd");
6271
6272     my $tmpdir = "/var/tmp/vzdumptmp$$";
6273     mkpath $tmpdir;
6274
6275     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6276     local $ENV{VZDUMP_VMID} = $vmid;
6277     local $ENV{VZDUMP_USER} = $user;
6278
6279     my $conffile = PVE::QemuConfig->config_file($vmid);
6280     my $tmpfn = "$conffile.$$.tmp";
6281
6282     # disable interrupts (always do cleanups)
6283     local $SIG{INT} =
6284         local $SIG{TERM} =
6285         local $SIG{QUIT} =
6286         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6287
6288     eval {
6289         # enable interrupts
6290         local $SIG{INT} =
6291             local $SIG{TERM} =
6292             local $SIG{QUIT} =
6293             local $SIG{HUP} =
6294             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6295
6296         if ($archive eq '-') {
6297             print "extracting archive from STDIN\n";
6298             run_command($cmd, input => "<&STDIN");
6299         } else {
6300             print "extracting archive '$archive'\n";
6301             run_command($cmd);
6302         }
6303
6304         return if $opts->{info};
6305
6306         # read new mapping
6307         my $map = {};
6308         my $statfile = "$tmpdir/qmrestore.stat";
6309         if (my $fd = IO::File->new($statfile, "r")) {
6310             while (defined (my $line = <$fd>)) {
6311                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6312                     $map->{$1} = $2 if $1;
6313                 } else {
6314                     print STDERR "unable to parse line in statfile - $line\n";
6315                 }
6316             }
6317             $fd->close();
6318         }
6319
6320         my $confsrc = "$tmpdir/qemu-server.conf";
6321
6322         my $srcfd = new IO::File($confsrc, "r") ||
6323             die "unable to open file '$confsrc'\n";
6324
6325         my $outfd = new IO::File ($tmpfn, "w") ||
6326             die "unable to write config for VM $vmid\n";
6327
6328         my $cookie = { netcount => 0 };
6329         while (defined (my $line = <$srcfd>)) {
6330             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6331         }
6332
6333         $srcfd->close();
6334         $outfd->close();
6335     };
6336     if (my $err = $@) {
6337         unlink $tmpfn;
6338         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6339         die $err;
6340     }
6341
6342     rmtree $tmpdir;
6343
6344     rename $tmpfn, $conffile ||
6345         die "unable to commit configuration file '$conffile'\n";
6346
6347     PVE::Cluster::cfs_update(); # make sure we read new file
6348
6349     eval { rescan($vmid, 1); };
6350     warn $@ if $@;
6351 };
6352
6353 sub foreach_storage_used_by_vm {
6354     my ($conf, $func) = @_;
6355
6356     my $sidhash = {};
6357
6358     foreach_drive($conf, sub {
6359         my ($ds, $drive) = @_;
6360         return if drive_is_cdrom($drive);
6361
6362         my $volid = $drive->{file};
6363
6364         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6365         $sidhash->{$sid} = $sid if $sid;
6366     });
6367
6368     foreach my $sid (sort keys %$sidhash) {
6369         &$func($sid);
6370     }
6371 }
6372
6373 my $qemu_snap_storage = {
6374     rbd => 1,
6375 };
6376 sub do_snapshots_with_qemu {
6377     my ($storecfg, $volid) = @_;
6378
6379     my $storage_name = PVE::Storage::parse_volume_id($volid);
6380     my $scfg = $storecfg->{ids}->{$storage_name};
6381
6382     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6383         return 1;
6384     }
6385
6386     if ($volid =~ m/\.(qcow2|qed)$/){
6387         return 1;
6388     }
6389
6390     return undef;
6391 }
6392
6393 sub qga_check_running {
6394     my ($vmid, $nowarn) = @_;
6395
6396     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6397     if ($@) {
6398         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6399         return 0;
6400     }
6401     return 1;
6402 }
6403
6404 sub template_create {
6405     my ($vmid, $conf, $disk) = @_;
6406
6407     my $storecfg = PVE::Storage::config();
6408
6409     foreach_drive($conf, sub {
6410         my ($ds, $drive) = @_;
6411
6412         return if drive_is_cdrom($drive);
6413         return if $disk && $ds ne $disk;
6414
6415         my $volid = $drive->{file};
6416         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6417
6418         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6419         $drive->{file} = $voliddst;
6420         $conf->{$ds} = print_drive($drive);
6421         PVE::QemuConfig->write_config($vmid, $conf);
6422     });
6423 }
6424
6425 sub convert_iscsi_path {
6426     my ($path) = @_;
6427
6428     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6429         my $portal = $1;
6430         my $target = $2;
6431         my $lun = $3;
6432
6433         my $initiator_name = get_initiator_name();
6434
6435         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6436                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6437     }
6438
6439     die "cannot convert iscsi path '$path', unkown format\n";
6440 }
6441
6442 sub qemu_img_convert {
6443     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6444
6445     my $storecfg = PVE::Storage::config();
6446     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6447     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6448
6449     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6450
6451     my $cachemode;
6452     my $src_path;
6453     my $src_is_iscsi = 0;
6454     my $src_format;
6455
6456     if ($src_storeid) {
6457         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6458         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6459         $src_format = qemu_img_format($src_scfg, $src_volname);
6460         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6461         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6462         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6463     } elsif (-f $src_volid) {
6464         $src_path = $src_volid;
6465         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6466             $src_format = $1;
6467         }
6468     }
6469
6470     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6471
6472     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6473     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6474     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6475     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6476
6477     my $cmd = [];
6478     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6479     push @$cmd, '-l', "snapshot.name=$snapname"
6480         if $snapname && $src_format && $src_format eq "qcow2";
6481     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6482     push @$cmd, '-T', $cachemode if defined($cachemode);
6483
6484     if ($src_is_iscsi) {
6485         push @$cmd, '--image-opts';
6486         $src_path = convert_iscsi_path($src_path);
6487     } elsif ($src_format) {
6488         push @$cmd, '-f', $src_format;
6489     }
6490
6491     if ($dst_is_iscsi) {
6492         push @$cmd, '--target-image-opts';
6493         $dst_path = convert_iscsi_path($dst_path);
6494     } else {
6495         push @$cmd, '-O', $dst_format;
6496     }
6497
6498     push @$cmd, $src_path;
6499
6500     if (!$dst_is_iscsi && $is_zero_initialized) {
6501         push @$cmd, "zeroinit:$dst_path";
6502     } else {
6503         push @$cmd, $dst_path;
6504     }
6505
6506     my $parser = sub {
6507         my $line = shift;
6508         if($line =~ m/\((\S+)\/100\%\)/){
6509             my $percent = $1;
6510             my $transferred = int($size * $percent / 100);
6511             my $remaining = $size - $transferred;
6512
6513             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6514         }
6515
6516     };
6517
6518     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6519     my $err = $@;
6520     die "copy failed: $err" if $err;
6521 }
6522
6523 sub qemu_img_format {
6524     my ($scfg, $volname) = @_;
6525
6526     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6527         return $1;
6528     } else {
6529         return "raw";
6530     }
6531 }
6532
6533 sub qemu_drive_mirror {
6534     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit) = @_;
6535
6536     $jobs = {} if !$jobs;
6537
6538     my $qemu_target;
6539     my $format;
6540     $jobs->{"drive-$drive"} = {};
6541
6542     if ($dst_volid =~ /^nbd:/) {
6543         $qemu_target = $dst_volid;
6544         $format = "nbd";
6545     } else {
6546         my $storecfg = PVE::Storage::config();
6547         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6548
6549         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6550
6551         $format = qemu_img_format($dst_scfg, $dst_volname);
6552
6553         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6554
6555         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6556     }
6557
6558     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6559     $opts->{format} = $format if $format;
6560
6561     if (defined($bwlimit)) {
6562         $opts->{speed} = $bwlimit * 1024;
6563         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6564     } else {
6565         print "drive mirror is starting for drive-$drive\n";
6566     }
6567
6568     # if a job already runs for this device we get an error, catch it for cleanup
6569     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6570     if (my $err = $@) {
6571         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6572         warn "$@\n" if $@;
6573         die "mirroring error: $err\n";
6574     }
6575
6576     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6577 }
6578
6579 # $completion can be either
6580 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6581 # 'cancel': wait until all jobs are ready, block-job-cancel them
6582 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6583 sub qemu_drive_mirror_monitor {
6584     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6585
6586     $completion //= 'complete';
6587
6588     eval {
6589         my $err_complete = 0;
6590
6591         while (1) {
6592             die "storage migration timed out\n" if $err_complete > 300;
6593
6594             my $stats = mon_cmd($vmid, "query-block-jobs");
6595
6596             my $running_mirror_jobs = {};
6597             foreach my $stat (@$stats) {
6598                 next if $stat->{type} ne 'mirror';
6599                 $running_mirror_jobs->{$stat->{device}} = $stat;
6600             }
6601
6602             my $readycounter = 0;
6603
6604             foreach my $job (keys %$jobs) {
6605
6606                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6607                     print "$job : finished\n";
6608                     delete $jobs->{$job};
6609                     next;
6610                 }
6611
6612                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6613
6614                 my $busy = $running_mirror_jobs->{$job}->{busy};
6615                 my $ready = $running_mirror_jobs->{$job}->{ready};
6616                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6617                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6618                     my $remaining = $total - $transferred;
6619                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6620
6621                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6622                 }
6623
6624                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6625             }
6626
6627             last if scalar(keys %$jobs) == 0;
6628
6629             if ($readycounter == scalar(keys %$jobs)) {
6630                 print "all mirroring jobs are ready \n";
6631                 last if $completion eq 'skip'; #do the complete later
6632
6633                 if ($vmiddst && $vmiddst != $vmid) {
6634                     my $agent_running = $qga && qga_check_running($vmid);
6635                     if ($agent_running) {
6636                         print "freeze filesystem\n";
6637                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6638                     } else {
6639                         print "suspend vm\n";
6640                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6641                     }
6642
6643                     # if we clone a disk for a new target vm, we don't switch the disk
6644                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6645
6646                     if ($agent_running) {
6647                         print "unfreeze filesystem\n";
6648                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6649                     } else {
6650                         print "resume vm\n";
6651                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6652                     }
6653
6654                     last;
6655                 } else {
6656
6657                     foreach my $job (keys %$jobs) {
6658                         # try to switch the disk if source and destination are on the same guest
6659                         print "$job: Completing block job...\n";
6660
6661                         my $op;
6662                         if ($completion eq 'complete') {
6663                             $op = 'block-job-complete';
6664                         } elsif ($completion eq 'cancel') {
6665                             $op = 'block-job-cancel';
6666                         } else {
6667                             die "invalid completion value: $completion\n";
6668                         }
6669                         eval { mon_cmd($vmid, $op, device => $job) };
6670                         if ($@ =~ m/cannot be completed/) {
6671                             print "$job: Block job cannot be completed, try again.\n";
6672                             $err_complete++;
6673                         }else {
6674                             print "$job: Completed successfully.\n";
6675                             $jobs->{$job}->{complete} = 1;
6676                         }
6677                     }
6678                 }
6679             }
6680             sleep 1;
6681         }
6682     };
6683     my $err = $@;
6684
6685     if ($err) {
6686         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6687         die "mirroring error: $err";
6688     }
6689
6690 }
6691
6692 sub qemu_blockjobs_cancel {
6693     my ($vmid, $jobs) = @_;
6694
6695     foreach my $job (keys %$jobs) {
6696         print "$job: Cancelling block job\n";
6697         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6698         $jobs->{$job}->{cancel} = 1;
6699     }
6700
6701     while (1) {
6702         my $stats = mon_cmd($vmid, "query-block-jobs");
6703
6704         my $running_jobs = {};
6705         foreach my $stat (@$stats) {
6706             $running_jobs->{$stat->{device}} = $stat;
6707         }
6708
6709         foreach my $job (keys %$jobs) {
6710
6711             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6712                 print "$job: Done.\n";
6713                 delete $jobs->{$job};
6714             }
6715         }
6716
6717         last if scalar(keys %$jobs) == 0;
6718
6719         sleep 1;
6720     }
6721 }
6722
6723 sub clone_disk {
6724     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6725         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
6726
6727     my $newvolid;
6728
6729     if (!$full) {
6730         print "create linked clone of drive $drivename ($drive->{file})\n";
6731         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6732         push @$newvollist, $newvolid;
6733     } else {
6734
6735         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6736         $storeid = $storage if $storage;
6737
6738         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6739         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6740
6741         print "create full clone of drive $drivename ($drive->{file})\n";
6742         my $name = undef;
6743         if (drive_is_cloudinit($drive)) {
6744             $name = "vm-$newvmid-cloudinit";
6745             $name .= ".$dst_format" if $dst_format ne 'raw';
6746             $snapname = undef;
6747             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6748         }
6749         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6750         push @$newvollist, $newvolid;
6751
6752         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6753
6754         if (drive_is_cloudinit($drive)) {
6755             goto no_data_clone;
6756         }
6757
6758         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6759         if (!$running || $snapname) {
6760             # TODO: handle bwlimits
6761             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6762         } else {
6763
6764             my $kvmver = get_running_qemu_version ($vmid);
6765             if (!min_version($kvmver, 2, 7)) {
6766                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6767                     if $drive->{iothread};
6768             }
6769
6770             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6771         }
6772     }
6773
6774 no_data_clone:
6775     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6776
6777     my $disk = $drive;
6778     $disk->{format} = undef;
6779     $disk->{file} = $newvolid;
6780     $disk->{size} = $size;
6781
6782     return $disk;
6783 }
6784
6785 sub get_running_qemu_version {
6786     my ($vmid) = @_;
6787     my $res = mon_cmd($vmid, "query-version");
6788     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6789 }
6790
6791 sub qemu_use_old_bios_files {
6792     my ($machine_type) = @_;
6793
6794     return if !$machine_type;
6795
6796     my $use_old_bios_files = undef;
6797
6798     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6799         $machine_type = $1;
6800         $use_old_bios_files = 1;
6801     } else {
6802         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6803         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6804         # load new efi bios files on migration. So this hack is required to allow
6805         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6806         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6807         $use_old_bios_files = !min_version($version, 2, 4);
6808     }
6809
6810     return ($use_old_bios_files, $machine_type);
6811 }
6812
6813 sub create_efidisk($$$$$) {
6814     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6815
6816     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6817     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6818
6819     my $vars_size_b = -s $ovmf_vars;
6820     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6821     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6822     PVE::Storage::activate_volumes($storecfg, [$volid]);
6823
6824     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6825     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6826
6827     return ($volid, $size/1024);
6828 }
6829
6830 sub vm_iothreads_list {
6831     my ($vmid) = @_;
6832
6833     my $res = mon_cmd($vmid, 'query-iothreads');
6834
6835     my $iothreads = {};
6836     foreach my $iothread (@$res) {
6837         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6838     }
6839
6840     return $iothreads;
6841 }
6842
6843 sub scsihw_infos {
6844     my ($conf, $drive) = @_;
6845
6846     my $maxdev = 0;
6847
6848     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6849         $maxdev = 7;
6850     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6851         $maxdev = 1;
6852     } else {
6853         $maxdev = 256;
6854     }
6855
6856     my $controller = int($drive->{index} / $maxdev);
6857     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6858
6859     return ($maxdev, $controller, $controller_prefix);
6860 }
6861
6862 sub windows_version {
6863     my ($ostype) = @_;
6864
6865     return 0 if !$ostype;
6866
6867     my $winversion = 0;
6868
6869     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6870         $winversion = 5;
6871     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6872         $winversion = 6;
6873     } elsif ($ostype =~ m/^win(\d+)$/) {
6874         $winversion = $1;
6875     }
6876
6877     return $winversion;
6878 }
6879
6880 sub resolve_dst_disk_format {
6881         my ($storecfg, $storeid, $src_volname, $format) = @_;
6882         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6883
6884         if (!$format) {
6885             # if no target format is specified, use the source disk format as hint
6886             if ($src_volname) {
6887                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6888                 $format = qemu_img_format($scfg, $src_volname);
6889             } else {
6890                 return $defFormat;
6891             }
6892         }
6893
6894         # test if requested format is supported - else use default
6895         my $supported = grep { $_ eq $format } @$validFormats;
6896         $format = $defFormat if !$supported;
6897         return $format;
6898 }
6899
6900 # NOTE: if this logic changes, please update docs & possibly gui logic
6901 sub find_vmstate_storage {
6902     my ($conf, $storecfg) = @_;
6903
6904     # first, return storage from conf if set
6905     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
6906
6907     my ($target, $shared, $local);
6908
6909     foreach_storage_used_by_vm($conf, sub {
6910         my ($sid) = @_;
6911         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
6912         my $dst = $scfg->{shared} ? \$shared : \$local;
6913         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
6914     });
6915
6916     # second, use shared storage where VM has at least one disk
6917     # third, use local storage where VM has at least one disk
6918     # fall back to local storage
6919     $target = $shared // $local // 'local';
6920
6921     return $target;
6922 }
6923
6924 sub generate_uuid {
6925     my ($uuid, $uuid_str);
6926     UUID::generate($uuid);
6927     UUID::unparse($uuid, $uuid_str);
6928     return $uuid_str;
6929 }
6930
6931 sub generate_smbios1_uuid {
6932     return "uuid=".generate_uuid();
6933 }
6934
6935 sub nbd_stop {
6936     my ($vmid) = @_;
6937
6938     mon_cmd($vmid, 'nbd-server-stop');
6939 }
6940
6941 sub create_reboot_request {
6942     my ($vmid) = @_;
6943     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
6944         or die "failed to create reboot trigger file: $!\n";
6945     close($fh);
6946 }
6947
6948 sub clear_reboot_request {
6949     my ($vmid) = @_;
6950     my $path = "/run/qemu-server/$vmid.reboot";
6951     my $res = 0;
6952
6953     $res = unlink($path);
6954     die "could not remove reboot request for $vmid: $!"
6955         if !$res && $! != POSIX::ENOENT;
6956
6957     return $res;
6958 }
6959
6960 # bash completion helper
6961
6962 sub complete_backup_archives {
6963     my ($cmdname, $pname, $cvalue) = @_;
6964
6965     my $cfg = PVE::Storage::config();
6966
6967     my $storeid;
6968
6969     if ($cvalue =~ m/^([^:]+):/) {
6970         $storeid = $1;
6971     }
6972
6973     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6974
6975     my $res = [];
6976     foreach my $id (keys %$data) {
6977         foreach my $item (@{$data->{$id}}) {
6978             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6979             push @$res, $item->{volid} if defined($item->{volid});
6980         }
6981     }
6982
6983     return $res;
6984 }
6985
6986 my $complete_vmid_full = sub {
6987     my ($running) = @_;
6988
6989     my $idlist = vmstatus();
6990
6991     my $res = [];
6992
6993     foreach my $id (keys %$idlist) {
6994         my $d = $idlist->{$id};
6995         if (defined($running)) {
6996             next if $d->{template};
6997             next if $running && $d->{status} ne 'running';
6998             next if !$running && $d->{status} eq 'running';
6999         }
7000         push @$res, $id;
7001
7002     }
7003     return $res;
7004 };
7005
7006 sub complete_vmid {
7007     return &$complete_vmid_full();
7008 }
7009
7010 sub complete_vmid_stopped {
7011     return &$complete_vmid_full(0);
7012 }
7013
7014 sub complete_vmid_running {
7015     return &$complete_vmid_full(1);
7016 }
7017
7018 sub complete_storage {
7019
7020     my $cfg = PVE::Storage::config();
7021     my $ids = $cfg->{ids};
7022
7023     my $res = [];
7024     foreach my $sid (keys %$ids) {
7025         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7026         next if !$ids->{$sid}->{content}->{images};
7027         push @$res, $sid;
7028     }
7029
7030     return $res;
7031 }
7032
7033 sub complete_migration_storage {
7034     my ($cmd, $param, $current_value, $all_args) = @_;
7035
7036     my $targetnode = @$all_args[1];
7037
7038     my $cfg = PVE::Storage::config();
7039     my $ids = $cfg->{ids};
7040
7041     my $res = [];
7042     foreach my $sid (keys %$ids) {
7043         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7044         next if !$ids->{$sid}->{content}->{images};
7045         push @$res, $sid;
7046     }
7047
7048     return $res;
7049 }
7050
7051 1;