PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_HOSTPCI_DEVICES = 16;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         pattern => '[-_.\w\d]+',
 850         optional => 1,
 851     },
 852     queues => {
 853         type => 'integer',
 854         minimum => 0, maximum => 16,
 855         description => 'Number of packet queues to be used on the device.',
 856         optional => 1,
 857     },
 858     rate => {
 859         type => 'number',
 860         minimum => 0,
 861         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 862         optional => 1,
 863     },
 864     tag => {
 865         type => 'integer',
 866         minimum => 1, maximum => 4094,
 867         description => 'VLAN tag to apply to packets on this interface.',
 868         optional => 1,
 869     },
 870     trunks => {
 871         type => 'string',
 872         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 873         description => 'VLAN trunks to pass through this interface.',
 874         format_description => 'vlanid[;vlanid...]',
 875         optional => 1,
 876     },
 877     firewall => {
 878         type => 'boolean',
 879         description => 'Whether this interface should be protected by the firewall.',
 880         optional => 1,
 881     },
 882     link_down => {
 883         type => 'boolean',
 884         description => 'Whether this interface should be disconnected (like pulling the plug).',
 885         optional => 1,
 886     },
 887 };
 888
 889 my $netdesc = {
 890     optional => 1,
 891     type => 'string', format => $net_fmt,
 892     description => "Specify network devices.",
 893 };
 894
 895 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 896
 897 my $ipconfig_fmt = {
 898     ip => {
 899         type => 'string',
 900         format => 'pve-ipv4-config',
 901         format_description => 'IPv4Format/CIDR',
 902         description => 'IPv4 address in CIDR format.',
 903         optional => 1,
 904         default => 'dhcp',
 905     },
 906     gw => {
 907         type => 'string',
 908         format => 'ipv4',
 909         format_description => 'GatewayIPv4',
 910         description => 'Default gateway for IPv4 traffic.',
 911         optional => 1,
 912         requires => 'ip',
 913     },
 914     ip6 => {
 915         type => 'string',
 916         format => 'pve-ipv6-config',
 917         format_description => 'IPv6Format/CIDR',
 918         description => 'IPv6 address in CIDR format.',
 919         optional => 1,
 920         default => 'dhcp',
 921     },
 922     gw6 => {
 923         type => 'string',
 924         format => 'ipv6',
 925         format_description => 'GatewayIPv6',
 926         description => 'Default gateway for IPv6 traffic.',
 927         optional => 1,
 928         requires => 'ip6',
 929     },
 930 };
 931 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 932 my $ipconfigdesc = {
 933     optional => 1,
 934     type => 'string', format => 'pve-qm-ipconfig',
 935     description => <<'EODESCR',
 936 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 937
 938 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 939
 940 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 941 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 942
 943 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 944 EODESCR
 945 };
 946 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 947
 948 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 949     $confdesc->{"net$i"} = $netdesc;
 950     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 951 }
 952
 953 foreach my $key (keys %$confdesc_cloudinit) {
 954     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 955 }
 956
 957 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 958 sub verify_volume_id_or_qm_path {
 959     my ($volid, $noerr) = @_;
 960
 961     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 962         return $volid;
 963     }
 964
 965     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 966     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 967     if ($@) {
 968         return undef if $noerr;
 969         die $@;
 970     }
 971     return $volid;
 972 }
 973
 974 my $usb_fmt = {
 975     host => {
 976         default_key => 1,
 977         type => 'string', format => 'pve-qm-usb-device',
 978         format_description => 'HOSTUSBDEVICE|spice',
 979         description => <<EODESCR,
 980 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 981
 982  'bus-port(.port)*' (decimal numbers) or
 983  'vendor_id:product_id' (hexadeciaml numbers) or
 984  'spice'
 985
 986 You can use the 'lsusb -t' command to list existing usb devices.
 987
 988 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 989
 990 The value 'spice' can be used to add a usb redirection devices for spice.
 991 EODESCR
 992     },
 993     usb3 => {
 994         optional => 1,
 995         type => 'boolean',
 996         description => "Specifies whether if given host option is a USB3 device or port.",
 997         default => 0,
 998     },
 999 };
1000
1001 my $usbdesc = {
1002     optional => 1,
1003     type => 'string', format => $usb_fmt,
1004     description => "Configure an USB device (n is 0 to 4).",
1005 };
1006 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1007
1008 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1009 my $hostpci_fmt = {
1010     host => {
1011         default_key => 1,
1012         type => 'string',
1013         pattern => qr/$PCIRE(;$PCIRE)*/,
1014         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1015         description => <<EODESCR,
1016 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1017 of PCI virtual functions of the host. HOSTPCIID syntax is:
1018
1019 'bus:dev.func' (hexadecimal numbers)
1020
1021 You can us the 'lspci' command to list existing PCI devices.
1022 EODESCR
1023     },
1024     rombar => {
1025         type => 'boolean',
1026         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1027         optional => 1,
1028         default => 1,
1029     },
1030     romfile => {
1031         type => 'string',
1032         pattern => '[^,;]+',
1033         format_description => 'string',
1034         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1035         optional => 1,
1036     },
1037     pcie => {
1038         type => 'boolean',
1039         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1040         optional => 1,
1041         default => 0,
1042     },
1043     'x-vga' => {
1044         type => 'boolean',
1045         description =>  "Enable vfio-vga device support.",
1046         optional => 1,
1047         default => 0,
1048     },
1049     'mdev' => {
1050         type => 'string',
1051         format_description => 'string',
1052         pattern => '[^/\.:]+',
1053         optional => 1,
1054         description => <<EODESCR
1055 The type of mediated device to use.
1056 An instance of this type will be created on startup of the VM and
1057 will be cleaned up when the VM stops.
1058 EODESCR
1059     }
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1121     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1122 }
1123
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1125     $confdesc->{"usb$i"} = $usbdesc;
1126 }
1127
1128 my $kvm_api_version = 0;
1129
1130 sub kvm_version {
1131     return $kvm_api_version if $kvm_api_version;
1132
1133     open my $fh, '<', '/dev/kvm'
1134         or return undef;
1135
1136     # 0xae00 => KVM_GET_API_VERSION
1137     $kvm_api_version = ioctl($fh, 0xae00, 0);
1138
1139     return $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version = {};
1143 my $kvm_mtime = {};
1144
1145 sub kvm_user_version {
1146     my ($binary) = @_;
1147
1148     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1149     my $st = stat($binary);
1150
1151     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1152     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1153         $cachedmtime == $st->mtime;
1154
1155     $kvm_user_version->{$binary} = 'unknown';
1156     $kvm_mtime->{$binary} = $st->mtime;
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version->{$binary} = $2;
1162         }
1163     };
1164
1165     eval { run_command([$binary, '--version'], outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version->{$binary};
1169
1170 }
1171
1172 sub kernel_has_vhost_net {
1173     return -c '/dev/vhost-net';
1174 }
1175
1176 sub option_exists {
1177     my $key = shift;
1178     return defined($confdesc->{$key});
1179 }
1180
1181 my $cdrom_path;
1182 sub get_cdrom_path {
1183
1184     return  $cdrom_path if $cdrom_path;
1185
1186     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1187     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1188     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1189 }
1190
1191 sub get_iso_path {
1192     my ($storecfg, $vmid, $cdrom) = @_;
1193
1194     if ($cdrom eq 'cdrom') {
1195         return get_cdrom_path();
1196     } elsif ($cdrom eq 'none') {
1197         return '';
1198     } elsif ($cdrom =~ m|^/|) {
1199         return $cdrom;
1200     } else {
1201         return PVE::Storage::path($storecfg, $cdrom);
1202     }
1203 }
1204
1205 # try to convert old style file names to volume IDs
1206 sub filename_to_volume_id {
1207     my ($vmid, $file, $media) = @_;
1208
1209      if (!($file eq 'none' || $file eq 'cdrom' ||
1210           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1211
1212         return undef if $file =~ m|/|;
1213
1214         if ($media && $media eq 'cdrom') {
1215             $file = "local:iso/$file";
1216         } else {
1217             $file = "local:$vmid/$file";
1218         }
1219     }
1220
1221     return $file;
1222 }
1223
1224 sub verify_media_type {
1225     my ($opt, $vtype, $media) = @_;
1226
1227     return if !$media;
1228
1229     my $etype;
1230     if ($media eq 'disk') {
1231         $etype = 'images';
1232     } elsif ($media eq 'cdrom') {
1233         $etype = 'iso';
1234     } else {
1235         die "internal error";
1236     }
1237
1238     return if ($vtype eq $etype);
1239
1240     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1241 }
1242
1243 sub cleanup_drive_path {
1244     my ($opt, $storecfg, $drive) = @_;
1245
1246     # try to convert filesystem paths to volume IDs
1247
1248     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1249         ($drive->{file} !~ m|^/dev/.+|) &&
1250         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1251         ($drive->{file} !~ m/^\d+$/)) {
1252         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1253         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1254         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1255         verify_media_type($opt, $vtype, $drive->{media});
1256         $drive->{file} = $volid;
1257     }
1258
1259     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1260 }
1261
1262 sub parse_hotplug_features {
1263     my ($data) = @_;
1264
1265     my $res = {};
1266
1267     return $res if $data eq '0';
1268
1269     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1270
1271     foreach my $feature (PVE::Tools::split_list($data)) {
1272         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1273             $res->{$1} = 1;
1274         } else {
1275             die "invalid hotplug feature '$feature'\n";
1276         }
1277     }
1278     return $res;
1279 }
1280
1281 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1282 sub pve_verify_hotplug_features {
1283     my ($value, $noerr) = @_;
1284
1285     return $value if parse_hotplug_features($value);
1286
1287     return undef if $noerr;
1288
1289     die "unable to parse hotplug option\n";
1290 }
1291
1292 sub scsi_inquiry {
1293     my($fh, $noerr) = @_;
1294
1295     my $SG_IO = 0x2285;
1296     my $SG_GET_VERSION_NUM = 0x2282;
1297
1298     my $versionbuf = "\x00" x 8;
1299     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1300     if (!$ret) {
1301         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1302         return undef;
1303     }
1304     my $version = unpack("I", $versionbuf);
1305     if ($version < 30000) {
1306         die "scsi generic interface too old\n"  if !$noerr;
1307         return undef;
1308     }
1309
1310     my $buf = "\x00" x 36;
1311     my $sensebuf = "\x00" x 8;
1312     my $cmd = pack("C x3 C x1", 0x12, 36);
1313
1314     # see /usr/include/scsi/sg.h
1315     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1316
1317     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1318                       length($sensebuf), 0, length($buf), $buf,
1319                       $cmd, $sensebuf, 6000);
1320
1321     $ret = ioctl($fh, $SG_IO, $packet);
1322     if (!$ret) {
1323         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1324         return undef;
1325     }
1326
1327     my @res = unpack($sg_io_hdr_t, $packet);
1328     if ($res[17] || $res[18]) {
1329         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1330         return undef;
1331     }
1332
1333     my $res = {};
1334     (my $byte0, my $byte1, $res->{vendor},
1335      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1336
1337     $res->{removable} = $byte1 & 128 ? 1 : 0;
1338     $res->{type} = $byte0 & 31;
1339
1340     return $res;
1341 }
1342
1343 sub path_is_scsi {
1344     my ($path) = @_;
1345
1346     my $fh = IO::File->new("+<$path") || return undef;
1347     my $res = scsi_inquiry($fh, 1);
1348     close($fh);
1349
1350     return $res;
1351 }
1352
1353 sub print_tabletdevice_full {
1354     my ($conf, $arch) = @_;
1355
1356     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1357
1358     # we use uhci for old VMs because tablet driver was buggy in older qemu
1359     my $usbbus;
1360     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1361         $usbbus = 'ehci';
1362     } else {
1363         $usbbus = 'uhci';
1364     }
1365
1366     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1367 }
1368
1369 sub print_keyboarddevice_full {
1370     my ($conf, $arch, $machine) = @_;
1371
1372     return undef if $arch ne 'aarch64';
1373
1374     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1375 }
1376
1377 sub print_drivedevice_full {
1378     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1379
1380     my $device = '';
1381     my $maxdev = 0;
1382
1383     if ($drive->{interface} eq 'virtio') {
1384         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1385         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1386         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1387     } elsif ($drive->{interface} eq 'scsi') {
1388
1389         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1390         my $unit = $drive->{index} % $maxdev;
1391         my $devicetype = 'hd';
1392         my $path = '';
1393         if (drive_is_cdrom($drive)) {
1394             $devicetype = 'cd';
1395         } else {
1396             if ($drive->{file} =~ m|^/|) {
1397                 $path = $drive->{file};
1398                 if (my $info = path_is_scsi($path)) {
1399                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1400                         $devicetype = 'block';
1401                     } elsif ($info->{type} == 1) { # tape
1402                         $devicetype = 'generic';
1403                     }
1404                 }
1405             } else {
1406                  $path = PVE::Storage::path($storecfg, $drive->{file});
1407             }
1408
1409             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1410             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1411             if ($path =~ m/^iscsi\:\/\// &&
1412                !min_version($version, 4, 1)) {
1413                 $devicetype = 'generic';
1414             }
1415         }
1416
1417         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1421         }
1422
1423         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1424             $device .= ",rotation_rate=1";
1425         }
1426         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1427
1428     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1429         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1430         my $controller = int($drive->{index} / $maxdev);
1431         my $unit = $drive->{index} % $maxdev;
1432         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1433
1434         $device = "ide-$devicetype";
1435         if ($drive->{interface} eq 'ide') {
1436             $device .= ",bus=ide.$controller,unit=$unit";
1437         } else {
1438             $device .= ",bus=ahci$controller.$unit";
1439         }
1440         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1441
1442         if ($devicetype eq 'hd') {
1443             if (my $model = $drive->{model}) {
1444                 $model = URI::Escape::uri_unescape($model);
1445                 $device .= ",model=$model";
1446             }
1447             if ($drive->{ssd}) {
1448                 $device .= ",rotation_rate=1";
1449             }
1450         }
1451         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1452     } elsif ($drive->{interface} eq 'usb') {
1453         die "implement me";
1454         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1455     } else {
1456         die "unsupported interface type";
1457     }
1458
1459     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1460
1461     if (my $serial = $drive->{serial}) {
1462         $serial = URI::Escape::uri_unescape($serial);
1463         $device .= ",serial=$serial";
1464     }
1465
1466
1467     return $device;
1468 }
1469
1470 sub get_initiator_name {
1471     my $initiator;
1472
1473     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1474     while (defined(my $line = <$fh>)) {
1475         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1476         $initiator = $1;
1477         last;
1478     }
1479     $fh->close();
1480
1481     return $initiator;
1482 }
1483
1484 sub print_drive_commandline_full {
1485     my ($storecfg, $vmid, $drive) = @_;
1486
1487     my $path;
1488     my $volid = $drive->{file};
1489     my $format;
1490
1491     if (drive_is_cdrom($drive)) {
1492         $path = get_iso_path($storecfg, $vmid, $volid);
1493     } else {
1494         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1495         if ($storeid) {
1496             $path = PVE::Storage::path($storecfg, $volid);
1497             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1498             $format = qemu_img_format($scfg, $volname);
1499         } else {
1500             $path = $volid;
1501             $format = "raw";
1502         }
1503    }
1504
1505     my $opts = '';
1506     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1507     foreach my $o (@qemu_drive_options) {
1508         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1509     }
1510
1511     # snapshot only accepts on|off
1512     if (defined($drive->{snapshot})) {
1513         my $v = $drive->{snapshot} ? 'on' : 'off';
1514         $opts .= ",snapshot=$v";
1515     }
1516
1517     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1518         my ($dir, $qmpname) = @$type;
1519         if (my $v = $drive->{"mbps$dir"}) {
1520             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1521         }
1522         if (my $v = $drive->{"mbps${dir}_max"}) {
1523             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1524         }
1525         if (my $v = $drive->{"bps${dir}_max_length"}) {
1526             $opts .= ",throttling.bps$qmpname-max-length=$v";
1527         }
1528         if (my $v = $drive->{"iops${dir}"}) {
1529             $opts .= ",throttling.iops$qmpname=$v";
1530         }
1531         if (my $v = $drive->{"iops${dir}_max"}) {
1532             $opts .= ",throttling.iops$qmpname-max=$v";
1533         }
1534         if (my $v = $drive->{"iops${dir}_max_length"}) {
1535             $opts .= ",throttling.iops$qmpname-max-length=$v";
1536         }
1537     }
1538
1539     $opts .= ",format=$format" if $format && !$drive->{format};
1540
1541     my $cache_direct = 0;
1542
1543     if (my $cache = $drive->{cache}) {
1544         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1545     } elsif (!drive_is_cdrom($drive)) {
1546         $opts .= ",cache=none";
1547         $cache_direct = 1;
1548     }
1549
1550     # aio native works only with O_DIRECT
1551     if (!$drive->{aio}) {
1552         if($cache_direct) {
1553             $opts .= ",aio=native";
1554         } else {
1555             $opts .= ",aio=threads";
1556         }
1557     }
1558
1559     if (!drive_is_cdrom($drive)) {
1560         my $detectzeroes;
1561         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1562             $detectzeroes = 'off';
1563         } elsif ($drive->{discard}) {
1564             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1565         } else {
1566             # This used to be our default with discard not being specified:
1567             $detectzeroes = 'on';
1568         }
1569         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1570     }
1571
1572     my $pathinfo = $path ? "file=$path," : '';
1573
1574     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1575 }
1576
1577 sub print_netdevice_full {
1578     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1579
1580     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1581
1582     my $device = $net->{model};
1583     if ($net->{model} eq 'virtio') {
1584          $device = 'virtio-net-pci';
1585      };
1586
1587     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1588     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1589     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1590         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1591         my $vectors = $net->{queues} * 2 + 2;
1592         $tmpstr .= ",vectors=$vectors,mq=on";
1593     }
1594     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1595
1596     if ($use_old_bios_files) {
1597         my $romfile;
1598         if ($device eq 'virtio-net-pci') {
1599             $romfile = 'pxe-virtio.rom';
1600         } elsif ($device eq 'e1000') {
1601             $romfile = 'pxe-e1000.rom';
1602         } elsif ($device eq 'ne2k') {
1603             $romfile = 'pxe-ne2k_pci.rom';
1604         } elsif ($device eq 'pcnet') {
1605             $romfile = 'pxe-pcnet.rom';
1606         } elsif ($device eq 'rtl8139') {
1607             $romfile = 'pxe-rtl8139.rom';
1608         }
1609         $tmpstr .= ",romfile=$romfile" if $romfile;
1610     }
1611
1612     return $tmpstr;
1613 }
1614
1615 sub print_netdev_full {
1616     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1617
1618     my $i = '';
1619     if ($netid =~ m/^net(\d+)$/) {
1620         $i = int($1);
1621     }
1622
1623     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1624
1625     my $ifname = "tap${vmid}i$i";
1626
1627     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1628     die "interface name '$ifname' is too long (max 15 character)\n"
1629         if length($ifname) >= 16;
1630
1631     my $vhostparam = '';
1632     if (is_native($arch)) {
1633         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1634     }
1635
1636     my $vmname = $conf->{name} || "vm$vmid";
1637
1638     my $netdev = "";
1639     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1640
1641     if ($net->{bridge}) {
1642         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1643     } else {
1644         $netdev = "type=user,id=$netid,hostname=$vmname";
1645     }
1646
1647     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1648
1649     return $netdev;
1650 }
1651
1652 my $vga_map = {
1653     'cirrus' => 'cirrus-vga',
1654     'std' => 'VGA',
1655     'vmware' => 'vmware-svga',
1656     'virtio' => 'virtio-vga',
1657 };
1658
1659 sub print_vga_device {
1660     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1661
1662     my $type = $vga_map->{$vga->{type}};
1663     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1664         $type = 'virtio-gpu';
1665     }
1666     my $vgamem_mb = $vga->{memory};
1667
1668     my $max_outputs = '';
1669     if ($qxlnum) {
1670         $type = $id ? 'qxl' : 'qxl-vga';
1671
1672         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1673             # set max outputs so linux can have up to 4 qxl displays with one device
1674             if (min_version($machine_version, 4, 1)) {
1675                 $max_outputs = ",max_outputs=4";
1676             }
1677         }
1678     }
1679
1680     die "no devicetype for $vga->{type}\n" if !$type;
1681
1682     my $memory = "";
1683     if ($vgamem_mb) {
1684         if ($vga->{type} eq 'virtio') {
1685             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1686             $memory = ",max_hostmem=$bytes";
1687         } elsif ($qxlnum) {
1688             # from https://www.spice-space.org/multiple-monitors.html
1689             $memory = ",vgamem_mb=$vga->{memory}";
1690             my $ram = $vgamem_mb * 4;
1691             my $vram = $vgamem_mb * 2;
1692             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1693         } else {
1694             $memory = ",vgamem_mb=$vga->{memory}";
1695         }
1696     } elsif ($qxlnum && $id) {
1697         $memory = ",ram_size=67108864,vram_size=33554432";
1698     }
1699
1700     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1701     my $vgaid = "vga" . ($id // '');
1702     my $pciaddr;
1703
1704     if ($q35 && $vgaid eq 'vga') {
1705         # the first display uses pcie.0 bus on q35 machines
1706         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1707     } else {
1708         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1709     }
1710
1711     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1712 }
1713
1714 sub parse_number_sets {
1715     my ($set) = @_;
1716     my $res = [];
1717     foreach my $part (split(/;/, $set)) {
1718         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1719             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1720             push @$res, [ $1, $2 ];
1721         } else {
1722             die "invalid range: $part\n";
1723         }
1724     }
1725     return $res;
1726 }
1727
1728 sub parse_numa {
1729     my ($data) = @_;
1730
1731     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1732     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1733     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1734     return $res;
1735 }
1736
1737 sub parse_hostpci {
1738     my ($value) = @_;
1739
1740     return undef if !$value;
1741
1742     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1743
1744     my @idlist = split(/;/, $res->{host});
1745     delete $res->{host};
1746     foreach my $id (@idlist) {
1747         my $devs = PVE::SysFSTools::lspci($id);
1748         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1749         push @{$res->{pciid}}, @$devs;
1750     }
1751     return $res;
1752 }
1753
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1755 sub parse_net {
1756     my ($data) = @_;
1757
1758     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1759     if ($@) {
1760         warn $@;
1761         return undef;
1762     }
1763     if (!defined($res->{macaddr})) {
1764         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1765         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1766     }
1767     return $res;
1768 }
1769
1770 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1771 sub parse_ipconfig {
1772     my ($data) = @_;
1773
1774     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1775     if ($@) {
1776         warn $@;
1777         return undef;
1778     }
1779
1780     if ($res->{gw} && !$res->{ip}) {
1781         warn 'gateway specified without specifying an IP address';
1782         return undef;
1783     }
1784     if ($res->{gw6} && !$res->{ip6}) {
1785         warn 'IPv6 gateway specified without specifying an IPv6 address';
1786         return undef;
1787     }
1788     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1789         warn 'gateway specified together with DHCP';
1790         return undef;
1791     }
1792     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1793         # gw6 + auto/dhcp
1794         warn "IPv6 gateway specified together with $res->{ip6} address";
1795         return undef;
1796     }
1797
1798     if (!$res->{ip} && !$res->{ip6}) {
1799         return { ip => 'dhcp', ip6 => 'dhcp' };
1800     }
1801
1802     return $res;
1803 }
1804
1805 sub print_net {
1806     my $net = shift;
1807
1808     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1809 }
1810
1811 sub add_random_macs {
1812     my ($settings) = @_;
1813
1814     foreach my $opt (keys %$settings) {
1815         next if $opt !~ m/^net(\d+)$/;
1816         my $net = parse_net($settings->{$opt});
1817         next if !$net;
1818         $settings->{$opt} = print_net($net);
1819     }
1820 }
1821
1822 sub vm_is_volid_owner {
1823     my ($storecfg, $vmid, $volid) = @_;
1824
1825     if ($volid !~  m|^/|) {
1826         my ($path, $owner);
1827         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1828         if ($owner && ($owner == $vmid)) {
1829             return 1;
1830         }
1831     }
1832
1833     return undef;
1834 }
1835
1836 sub vmconfig_register_unused_drive {
1837     my ($storecfg, $vmid, $conf, $drive) = @_;
1838
1839     if (drive_is_cloudinit($drive)) {
1840         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1841         warn $@ if $@;
1842     } elsif (!drive_is_cdrom($drive)) {
1843         my $volid = $drive->{file};
1844         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1845             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1846         }
1847     }
1848 }
1849
1850 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1851 my $smbios1_fmt = {
1852     uuid => {
1853         type => 'string',
1854         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1855         format_description => 'UUID',
1856         description => "Set SMBIOS1 UUID.",
1857         optional => 1,
1858     },
1859     version => {
1860         type => 'string',
1861         pattern => '[A-Za-z0-9+\/]+={0,2}',
1862         format_description => 'Base64 encoded string',
1863         description => "Set SMBIOS1 version.",
1864         optional => 1,
1865     },
1866     serial => {
1867         type => 'string',
1868         pattern => '[A-Za-z0-9+\/]+={0,2}',
1869         format_description => 'Base64 encoded string',
1870         description => "Set SMBIOS1 serial number.",
1871         optional => 1,
1872     },
1873     manufacturer => {
1874         type => 'string',
1875         pattern => '[A-Za-z0-9+\/]+={0,2}',
1876         format_description => 'Base64 encoded string',
1877         description => "Set SMBIOS1 manufacturer.",
1878         optional => 1,
1879     },
1880     product => {
1881         type => 'string',
1882         pattern => '[A-Za-z0-9+\/]+={0,2}',
1883         format_description => 'Base64 encoded string',
1884         description => "Set SMBIOS1 product ID.",
1885         optional => 1,
1886     },
1887     sku => {
1888         type => 'string',
1889         pattern => '[A-Za-z0-9+\/]+={0,2}',
1890         format_description => 'Base64 encoded string',
1891         description => "Set SMBIOS1 SKU string.",
1892         optional => 1,
1893     },
1894     family => {
1895         type => 'string',
1896         pattern => '[A-Za-z0-9+\/]+={0,2}',
1897         format_description => 'Base64 encoded string',
1898         description => "Set SMBIOS1 family string.",
1899         optional => 1,
1900     },
1901     base64 => {
1902         type => 'boolean',
1903         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1904         optional => 1,
1905     },
1906 };
1907
1908 sub parse_smbios1 {
1909     my ($data) = @_;
1910
1911     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1912     warn $@ if $@;
1913     return $res;
1914 }
1915
1916 sub print_smbios1 {
1917     my ($smbios1) = @_;
1918     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1919 }
1920
1921 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1922
1923 sub parse_watchdog {
1924     my ($value) = @_;
1925
1926     return undef if !$value;
1927
1928     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1929     warn $@ if $@;
1930     return $res;
1931 }
1932
1933 sub parse_guest_agent {
1934     my ($value) = @_;
1935
1936     return {} if !defined($value->{agent});
1937
1938     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1939     warn $@ if $@;
1940
1941     # if the agent is disabled ignore the other potentially set properties
1942     return {} if !$res->{enabled};
1943     return $res;
1944 }
1945
1946 sub parse_vga {
1947     my ($value) = @_;
1948
1949     return {} if !$value;
1950     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1951     warn $@ if $@;
1952     return $res;
1953 }
1954
1955 sub parse_rng {
1956     my ($value) = @_;
1957
1958     return undef if !$value;
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1961     warn $@ if $@;
1962     return $res;
1963 }
1964
1965 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1966 sub verify_usb_device {
1967     my ($value, $noerr) = @_;
1968
1969     return $value if parse_usb_device($value);
1970
1971     return undef if $noerr;
1972
1973     die "unable to parse usb device\n";
1974 }
1975
1976 # add JSON properties for create and set function
1977 sub json_config_properties {
1978     my $prop = shift;
1979
1980     foreach my $opt (keys %$confdesc) {
1981         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1982             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1983         $prop->{$opt} = $confdesc->{$opt};
1984     }
1985
1986     return $prop;
1987 }
1988
1989 # return copy of $confdesc_cloudinit to generate documentation
1990 sub cloudinit_config_properties {
1991
1992     return dclone($confdesc_cloudinit);
1993 }
1994
1995 sub check_type {
1996     my ($key, $value) = @_;
1997
1998     die "unknown setting '$key'\n" if !$confdesc->{$key};
1999
2000     my $type = $confdesc->{$key}->{type};
2001
2002     if (!defined($value)) {
2003         die "got undefined value\n";
2004     }
2005
2006     if ($value =~ m/[\n\r]/) {
2007         die "property contains a line feed\n";
2008     }
2009
2010     if ($type eq 'boolean') {
2011         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2012         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2013         die "type check ('boolean') failed - got '$value'\n";
2014     } elsif ($type eq 'integer') {
2015         return int($1) if $value =~ m/^(\d+)$/;
2016         die "type check ('integer') failed - got '$value'\n";
2017     } elsif ($type eq 'number') {
2018         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2019         die "type check ('number') failed - got '$value'\n";
2020     } elsif ($type eq 'string') {
2021         if (my $fmt = $confdesc->{$key}->{format}) {
2022             PVE::JSONSchema::check_format($fmt, $value);
2023             return $value;
2024         }
2025         $value =~ s/^\"(.*)\"$/$1/;
2026         return $value;
2027     } else {
2028         die "internal error"
2029     }
2030 }
2031
2032 sub destroy_vm {
2033     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2034
2035     my $conf = PVE::QemuConfig->load_config($vmid);
2036
2037     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2038
2039     if ($conf->{template}) {
2040         # check if any base image is still used by a linked clone
2041         PVE::QemuConfig->foreach_volume($conf, sub {
2042                 my ($ds, $drive) = @_;
2043                 return if drive_is_cdrom($drive);
2044
2045                 my $volid = $drive->{file};
2046                 return if !$volid || $volid =~ m|^/|;
2047
2048                 die "base volume '$volid' is still in use by linked cloned\n"
2049                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2050
2051         });
2052     }
2053
2054     # only remove disks owned by this VM
2055     PVE::QemuConfig->foreach_volume($conf, sub {
2056         my ($ds, $drive) = @_;
2057         return if drive_is_cdrom($drive, 1);
2058
2059         my $volid = $drive->{file};
2060         return if !$volid || $volid =~ m|^/|;
2061
2062         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2063         return if !$path || !$owner || ($owner != $vmid);
2064
2065         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2066         warn "Could not remove disk '$volid', check manually: $@" if $@;
2067     });
2068
2069     # also remove unused disk
2070     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2071     PVE::Storage::foreach_volid($vmdisks, sub {
2072         my ($volid, $sid, $volname, $d) = @_;
2073         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2074         warn $@ if $@;
2075     });
2076
2077     if (defined $replacement_conf) {
2078         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2079     } else {
2080         PVE::QemuConfig->destroy_config($vmid);
2081     }
2082 }
2083
2084 sub parse_vm_config {
2085     my ($filename, $raw) = @_;
2086
2087     return undef if !defined($raw);
2088
2089     my $res = {
2090         digest => Digest::SHA::sha1_hex($raw),
2091         snapshots => {},
2092         pending => {},
2093     };
2094
2095     $filename =~ m|/qemu-server/(\d+)\.conf$|
2096         || die "got strange filename '$filename'";
2097
2098     my $vmid = $1;
2099
2100     my $conf = $res;
2101     my $descr;
2102     my $section = '';
2103
2104     my @lines = split(/\n/, $raw);
2105     foreach my $line (@lines) {
2106         next if $line =~ m/^\s*$/;
2107
2108         if ($line =~ m/^\[PENDING\]\s*$/i) {
2109             $section = 'pending';
2110             if (defined($descr)) {
2111                 $descr =~ s/\s+$//;
2112                 $conf->{description} = $descr;
2113             }
2114             $descr = undef;
2115             $conf = $res->{$section} = {};
2116             next;
2117
2118         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2119             $section = $1;
2120             if (defined($descr)) {
2121                 $descr =~ s/\s+$//;
2122                 $conf->{description} = $descr;
2123             }
2124             $descr = undef;
2125             $conf = $res->{snapshots}->{$section} = {};
2126             next;
2127         }
2128
2129         if ($line =~ m/^\#(.*)\s*$/) {
2130             $descr = '' if !defined($descr);
2131             $descr .= PVE::Tools::decode_text($1) . "\n";
2132             next;
2133         }
2134
2135         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2136             $descr = '' if !defined($descr);
2137             $descr .= PVE::Tools::decode_text($2);
2138         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2139             $conf->{snapstate} = $1;
2140         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2141             my $key = $1;
2142             my $value = $2;
2143             $conf->{$key} = $value;
2144         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2145             my $value = $1;
2146             if ($section eq 'pending') {
2147                 $conf->{delete} = $value; # we parse this later
2148             } else {
2149                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2150             }
2151         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2152             my $key = $1;
2153             my $value = $2;
2154             eval { $value = check_type($key, $value); };
2155             if ($@) {
2156                 warn "vm $vmid - unable to parse value of '$key' - $@";
2157             } else {
2158                 $key = 'ide2' if $key eq 'cdrom';
2159                 my $fmt = $confdesc->{$key}->{format};
2160                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2161                     my $v = parse_drive($key, $value);
2162                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2163                         $v->{file} = $volid;
2164                         $value = print_drive($v);
2165                     } else {
2166                         warn "vm $vmid - unable to parse value of '$key'\n";
2167                         next;
2168                     }
2169                 }
2170
2171                 $conf->{$key} = $value;
2172             }
2173         }
2174     }
2175
2176     if (defined($descr)) {
2177         $descr =~ s/\s+$//;
2178         $conf->{description} = $descr;
2179     }
2180     delete $res->{snapstate}; # just to be sure
2181
2182     return $res;
2183 }
2184
2185 sub write_vm_config {
2186     my ($filename, $conf) = @_;
2187
2188     delete $conf->{snapstate}; # just to be sure
2189
2190     if ($conf->{cdrom}) {
2191         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2192         $conf->{ide2} = $conf->{cdrom};
2193         delete $conf->{cdrom};
2194     }
2195
2196     # we do not use 'smp' any longer
2197     if ($conf->{sockets}) {
2198         delete $conf->{smp};
2199     } elsif ($conf->{smp}) {
2200         $conf->{sockets} = $conf->{smp};
2201         delete $conf->{cores};
2202         delete $conf->{smp};
2203     }
2204
2205     my $used_volids = {};
2206
2207     my $cleanup_config = sub {
2208         my ($cref, $pending, $snapname) = @_;
2209
2210         foreach my $key (keys %$cref) {
2211             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2212                 $key eq 'snapstate' || $key eq 'pending';
2213             my $value = $cref->{$key};
2214             if ($key eq 'delete') {
2215                 die "propertry 'delete' is only allowed in [PENDING]\n"
2216                     if !$pending;
2217                 # fixme: check syntax?
2218                 next;
2219             }
2220             eval { $value = check_type($key, $value); };
2221             die "unable to parse value of '$key' - $@" if $@;
2222
2223             $cref->{$key} = $value;
2224
2225             if (!$snapname && is_valid_drivename($key)) {
2226                 my $drive = parse_drive($key, $value);
2227                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2228             }
2229         }
2230     };
2231
2232     &$cleanup_config($conf);
2233
2234     &$cleanup_config($conf->{pending}, 1);
2235
2236     foreach my $snapname (keys %{$conf->{snapshots}}) {
2237         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2238         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2239     }
2240
2241     # remove 'unusedX' settings if we re-add a volume
2242     foreach my $key (keys %$conf) {
2243         my $value = $conf->{$key};
2244         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2245             delete $conf->{$key};
2246         }
2247     }
2248
2249     my $generate_raw_config = sub {
2250         my ($conf, $pending) = @_;
2251
2252         my $raw = '';
2253
2254         # add description as comment to top of file
2255         if (defined(my $descr = $conf->{description})) {
2256             if ($descr) {
2257                 foreach my $cl (split(/\n/, $descr)) {
2258                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2259                 }
2260             } else {
2261                 $raw .= "#\n" if $pending;
2262             }
2263         }
2264
2265         foreach my $key (sort keys %$conf) {
2266             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2267             $raw .= "$key: $conf->{$key}\n";
2268         }
2269         return $raw;
2270     };
2271
2272     my $raw = &$generate_raw_config($conf);
2273
2274     if (scalar(keys %{$conf->{pending}})){
2275         $raw .= "\n[PENDING]\n";
2276         $raw .= &$generate_raw_config($conf->{pending}, 1);
2277     }
2278
2279     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2280         $raw .= "\n[$snapname]\n";
2281         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2282     }
2283
2284     return $raw;
2285 }
2286
2287 sub load_defaults {
2288
2289     my $res = {};
2290
2291     # we use static defaults from our JSON schema configuration
2292     foreach my $key (keys %$confdesc) {
2293         if (defined(my $default = $confdesc->{$key}->{default})) {
2294             $res->{$key} = $default;
2295         }
2296     }
2297
2298     return $res;
2299 }
2300
2301 sub config_list {
2302     my $vmlist = PVE::Cluster::get_vmlist();
2303     my $res = {};
2304     return $res if !$vmlist || !$vmlist->{ids};
2305     my $ids = $vmlist->{ids};
2306     my $nodename = nodename();
2307
2308     foreach my $vmid (keys %$ids) {
2309         my $d = $ids->{$vmid};
2310         next if !$d->{node} || $d->{node} ne $nodename;
2311         next if !$d->{type} || $d->{type} ne 'qemu';
2312         $res->{$vmid}->{exists} = 1;
2313     }
2314     return $res;
2315 }
2316
2317 # test if VM uses local resources (to prevent migration)
2318 sub check_local_resources {
2319     my ($conf, $noerr) = @_;
2320
2321     my @loc_res = ();
2322
2323     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2324     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2325
2326     push @loc_res, "ivshmem" if $conf->{ivshmem};
2327
2328     foreach my $k (keys %$conf) {
2329         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2330         # sockets are safe: they will recreated be on the target side post-migrate
2331         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2332         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2333     }
2334
2335     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2336
2337     return \@loc_res;
2338 }
2339
2340 # check if used storages are available on all nodes (use by migrate)
2341 sub check_storage_availability {
2342     my ($storecfg, $conf, $node) = @_;
2343
2344     PVE::QemuConfig->foreach_volume($conf, sub {
2345         my ($ds, $drive) = @_;
2346
2347         my $volid = $drive->{file};
2348         return if !$volid;
2349
2350         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2351         return if !$sid;
2352
2353         # check if storage is available on both nodes
2354         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2355         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2356    });
2357 }
2358
2359 # list nodes where all VM images are available (used by has_feature API)
2360 sub shared_nodes {
2361     my ($conf, $storecfg) = @_;
2362
2363     my $nodelist = PVE::Cluster::get_nodelist();
2364     my $nodehash = { map { $_ => 1 } @$nodelist };
2365     my $nodename = nodename();
2366
2367     PVE::QemuConfig->foreach_volume($conf, sub {
2368         my ($ds, $drive) = @_;
2369
2370         my $volid = $drive->{file};
2371         return if !$volid;
2372
2373         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2374         if ($storeid) {
2375             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2376             if ($scfg->{disable}) {
2377                 $nodehash = {};
2378             } elsif (my $avail = $scfg->{nodes}) {
2379                 foreach my $node (keys %$nodehash) {
2380                     delete $nodehash->{$node} if !$avail->{$node};
2381                 }
2382             } elsif (!$scfg->{shared}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     delete $nodehash->{$node} if $node ne $nodename
2385                 }
2386             }
2387         }
2388     });
2389
2390     return $nodehash
2391 }
2392
2393 sub check_local_storage_availability {
2394     my ($conf, $storecfg) = @_;
2395
2396     my $nodelist = PVE::Cluster::get_nodelist();
2397     my $nodehash = { map { $_ => {} } @$nodelist };
2398
2399     PVE::QemuConfig->foreach_volume($conf, sub {
2400         my ($ds, $drive) = @_;
2401
2402         my $volid = $drive->{file};
2403         return if !$volid;
2404
2405         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2406         if ($storeid) {
2407             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2408
2409             if ($scfg->{disable}) {
2410                 foreach my $node (keys %$nodehash) {
2411                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2412                 }
2413             } elsif (my $avail = $scfg->{nodes}) {
2414                 foreach my $node (keys %$nodehash) {
2415                     if (!$avail->{$node}) {
2416                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2417                     }
2418                 }
2419             }
2420         }
2421     });
2422
2423     foreach my $node (values %$nodehash) {
2424         if (my $unavail = $node->{unavailable_storages}) {
2425             $node->{unavailable_storages} = [ sort keys %$unavail ];
2426         }
2427     }
2428
2429     return $nodehash
2430 }
2431
2432 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2433 sub check_running {
2434     my ($vmid, $nocheck, $node) = @_;
2435
2436     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2437     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2438 }
2439
2440 sub vzlist {
2441
2442     my $vzlist = config_list();
2443
2444     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2445
2446     while (defined(my $de = $fd->read)) {
2447         next if $de !~ m/^(\d+)\.pid$/;
2448         my $vmid = $1;
2449         next if !defined($vzlist->{$vmid});
2450         if (my $pid = check_running($vmid)) {
2451             $vzlist->{$vmid}->{pid} = $pid;
2452         }
2453     }
2454
2455     return $vzlist;
2456 }
2457
2458 our $vmstatus_return_properties = {
2459     vmid => get_standard_option('pve-vmid'),
2460     status => {
2461         description => "Qemu process status.",
2462         type => 'string',
2463         enum => ['stopped', 'running'],
2464     },
2465     maxmem => {
2466         description => "Maximum memory in bytes.",
2467         type => 'integer',
2468         optional => 1,
2469         renderer => 'bytes',
2470     },
2471     maxdisk => {
2472         description => "Root disk size in bytes.",
2473         type => 'integer',
2474         optional => 1,
2475         renderer => 'bytes',
2476     },
2477     name => {
2478         description => "VM name.",
2479         type => 'string',
2480         optional => 1,
2481     },
2482     qmpstatus => {
2483         description => "Qemu QMP agent status.",
2484         type => 'string',
2485         optional => 1,
2486     },
2487     pid => {
2488         description => "PID of running qemu process.",
2489         type => 'integer',
2490         optional => 1,
2491     },
2492     uptime => {
2493         description => "Uptime.",
2494         type => 'integer',
2495         optional => 1,
2496         renderer => 'duration',
2497     },
2498     cpus => {
2499         description => "Maximum usable CPUs.",
2500         type => 'number',
2501         optional => 1,
2502     },
2503     lock => {
2504         description => "The current config lock, if any.",
2505         type => 'string',
2506         optional => 1,
2507     },
2508     tags => {
2509         description  => "The current configured tags, if any",
2510         type => 'string',
2511         optional => 1,
2512     },
2513 };
2514
2515 my $last_proc_pid_stat;
2516
2517 # get VM status information
2518 # This must be fast and should not block ($full == false)
2519 # We only query KVM using QMP if $full == true (this can be slow)
2520 sub vmstatus {
2521     my ($opt_vmid, $full) = @_;
2522
2523     my $res = {};
2524
2525     my $storecfg = PVE::Storage::config();
2526
2527     my $list = vzlist();
2528     my $defaults = load_defaults();
2529
2530     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2531
2532     my $cpucount = $cpuinfo->{cpus} || 1;
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536
2537         my $conf = PVE::QemuConfig->load_config($vmid);
2538
2539         my $d = { vmid => $vmid };
2540         $d->{pid} = $list->{$vmid}->{pid};
2541
2542         # fixme: better status?
2543         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2544
2545         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2546         if (defined($size)) {
2547             $d->{disk} = 0; # no info available
2548             $d->{maxdisk} = $size;
2549         } else {
2550             $d->{disk} = 0;
2551             $d->{maxdisk} = 0;
2552         }
2553
2554         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2555             * ($conf->{cores} || $defaults->{cores});
2556         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2557         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2558
2559         $d->{name} = $conf->{name} || "VM $vmid";
2560         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2561             : $defaults->{memory}*(1024*1024);
2562
2563         if ($conf->{balloon}) {
2564             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2565             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2566                 : $defaults->{shares};
2567         }
2568
2569         $d->{uptime} = 0;
2570         $d->{cpu} = 0;
2571         $d->{mem} = 0;
2572
2573         $d->{netout} = 0;
2574         $d->{netin} = 0;
2575
2576         $d->{diskread} = 0;
2577         $d->{diskwrite} = 0;
2578
2579         $d->{template} = PVE::QemuConfig->is_template($conf);
2580
2581         $d->{serial} = 1 if conf_has_serial($conf);
2582         $d->{lock} = $conf->{lock} if $conf->{lock};
2583         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2584
2585         $res->{$vmid} = $d;
2586     }
2587
2588     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2589     foreach my $dev (keys %$netdev) {
2590         next if $dev !~ m/^tap([1-9]\d*)i/;
2591         my $vmid = $1;
2592         my $d = $res->{$vmid};
2593         next if !$d;
2594
2595         $d->{netout} += $netdev->{$dev}->{receive};
2596         $d->{netin} += $netdev->{$dev}->{transmit};
2597
2598         if ($full) {
2599             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2600             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2601         }
2602
2603     }
2604
2605     my $ctime = gettimeofday;
2606
2607     foreach my $vmid (keys %$list) {
2608
2609         my $d = $res->{$vmid};
2610         my $pid = $d->{pid};
2611         next if !$pid;
2612
2613         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2614         next if !$pstat; # not running
2615
2616         my $used = $pstat->{utime} + $pstat->{stime};
2617
2618         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2619
2620         if ($pstat->{vsize}) {
2621             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2622         }
2623
2624         my $old = $last_proc_pid_stat->{$pid};
2625         if (!$old) {
2626             $last_proc_pid_stat->{$pid} = {
2627                 time => $ctime,
2628                 used => $used,
2629                 cpu => 0,
2630             };
2631             next;
2632         }
2633
2634         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2635
2636         if ($dtime > 1000) {
2637             my $dutime = $used -  $old->{used};
2638
2639             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2640             $last_proc_pid_stat->{$pid} = {
2641                 time => $ctime,
2642                 used => $used,
2643                 cpu => $d->{cpu},
2644             };
2645         } else {
2646             $d->{cpu} = $old->{cpu};
2647         }
2648     }
2649
2650     return $res if !$full;
2651
2652     my $qmpclient = PVE::QMPClient->new();
2653
2654     my $ballooncb = sub {
2655         my ($vmid, $resp) = @_;
2656
2657         my $info = $resp->{'return'};
2658         return if !$info->{max_mem};
2659
2660         my $d = $res->{$vmid};
2661
2662         # use memory assigned to VM
2663         $d->{maxmem} = $info->{max_mem};
2664         $d->{balloon} = $info->{actual};
2665
2666         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2667             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2668             $d->{freemem} = $info->{free_mem};
2669         }
2670
2671         $d->{ballooninfo} = $info;
2672     };
2673
2674     my $blockstatscb = sub {
2675         my ($vmid, $resp) = @_;
2676         my $data = $resp->{'return'} || [];
2677         my $totalrdbytes = 0;
2678         my $totalwrbytes = 0;
2679
2680         for my $blockstat (@$data) {
2681             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2682             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2683
2684             $blockstat->{device} =~ s/drive-//;
2685             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2686         }
2687         $res->{$vmid}->{diskread} = $totalrdbytes;
2688         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2689     };
2690
2691     my $statuscb = sub {
2692         my ($vmid, $resp) = @_;
2693
2694         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2695         # this fails if ballon driver is not loaded, so this must be
2696         # the last commnand (following command are aborted if this fails).
2697         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2698
2699         my $status = 'unknown';
2700         if (!defined($status = $resp->{'return'}->{status})) {
2701             warn "unable to get VM status\n";
2702             return;
2703         }
2704
2705         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2706     };
2707
2708     foreach my $vmid (keys %$list) {
2709         next if $opt_vmid && ($vmid ne $opt_vmid);
2710         next if !$res->{$vmid}->{pid}; # not running
2711         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2712     }
2713
2714     $qmpclient->queue_execute(undef, 2);
2715
2716     foreach my $vmid (keys %$list) {
2717         next if $opt_vmid && ($vmid ne $opt_vmid);
2718         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2719     }
2720
2721     return $res;
2722 }
2723
2724 sub conf_has_serial {
2725     my ($conf) = @_;
2726
2727     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2728         if ($conf->{"serial$i"}) {
2729             return 1;
2730         }
2731     }
2732
2733     return 0;
2734 }
2735
2736 sub conf_has_audio {
2737     my ($conf, $id) = @_;
2738
2739     $id //= 0;
2740     my $audio = $conf->{"audio$id"};
2741     return undef if !defined($audio);
2742
2743     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2744     my $audiodriver = $audioproperties->{driver} // 'spice';
2745
2746     return {
2747         dev => $audioproperties->{device},
2748         dev_id => "audiodev$id",
2749         backend => $audiodriver,
2750         backend_id => "$audiodriver-backend${id}",
2751     };
2752 }
2753
2754 sub audio_devs {
2755     my ($audio, $audiopciaddr) = @_;
2756
2757     my $devs = [];
2758
2759     my $id = $audio->{dev_id};
2760     my $audiodev = "audiodev=$audio->{backend_id}";
2761
2762     if ($audio->{dev} eq 'AC97') {
2763         push @$devs, '-device', "AC97,id=${id}${audiopciaddr},$audiodev";
2764     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2765         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2766         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0,$audiodev";
2767         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1,$audiodev";
2768     } else {
2769         die "unkown audio device '$audio->{dev}', implement me!";
2770     }
2771
2772     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2773
2774     return $devs;
2775 }
2776
2777 sub vga_conf_has_spice {
2778     my ($vga) = @_;
2779
2780     my $vgaconf = parse_vga($vga);
2781     my $vgatype = $vgaconf->{type};
2782     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2783
2784     return $1 || 1;
2785 }
2786
2787 sub is_native($) {
2788     my ($arch) = @_;
2789     return get_host_arch() eq $arch;
2790 }
2791
2792 sub get_vm_arch {
2793     my ($conf) = @_;
2794     return $conf->{arch} // get_host_arch();
2795 }
2796
2797 my $default_machines = {
2798     x86_64 => 'pc',
2799     aarch64 => 'virt',
2800 };
2801
2802 sub get_vm_machine {
2803     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2804
2805     my $machine = $forcemachine || $conf->{machine};
2806
2807     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2808         $arch //= 'x86_64';
2809         $machine ||= $default_machines->{$arch};
2810         if ($add_pve_version) {
2811             $kvmversion //= kvm_user_version();
2812             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2813             $machine .= "+pve$pvever";
2814         }
2815     }
2816
2817     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2818         # for version-pinned machines that do not include a pve-version (e.g.
2819         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2820         $machine .= '+pve0';
2821     }
2822
2823     return $machine;
2824 }
2825
2826 sub get_ovmf_files($) {
2827     my ($arch) = @_;
2828
2829     my $ovmf = $OVMF->{$arch}
2830         or die "no OVMF images known for architecture '$arch'\n";
2831
2832     return @$ovmf;
2833 }
2834
2835 my $Arch2Qemu = {
2836     aarch64 => '/usr/bin/qemu-system-aarch64',
2837     x86_64 => '/usr/bin/qemu-system-x86_64',
2838 };
2839 sub get_command_for_arch($) {
2840     my ($arch) = @_;
2841     return '/usr/bin/kvm' if is_native($arch);
2842
2843     my $cmd = $Arch2Qemu->{$arch}
2844         or die "don't know how to emulate architecture '$arch'\n";
2845     return $cmd;
2846 }
2847
2848 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2849 # to use in a QEMU command line (-cpu element), first array_intersect the result
2850 # of query_supported_ with query_understood_. This is necessary because:
2851 #
2852 # a) query_understood_ returns flags the host cannot use and
2853 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2854 #    flags, but CPU settings - with most of them being flags. Those settings
2855 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2856 #
2857 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2858 # expensive. If you need the value returned from this, you can get it much
2859 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2860 # $accel being 'kvm' or 'tcg'.
2861 #
2862 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2863 # changes, automatically populating pmxcfs.
2864 #
2865 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2866 # since kvm and tcg machines support different flags
2867 #
2868 sub query_supported_cpu_flags {
2869     my ($arch) = @_;
2870
2871     $arch //= get_host_arch();
2872     my $default_machine = $default_machines->{$arch};
2873
2874     my $flags = {};
2875
2876     # FIXME: Once this is merged, the code below should work for ARM as well:
2877     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2878     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2879         $arch eq "aarch64";
2880
2881     my $kvm_supported = defined(kvm_version());
2882     my $qemu_cmd = get_command_for_arch($arch);
2883     my $fakevmid = -1;
2884     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2885
2886     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2887     my $query_supported_run_qemu = sub {
2888         my ($kvm) = @_;
2889
2890         my $flags = {};
2891         my $cmd = [
2892             $qemu_cmd,
2893             '-machine', $default_machine,
2894             '-display', 'none',
2895             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2896             '-mon', 'chardev=qmp,mode=control',
2897             '-pidfile', $pidfile,
2898             '-S', '-daemonize'
2899         ];
2900
2901         if (!$kvm) {
2902             push @$cmd, '-accel', 'tcg';
2903         }
2904
2905         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2906         die "QEMU flag querying VM exited with code " . $rc if $rc;
2907
2908         eval {
2909             my $cmd_result = mon_cmd(
2910                 $fakevmid,
2911                 'query-cpu-model-expansion',
2912                 type => 'full',
2913                 model => { name => 'host' }
2914             );
2915
2916             my $props = $cmd_result->{model}->{props};
2917             foreach my $prop (keys %$props) {
2918                 next if $props->{$prop} ne '1';
2919                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2920                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2921                 # We only keep those with underscores, to match /proc/cpuinfo
2922                 $prop =~ s/\.|-/_/g;
2923                 $flags->{$prop} = 1;
2924             }
2925         };
2926         my $err = $@;
2927
2928         # force stop with 10 sec timeout and 'nocheck'
2929         # always stop, even if QMP failed
2930         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2931
2932         die $err if $err;
2933
2934         return [ sort keys %$flags ];
2935     };
2936
2937     # We need to query QEMU twice, since KVM and TCG have different supported flags
2938     PVE::QemuConfig->lock_config($fakevmid, sub {
2939         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2940         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2941
2942         if ($kvm_supported) {
2943             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2944             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2945         }
2946     });
2947
2948     return $flags;
2949 }
2950
2951 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2952 my $understood_cpu_flag_dir = "/usr/share/kvm";
2953 sub query_understood_cpu_flags {
2954     my $arch = get_host_arch();
2955     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2956
2957     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2958         if ! -e $filepath;
2959
2960     my $raw = file_get_contents($filepath);
2961     $raw =~ s/^\s+|\s+$//g;
2962     my @flags = split(/\s+/, $raw);
2963
2964     return \@flags;
2965 }
2966
2967 sub config_to_command {
2968     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2969
2970     my $cmd = [];
2971     my $globalFlags = [];
2972     my $machineFlags = [];
2973     my $rtcFlags = [];
2974     my $devices = [];
2975     my $pciaddr = '';
2976     my $bridges = {};
2977     my $ostype = $conf->{ostype};
2978     my $winversion = windows_version($ostype);
2979     my $kvm = $conf->{kvm};
2980     my $nodename = nodename();
2981
2982     my $arch = get_vm_arch($conf);
2983     my $kvm_binary = get_command_for_arch($arch);
2984     my $kvmver = kvm_user_version($kvm_binary);
2985
2986     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2987         $kvmver //= "undefined";
2988         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2989     }
2990
2991     my $add_pve_version = min_version($kvmver, 4, 1);
2992
2993     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2994     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2995     $kvm //= 1 if is_native($arch);
2996
2997     $machine_version =~ m/(\d+)\.(\d+)/;
2998     my ($machine_major, $machine_minor) = ($1, $2);
2999
3000     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3001         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3002     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3003         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3004     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3005         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3006         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3007             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3008     }
3009
3010     # if a specific +pve version is required for a feature, use $version_guard
3011     # instead of min_version to allow machines to be run with the minimum
3012     # required version
3013     my $required_pve_version = 0;
3014     my $version_guard = sub {
3015         my ($major, $minor, $pve) = @_;
3016         return 0 if !min_version($machine_version, $major, $minor, $pve);
3017         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3018         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3019         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3020         return 1;
3021     };
3022
3023     if ($kvm) {
3024         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3025             if !defined kvm_version();
3026     }
3027
3028     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3029     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3030     my $use_old_bios_files = undef;
3031     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3032
3033     my $cpuunits = defined($conf->{cpuunits}) ?
3034             $conf->{cpuunits} : $defaults->{cpuunits};
3035
3036     push @$cmd, $kvm_binary;
3037
3038     push @$cmd, '-id', $vmid;
3039
3040     my $vmname = $conf->{name} || "vm$vmid";
3041
3042     push @$cmd, '-name', $vmname;
3043
3044     my $use_virtio = 0;
3045
3046     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3047     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3048     push @$cmd, '-mon', "chardev=qmp,mode=control";
3049
3050     if (min_version($machine_version, 2, 12)) {
3051         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3052         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3053     }
3054
3055     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3056
3057     push @$cmd, '-daemonize';
3058
3059     if ($conf->{smbios1}) {
3060         my $smbios_conf = parse_smbios1($conf->{smbios1});
3061         if ($smbios_conf->{base64}) {
3062             # Do not pass base64 flag to qemu
3063             delete $smbios_conf->{base64};
3064             my $smbios_string = "";
3065             foreach my $key (keys %$smbios_conf) {
3066                 my $value;
3067                 if ($key eq "uuid") {
3068                     $value = $smbios_conf->{uuid}
3069                 } else {
3070                     $value = decode_base64($smbios_conf->{$key});
3071                 }
3072                 # qemu accepts any binary data, only commas need escaping by double comma
3073                 $value =~ s/,/,,/g;
3074                 $smbios_string .= "," . $key . "=" . $value if $value;
3075             }
3076             push @$cmd, '-smbios', "type=1" . $smbios_string;
3077         } else {
3078             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3079         }
3080     }
3081
3082     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3083         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3084         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3085
3086         my ($path, $format);
3087         if (my $efidisk = $conf->{efidisk0}) {
3088             my $d = parse_drive('efidisk0', $efidisk);
3089             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3090             $format = $d->{format};
3091             if ($storeid) {
3092                 $path = PVE::Storage::path($storecfg, $d->{file});
3093                 if (!defined($format)) {
3094                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3095                     $format = qemu_img_format($scfg, $volname);
3096                 }
3097             } else {
3098                 $path = $d->{file};
3099                 die "efidisk format must be specified\n"
3100                     if !defined($format);
3101             }
3102         } else {
3103             warn "no efidisk configured! Using temporary efivars disk.\n";
3104             $path = "/tmp/$vmid-ovmf.fd";
3105             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3106             $format = 'raw';
3107         }
3108
3109         my $size_str = "";
3110
3111         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3112             $size_str = ",size=" . (-s $ovmf_vars);
3113         }
3114
3115         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3116         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3117     }
3118
3119     # load q35 config
3120     if ($q35) {
3121         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3122         if (min_version($machine_version, 4, 0)) {
3123             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3124         } else {
3125             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3126         }
3127     }
3128
3129     if ($conf->{vmgenid}) {
3130         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3131     }
3132
3133     # add usb controllers
3134     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3135     push @$devices, @usbcontrollers if @usbcontrollers;
3136     my $vga = parse_vga($conf->{vga});
3137
3138     my $qxlnum = vga_conf_has_spice($conf->{vga});
3139     $vga->{type} = 'qxl' if $qxlnum;
3140
3141     if (!$vga->{type}) {
3142         if ($arch eq 'aarch64') {
3143             $vga->{type} = 'virtio';
3144         } elsif (min_version($machine_version, 2, 9)) {
3145             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3146         } else {
3147             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3148         }
3149     }
3150
3151     # enable absolute mouse coordinates (needed by vnc)
3152     my $tablet;
3153     if (defined($conf->{tablet})) {
3154         $tablet = $conf->{tablet};
3155     } else {
3156         $tablet = $defaults->{tablet};
3157         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3158         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3159     }
3160
3161     if ($tablet) {
3162         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3163         my $kbd = print_keyboarddevice_full($conf, $arch);
3164         push @$devices, '-device', $kbd if defined($kbd);
3165     }
3166
3167     my $kvm_off = 0;
3168     my $gpu_passthrough;
3169
3170     # host pci devices
3171     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3172         my $id = "hostpci$i";
3173         my $d = parse_hostpci($conf->{$id});
3174         next if !$d;
3175
3176         if (my $pcie = $d->{pcie}) {
3177             die "q35 machine model is not enabled" if !$q35;
3178             # win7 wants to have the pcie devices directly on the pcie bus
3179             # instead of in the root port
3180             if ($winversion == 7) {
3181                 $pciaddr = print_pcie_addr("${id}bus0");
3182             } else {
3183                 # add more root ports if needed, 4 are present by default
3184                 # by pve-q35 cfgs, rest added here on demand.
3185                 if ($i > 3) {
3186                     push @$devices, '-device', print_pcie_root_port($i);
3187                 }
3188                 $pciaddr = print_pcie_addr($id);
3189             }
3190         } else {
3191             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3192         }
3193
3194         my $xvga = '';
3195         if ($d->{'x-vga'}) {
3196             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3197             $kvm_off = 1;
3198             $vga->{type} = 'none' if !defined($conf->{vga});
3199             $gpu_passthrough = 1;
3200         }
3201
3202         my $pcidevices = $d->{pciid};
3203         my $multifunction = 1 if @$pcidevices > 1;
3204
3205         my $sysfspath;
3206         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3207             my $pci_id = $pcidevices->[0]->{id};
3208             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3209             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3210         } elsif ($d->{mdev}) {
3211             warn "ignoring mediated device '$id' with multifunction device\n";
3212         }
3213
3214         my $j=0;
3215         foreach my $pcidevice (@$pcidevices) {
3216             my $devicestr = "vfio-pci";
3217
3218             if ($sysfspath) {
3219                 $devicestr .= ",sysfsdev=$sysfspath";
3220             } else {
3221                 $devicestr .= ",host=$pcidevice->{id}";
3222             }
3223
3224             my $mf_addr = $multifunction ? ".$j" : '';
3225             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3226
3227             if ($j == 0) {
3228                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3229                 $devicestr .= "$xvga";
3230                 $devicestr .= ",multifunction=on" if $multifunction;
3231                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3232             }
3233
3234             push @$devices, '-device', $devicestr;
3235             $j++;
3236         }
3237     }
3238
3239     # usb devices
3240     my $usb_dev_features = {};
3241     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3242
3243     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3244     push @$devices, @usbdevices if @usbdevices;
3245     # serial devices
3246     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3247         if (my $path = $conf->{"serial$i"}) {
3248             if ($path eq 'socket') {
3249                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3250                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3251                 # On aarch64, serial0 is the UART device. Qemu only allows
3252                 # connecting UART devices via the '-serial' command line, as
3253                 # the device has a fixed slot on the hardware...
3254                 if ($arch eq 'aarch64' && $i == 0) {
3255                     push @$devices, '-serial', "chardev:serial$i";
3256                 } else {
3257                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3258                 }
3259             } else {
3260                 die "no such serial device\n" if ! -c $path;
3261                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3262                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3263             }
3264         }
3265     }
3266
3267     # parallel devices
3268     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3269         if (my $path = $conf->{"parallel$i"}) {
3270             die "no such parallel device\n" if ! -c $path;
3271             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3272             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3273             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3274         }
3275     }
3276
3277     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3278         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3279         my $audio_devs = audio_devs($audio, $audiopciaddr);
3280         push @$devices, @$audio_devs;
3281     }
3282
3283     my $sockets = 1;
3284     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3285     $sockets = $conf->{sockets} if  $conf->{sockets};
3286
3287     my $cores = $conf->{cores} || 1;
3288
3289     my $maxcpus = $sockets * $cores;
3290
3291     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3292
3293     my $allowed_vcpus = $cpuinfo->{cpus};
3294
3295     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3296         if ($allowed_vcpus < $maxcpus);
3297
3298     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3299
3300         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3301         for (my $i = 2; $i <= $vcpus; $i++)  {
3302             my $cpustr = print_cpu_device($conf,$i);
3303             push @$cmd, '-device', $cpustr;
3304         }
3305
3306     } else {
3307
3308         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3309     }
3310     push @$cmd, '-nodefaults';
3311
3312     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3313
3314     my $bootindex_hash = {};
3315     my $i = 1;
3316     foreach my $o (split(//, $bootorder)) {
3317         $bootindex_hash->{$o} = $i*100;
3318         $i++;
3319     }
3320
3321     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3322
3323     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3324
3325     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3326
3327     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3328         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3329         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3330         push @$cmd,  '-vnc', "unix:$socket,password";
3331     } else {
3332         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3333         push @$cmd, '-nographic';
3334     }
3335
3336     # time drift fix
3337     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3338     my $useLocaltime = $conf->{localtime};
3339
3340     if ($winversion >= 5) { # windows
3341         $useLocaltime = 1 if !defined($conf->{localtime});
3342
3343         # use time drift fix when acpi is enabled
3344         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3345             $tdf = 1 if !defined($conf->{tdf});
3346         }
3347     }
3348
3349     if ($winversion >= 6) {
3350         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3351         push @$cmd, '-no-hpet';
3352     }
3353
3354     push @$rtcFlags, 'driftfix=slew' if $tdf;
3355
3356     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3357         push @$rtcFlags, "base=$conf->{startdate}";
3358     } elsif ($useLocaltime) {
3359         push @$rtcFlags, 'base=localtime';
3360     }
3361
3362     if ($forcecpu) {
3363         push @$cmd, '-cpu', $forcecpu;
3364     } else {
3365         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3366     }
3367
3368     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3369
3370     push @$cmd, '-S' if $conf->{freeze};
3371
3372     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3373
3374     my $guest_agent = parse_guest_agent($conf);
3375
3376     if ($guest_agent->{enabled}) {
3377         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3378         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3379
3380         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3381             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3382             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3383             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3384         } elsif ($guest_agent->{type} eq 'isa') {
3385             push @$devices, '-device', "isa-serial,chardev=qga0";
3386         }
3387     }
3388
3389     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3390     if ($rng && &$version_guard(4, 1, 2)) {
3391         check_rng_source($rng->{source});
3392
3393         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3394         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3395         my $limiter_str = "";
3396         if ($max_bytes) {
3397             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3398         }
3399
3400         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3401         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3402         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3403     }
3404
3405     my $spice_port;
3406
3407     if ($qxlnum) {
3408         if ($qxlnum > 1) {
3409             if ($winversion){
3410                 for (my $i = 1; $i < $qxlnum; $i++){
3411                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3412                 }
3413             } else {
3414                 # assume other OS works like Linux
3415                 my ($ram, $vram) = ("134217728", "67108864");
3416                 if ($vga->{memory}) {
3417                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3418                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3419                 }
3420                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3421                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3422             }
3423         }
3424
3425         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3426
3427         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3428         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3429         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3430
3431         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3432         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3433         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3434
3435         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3436         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3437
3438         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3439         if ($spice_enhancement->{foldersharing}) {
3440             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3441             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3442         }
3443
3444         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3445         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3446         push @$devices, '-spice', "$spice_opts";
3447     }
3448
3449     # enable balloon by default, unless explicitly disabled
3450     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3451         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3452         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3453     }
3454
3455     if ($conf->{watchdog}) {
3456         my $wdopts = parse_watchdog($conf->{watchdog});
3457         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3458         my $watchdog = $wdopts->{model} || 'i6300esb';
3459         push @$devices, '-device', "$watchdog$pciaddr";
3460         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3461     }
3462
3463     my $vollist = [];
3464     my $scsicontroller = {};
3465     my $ahcicontroller = {};
3466     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3467
3468     # Add iscsi initiator name if available
3469     if (my $initiator = get_initiator_name()) {
3470         push @$devices, '-iscsi', "initiator-name=$initiator";
3471     }
3472
3473     PVE::QemuConfig->foreach_volume($conf, sub {
3474         my ($ds, $drive) = @_;
3475
3476         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3477             push @$vollist, $drive->{file};
3478         }
3479
3480         # ignore efidisk here, already added in bios/fw handling code above
3481         return if $drive->{interface} eq 'efidisk';
3482
3483         $use_virtio = 1 if $ds =~ m/^virtio/;
3484
3485         if (drive_is_cdrom ($drive)) {
3486             if ($bootindex_hash->{d}) {
3487                 $drive->{bootindex} = $bootindex_hash->{d};
3488                 $bootindex_hash->{d} += 1;
3489             }
3490         } else {
3491             if ($bootindex_hash->{c}) {
3492                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3493                 $bootindex_hash->{c} += 1;
3494             }
3495         }
3496
3497         if ($drive->{interface} eq 'virtio'){
3498            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3499         }
3500
3501         if ($drive->{interface} eq 'scsi') {
3502
3503             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3504
3505             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3506                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3507
3508             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3509             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3510
3511             my $iothread = '';
3512             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3513                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3514                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3515             } elsif ($drive->{iothread}) {
3516                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3517             }
3518
3519             my $queues = '';
3520             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3521                 $queues = ",num_queues=$drive->{queues}";
3522             }
3523
3524             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3525             $scsicontroller->{$controller}=1;
3526         }
3527
3528         if ($drive->{interface} eq 'sata') {
3529             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3530             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3531             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3532             $ahcicontroller->{$controller}=1;
3533         }
3534
3535         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3536         push @$devices, '-drive',$drive_cmd;
3537         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3538     });
3539
3540     for (my $i = 0; $i < $MAX_NETS; $i++) {
3541          next if !$conf->{"net$i"};
3542          my $d = parse_net($conf->{"net$i"});
3543          next if !$d;
3544
3545          $use_virtio = 1 if $d->{model} eq 'virtio';
3546
3547          if ($bootindex_hash->{n}) {
3548             $d->{bootindex} = $bootindex_hash->{n};
3549             $bootindex_hash->{n} += 1;
3550          }
3551
3552          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3553          push @$devices, '-netdev', $netdevfull;
3554
3555          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3556          push @$devices, '-device', $netdevicefull;
3557     }
3558
3559     if ($conf->{ivshmem}) {
3560         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3561
3562         my $bus;
3563         if ($q35) {
3564             $bus = print_pcie_addr("ivshmem");
3565         } else {
3566             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3567         }
3568
3569         my $ivshmem_name = $ivshmem->{name} // $vmid;
3570         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3571
3572         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3573         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3574     }
3575
3576     # pci.4 is nested in pci.1
3577     $bridges->{1} = 1 if $bridges->{4};
3578
3579     if (!$q35) {
3580         # add pci bridges
3581         if (min_version($machine_version, 2, 3)) {
3582            $bridges->{1} = 1;
3583            $bridges->{2} = 1;
3584         }
3585
3586         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3587
3588     }
3589
3590     for my $k (sort {$b cmp $a} keys %$bridges) {
3591         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3592         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3593         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3594         if ($q35) {
3595             # add after -readconfig pve-q35.cfg
3596             splice @$devices, 2, 0, '-device', $devstr;
3597         } else {
3598             unshift @$devices, '-device', $devstr if $k > 0;
3599         }
3600     }
3601
3602     if (!$kvm) {
3603         push @$machineFlags, 'accel=tcg';
3604     }
3605
3606     my $machine_type_min = $machine_type;
3607     if ($add_pve_version) {
3608         $machine_type_min =~ s/\+pve\d+$//;
3609         $machine_type_min .= "+pve$required_pve_version";
3610     }
3611     push @$machineFlags, "type=${machine_type_min}";
3612
3613     push @$cmd, @$devices;
3614     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3615     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3616     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3617
3618     if (my $vmstate = $conf->{vmstate}) {
3619         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3620         push @$vollist, $vmstate;
3621         push @$cmd, '-loadstate', $statepath;
3622         print "activating and using '$vmstate' as vmstate\n";
3623     }
3624
3625     # add custom args
3626     if ($conf->{args}) {
3627         my $aa = PVE::Tools::split_args($conf->{args});
3628         push @$cmd, @$aa;
3629     }
3630
3631     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3632 }
3633
3634 sub check_rng_source {
3635     my ($source) = @_;
3636
3637     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3638     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3639         if ! -e $source;
3640
3641     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3642     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3643         # Needs to abort, otherwise QEMU crashes on first rng access.
3644         # Note that rng_current cannot be changed to 'none' manually, so
3645         # once the VM is past this point, it is no longer an issue.
3646         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3647             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3648             . " a compatible hardware-RNG is attached to the host.\n";
3649     }
3650 }
3651
3652 sub spice_port {
3653     my ($vmid) = @_;
3654
3655     my $res = mon_cmd($vmid, 'query-spice');
3656
3657     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3658 }
3659
3660 sub vm_devices_list {
3661     my ($vmid) = @_;
3662
3663     my $res = mon_cmd($vmid, 'query-pci');
3664     my $devices_to_check = [];
3665     my $devices = {};
3666     foreach my $pcibus (@$res) {
3667         push @$devices_to_check, @{$pcibus->{devices}},
3668     }
3669
3670     while (@$devices_to_check) {
3671         my $to_check = [];
3672         for my $d (@$devices_to_check) {
3673             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3674             next if !$d->{'pci_bridge'};
3675
3676             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3677             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3678         }
3679         $devices_to_check = $to_check;
3680     }
3681
3682     my $resblock = mon_cmd($vmid, 'query-block');
3683     foreach my $block (@$resblock) {
3684         if($block->{device} =~ m/^drive-(\S+)/){
3685                 $devices->{$1} = 1;
3686         }
3687     }
3688
3689     my $resmice = mon_cmd($vmid, 'query-mice');
3690     foreach my $mice (@$resmice) {
3691         if ($mice->{name} eq 'QEMU HID Tablet') {
3692             $devices->{tablet} = 1;
3693             last;
3694         }
3695     }
3696
3697     # for usb devices there is no query-usb
3698     # but we can iterate over the entries in
3699     # qom-list path=/machine/peripheral
3700     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3701     foreach my $per (@$resperipheral) {
3702         if ($per->{name} =~ m/^usb\d+$/) {
3703             $devices->{$per->{name}} = 1;
3704         }
3705     }
3706
3707     return $devices;
3708 }
3709
3710 sub vm_deviceplug {
3711     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3712
3713     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3714
3715     my $devices_list = vm_devices_list($vmid);
3716     return 1 if defined($devices_list->{$deviceid});
3717
3718     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3719
3720     if ($deviceid eq 'tablet') {
3721
3722         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3723
3724     } elsif ($deviceid eq 'keyboard') {
3725
3726         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3727
3728     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3729
3730         die "usb hotplug currently not reliable\n";
3731         # since we can't reliably hot unplug all added usb devices
3732         # and usb passthrough disables live migration
3733         # we disable usb hotplugging for now
3734         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3735
3736     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3737
3738         qemu_iothread_add($vmid, $deviceid, $device);
3739
3740         qemu_driveadd($storecfg, $vmid, $device);
3741         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3742
3743         qemu_deviceadd($vmid, $devicefull);
3744         eval { qemu_deviceaddverify($vmid, $deviceid); };
3745         if (my $err = $@) {
3746             eval { qemu_drivedel($vmid, $deviceid); };
3747             warn $@ if $@;
3748             die $err;
3749         }
3750
3751     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3752
3753
3754         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3755         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3756         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3757
3758         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3759
3760         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3761             qemu_iothread_add($vmid, $deviceid, $device);
3762             $devicefull .= ",iothread=iothread-$deviceid";
3763         }
3764
3765         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3766             $devicefull .= ",num_queues=$device->{queues}";
3767         }
3768
3769         qemu_deviceadd($vmid, $devicefull);
3770         qemu_deviceaddverify($vmid, $deviceid);
3771
3772     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3773
3774         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3775         qemu_driveadd($storecfg, $vmid, $device);
3776
3777         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3778         eval { qemu_deviceadd($vmid, $devicefull); };
3779         if (my $err = $@) {
3780             eval { qemu_drivedel($vmid, $deviceid); };
3781             warn $@ if $@;
3782             die $err;
3783         }
3784
3785     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3786
3787         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3788
3789         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3790         my $use_old_bios_files = undef;
3791         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3792
3793         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3794         qemu_deviceadd($vmid, $netdevicefull);
3795         eval {
3796             qemu_deviceaddverify($vmid, $deviceid);
3797             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3798         };
3799         if (my $err = $@) {
3800             eval { qemu_netdevdel($vmid, $deviceid); };
3801             warn $@ if $@;
3802             die $err;
3803         }
3804
3805     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3806
3807         my $bridgeid = $2;
3808         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3809         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3810
3811         qemu_deviceadd($vmid, $devicefull);
3812         qemu_deviceaddverify($vmid, $deviceid);
3813
3814     } else {
3815         die "can't hotplug device '$deviceid'\n";
3816     }
3817
3818     return 1;
3819 }
3820
3821 # fixme: this should raise exceptions on error!
3822 sub vm_deviceunplug {
3823     my ($vmid, $conf, $deviceid) = @_;
3824
3825     my $devices_list = vm_devices_list($vmid);
3826     return 1 if !defined($devices_list->{$deviceid});
3827
3828     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3829
3830     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3831
3832         qemu_devicedel($vmid, $deviceid);
3833
3834     } elsif ($deviceid =~ m/^usb\d+$/) {
3835
3836         die "usb hotplug currently not reliable\n";
3837         # when unplugging usb devices this way,
3838         # there may be remaining usb controllers/hubs
3839         # so we disable it for now
3840         qemu_devicedel($vmid, $deviceid);
3841         qemu_devicedelverify($vmid, $deviceid);
3842
3843     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3844
3845         qemu_devicedel($vmid, $deviceid);
3846         qemu_devicedelverify($vmid, $deviceid);
3847         qemu_drivedel($vmid, $deviceid);
3848         qemu_iothread_del($conf, $vmid, $deviceid);
3849
3850     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3851
3852         qemu_devicedel($vmid, $deviceid);
3853         qemu_devicedelverify($vmid, $deviceid);
3854         qemu_iothread_del($conf, $vmid, $deviceid);
3855
3856     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3857
3858         qemu_devicedel($vmid, $deviceid);
3859         qemu_drivedel($vmid, $deviceid);
3860         qemu_deletescsihw($conf, $vmid, $deviceid);
3861
3862     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3863
3864         qemu_devicedel($vmid, $deviceid);
3865         qemu_devicedelverify($vmid, $deviceid);
3866         qemu_netdevdel($vmid, $deviceid);
3867
3868     } else {
3869         die "can't unplug device '$deviceid'\n";
3870     }
3871
3872     return 1;
3873 }
3874
3875 sub qemu_deviceadd {
3876     my ($vmid, $devicefull) = @_;
3877
3878     $devicefull = "driver=".$devicefull;
3879     my %options =  split(/[=,]/, $devicefull);
3880
3881     mon_cmd($vmid, "device_add" , %options);
3882 }
3883
3884 sub qemu_devicedel {
3885     my ($vmid, $deviceid) = @_;
3886
3887     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3888 }
3889
3890 sub qemu_iothread_add {
3891     my($vmid, $deviceid, $device) = @_;
3892
3893     if ($device->{iothread}) {
3894         my $iothreads = vm_iothreads_list($vmid);
3895         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3896     }
3897 }
3898
3899 sub qemu_iothread_del {
3900     my($conf, $vmid, $deviceid) = @_;
3901
3902     my $confid = $deviceid;
3903     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3904         $confid = 'scsi' . $1;
3905     }
3906     my $device = parse_drive($confid, $conf->{$confid});
3907     if ($device->{iothread}) {
3908         my $iothreads = vm_iothreads_list($vmid);
3909         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3910     }
3911 }
3912
3913 sub qemu_objectadd {
3914     my($vmid, $objectid, $qomtype) = @_;
3915
3916     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3917
3918     return 1;
3919 }
3920
3921 sub qemu_objectdel {
3922     my($vmid, $objectid) = @_;
3923
3924     mon_cmd($vmid, "object-del", id => $objectid);
3925
3926     return 1;
3927 }
3928
3929 sub qemu_driveadd {
3930     my ($storecfg, $vmid, $device) = @_;
3931
3932     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3933     $drive =~ s/\\/\\\\/g;
3934     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3935
3936     # If the command succeeds qemu prints: "OK"
3937     return 1 if $ret =~ m/OK/s;
3938
3939     die "adding drive failed: $ret\n";
3940 }
3941
3942 sub qemu_drivedel {
3943     my($vmid, $deviceid) = @_;
3944
3945     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3946     $ret =~ s/^\s+//;
3947
3948     return 1 if $ret eq "";
3949
3950     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3951     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3952
3953     die "deleting drive $deviceid failed : $ret\n";
3954 }
3955
3956 sub qemu_deviceaddverify {
3957     my ($vmid, $deviceid) = @_;
3958
3959     for (my $i = 0; $i <= 5; $i++) {
3960          my $devices_list = vm_devices_list($vmid);
3961          return 1 if defined($devices_list->{$deviceid});
3962          sleep 1;
3963     }
3964
3965     die "error on hotplug device '$deviceid'\n";
3966 }
3967
3968
3969 sub qemu_devicedelverify {
3970     my ($vmid, $deviceid) = @_;
3971
3972     # need to verify that the device is correctly removed as device_del
3973     # is async and empty return is not reliable
3974
3975     for (my $i = 0; $i <= 5; $i++) {
3976          my $devices_list = vm_devices_list($vmid);
3977          return 1 if !defined($devices_list->{$deviceid});
3978          sleep 1;
3979     }
3980
3981     die "error on hot-unplugging device '$deviceid'\n";
3982 }
3983
3984 sub qemu_findorcreatescsihw {
3985     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3986
3987     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3988
3989     my $scsihwid="$controller_prefix$controller";
3990     my $devices_list = vm_devices_list($vmid);
3991
3992     if(!defined($devices_list->{$scsihwid})) {
3993         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3994     }
3995
3996     return 1;
3997 }
3998
3999 sub qemu_deletescsihw {
4000     my ($conf, $vmid, $opt) = @_;
4001
4002     my $device = parse_drive($opt, $conf->{$opt});
4003
4004     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4005         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4006         return 1;
4007     }
4008
4009     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4010
4011     my $devices_list = vm_devices_list($vmid);
4012     foreach my $opt (keys %{$devices_list}) {
4013         if (is_valid_drivename($opt)) {
4014             my $drive = parse_drive($opt, $conf->{$opt});
4015             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4016                 return 1;
4017             }
4018         }
4019     }
4020
4021     my $scsihwid="scsihw$controller";
4022
4023     vm_deviceunplug($vmid, $conf, $scsihwid);
4024
4025     return 1;
4026 }
4027
4028 sub qemu_add_pci_bridge {
4029     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4030
4031     my $bridges = {};
4032
4033     my $bridgeid;
4034
4035     print_pci_addr($device, $bridges, $arch, $machine_type);
4036
4037     while (my ($k, $v) = each %$bridges) {
4038         $bridgeid = $k;
4039     }
4040     return 1 if !defined($bridgeid) || $bridgeid < 1;
4041
4042     my $bridge = "pci.$bridgeid";
4043     my $devices_list = vm_devices_list($vmid);
4044
4045     if (!defined($devices_list->{$bridge})) {
4046         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4047     }
4048
4049     return 1;
4050 }
4051
4052 sub qemu_set_link_status {
4053     my ($vmid, $device, $up) = @_;
4054
4055     mon_cmd($vmid, "set_link", name => $device,
4056                up => $up ? JSON::true : JSON::false);
4057 }
4058
4059 sub qemu_netdevadd {
4060     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4061
4062     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4063     my %options =  split(/[=,]/, $netdev);
4064
4065     mon_cmd($vmid, "netdev_add",  %options);
4066     return 1;
4067 }
4068
4069 sub qemu_netdevdel {
4070     my ($vmid, $deviceid) = @_;
4071
4072     mon_cmd($vmid, "netdev_del", id => $deviceid);
4073 }
4074
4075 sub qemu_usb_hotplug {
4076     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4077
4078     return if !$device;
4079
4080     # remove the old one first
4081     vm_deviceunplug($vmid, $conf, $deviceid);
4082
4083     # check if xhci controller is necessary and available
4084     if ($device->{usb3}) {
4085
4086         my $devicelist = vm_devices_list($vmid);
4087
4088         if (!$devicelist->{xhci}) {
4089             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4090             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4091         }
4092     }
4093     my $d = parse_usb_device($device->{host});
4094     $d->{usb3} = $device->{usb3};
4095
4096     # add the new one
4097     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4098 }
4099
4100 sub qemu_cpu_hotplug {
4101     my ($vmid, $conf, $vcpus) = @_;
4102
4103     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4104
4105     my $sockets = 1;
4106     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4107     $sockets = $conf->{sockets} if  $conf->{sockets};
4108     my $cores = $conf->{cores} || 1;
4109     my $maxcpus = $sockets * $cores;
4110
4111     $vcpus = $maxcpus if !$vcpus;
4112
4113     die "you can't add more vcpus than maxcpus\n"
4114         if $vcpus > $maxcpus;
4115
4116     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4117
4118     if ($vcpus < $currentvcpus) {
4119
4120         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4121
4122             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4123                 qemu_devicedel($vmid, "cpu$i");
4124                 my $retry = 0;
4125                 my $currentrunningvcpus = undef;
4126                 while (1) {
4127                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4128                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4129                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4130                     $retry++;
4131                     sleep 1;
4132                 }
4133                 #update conf after each succesfull cpu unplug
4134                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4135                 PVE::QemuConfig->write_config($vmid, $conf);
4136             }
4137         } else {
4138             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4139         }
4140
4141         return;
4142     }
4143
4144     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4145     die "vcpus in running vm does not match its configuration\n"
4146         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4147
4148     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4149
4150         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4151             my $cpustr = print_cpu_device($conf, $i);
4152             qemu_deviceadd($vmid, $cpustr);
4153
4154             my $retry = 0;
4155             my $currentrunningvcpus = undef;
4156             while (1) {
4157                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4158                 last if scalar(@{$currentrunningvcpus}) == $i;
4159                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4160                 sleep 1;
4161                 $retry++;
4162             }
4163             #update conf after each succesfull cpu hotplug
4164             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4165             PVE::QemuConfig->write_config($vmid, $conf);
4166         }
4167     } else {
4168
4169         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4170             mon_cmd($vmid, "cpu-add", id => int($i));
4171         }
4172     }
4173 }
4174
4175 sub qemu_block_set_io_throttle {
4176     my ($vmid, $deviceid,
4177         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4178         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4179         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4180         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4181
4182     return if !check_running($vmid) ;
4183
4184     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4185         bps => int($bps),
4186         bps_rd => int($bps_rd),
4187         bps_wr => int($bps_wr),
4188         iops => int($iops),
4189         iops_rd => int($iops_rd),
4190         iops_wr => int($iops_wr),
4191         bps_max => int($bps_max),
4192         bps_rd_max => int($bps_rd_max),
4193         bps_wr_max => int($bps_wr_max),
4194         iops_max => int($iops_max),
4195         iops_rd_max => int($iops_rd_max),
4196         iops_wr_max => int($iops_wr_max),
4197         bps_max_length => int($bps_max_length),
4198         bps_rd_max_length => int($bps_rd_max_length),
4199         bps_wr_max_length => int($bps_wr_max_length),
4200         iops_max_length => int($iops_max_length),
4201         iops_rd_max_length => int($iops_rd_max_length),
4202         iops_wr_max_length => int($iops_wr_max_length),
4203     );
4204
4205 }
4206
4207 # old code, only used to shutdown old VM after update
4208 sub __read_avail {
4209     my ($fh, $timeout) = @_;
4210
4211     my $sel = new IO::Select;
4212     $sel->add($fh);
4213
4214     my $res = '';
4215     my $buf;
4216
4217     my @ready;
4218     while (scalar (@ready = $sel->can_read($timeout))) {
4219         my $count;
4220         if ($count = $fh->sysread($buf, 8192)) {
4221             if ($buf =~ /^(.*)\(qemu\) $/s) {
4222                 $res .= $1;
4223                 last;
4224             } else {
4225                 $res .= $buf;
4226             }
4227         } else {
4228             if (!defined($count)) {
4229                 die "$!\n";
4230             }
4231             last;
4232         }
4233     }
4234
4235     die "monitor read timeout\n" if !scalar(@ready);
4236
4237     return $res;
4238 }
4239
4240 sub qemu_block_resize {
4241     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4242
4243     my $running = check_running($vmid);
4244
4245     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4246
4247     return if !$running;
4248
4249     my $padding = (1024 - $size % 1024) % 1024;
4250     $size = $size + $padding;
4251
4252     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4253
4254 }
4255
4256 sub qemu_volume_snapshot {
4257     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4258
4259     my $running = check_running($vmid);
4260
4261     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4262         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4263     } else {
4264         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4265     }
4266 }
4267
4268 sub qemu_volume_snapshot_delete {
4269     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4270
4271     my $running = check_running($vmid);
4272
4273     if($running) {
4274
4275         $running = undef;
4276         my $conf = PVE::QemuConfig->load_config($vmid);
4277         PVE::QemuConfig->foreach_volume($conf, sub {
4278             my ($ds, $drive) = @_;
4279             $running = 1 if $drive->{file} eq $volid;
4280         });
4281     }
4282
4283     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4284         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4285     } else {
4286         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4287     }
4288 }
4289
4290 sub set_migration_caps {
4291     my ($vmid) = @_;
4292
4293     my $cap_ref = [];
4294
4295     my $enabled_cap = {
4296         "auto-converge" => 1,
4297         "xbzrle" => 1,
4298         "x-rdma-pin-all" => 0,
4299         "zero-blocks" => 0,
4300         "compress" => 0
4301     };
4302
4303     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4304
4305     for my $supported_capability (@$supported_capabilities) {
4306         push @$cap_ref, {
4307             capability => $supported_capability->{capability},
4308             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4309         };
4310     }
4311
4312     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4313 }
4314
4315 sub foreach_volid {
4316     my ($conf, $func, @param) = @_;
4317
4318     my $volhash = {};
4319
4320     my $test_volid = sub {
4321         my ($key, $drive, $snapname) = @_;
4322
4323         my $volid = $drive->{file};
4324         return if !$volid;
4325
4326         $volhash->{$volid}->{cdrom} //= 1;
4327         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4328
4329         my $replicate = $drive->{replicate} // 1;
4330         $volhash->{$volid}->{replicate} //= 0;
4331         $volhash->{$volid}->{replicate} = 1 if $replicate;
4332
4333         $volhash->{$volid}->{shared} //= 0;
4334         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4335
4336         $volhash->{$volid}->{referenced_in_config} //= 0;
4337         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4338
4339         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4340             if defined($snapname);
4341
4342         my $size = $drive->{size};
4343         $volhash->{$volid}->{size} //= $size if $size;
4344
4345         $volhash->{$volid}->{is_vmstate} //= 0;
4346         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4347
4348         $volhash->{$volid}->{is_unused} //= 0;
4349         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4350     };
4351
4352     my $include_opts = {
4353         extra_keys => ['vmstate'],
4354         include_unused => 1,
4355     };
4356
4357     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4358     foreach my $snapname (keys %{$conf->{snapshots}}) {
4359         my $snap = $conf->{snapshots}->{$snapname};
4360         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4361     }
4362
4363     foreach my $volid (keys %$volhash) {
4364         &$func($volid, $volhash->{$volid}, @param);
4365     }
4366 }
4367
4368 my $fast_plug_option = {
4369     'lock' => 1,
4370     'name' => 1,
4371     'onboot' => 1,
4372     'shares' => 1,
4373     'startup' => 1,
4374     'description' => 1,
4375     'protection' => 1,
4376     'vmstatestorage' => 1,
4377     'hookscript' => 1,
4378     'tags' => 1,
4379 };
4380
4381 # hotplug changes in [PENDING]
4382 # $selection hash can be used to only apply specified options, for
4383 # example: { cores => 1 } (only apply changed 'cores')
4384 # $errors ref is used to return error messages
4385 sub vmconfig_hotplug_pending {
4386     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4387
4388     my $defaults = load_defaults();
4389     my $arch = get_vm_arch($conf);
4390     my $machine_type = get_vm_machine($conf, undef, $arch);
4391
4392     # commit values which do not have any impact on running VM first
4393     # Note: those option cannot raise errors, we we do not care about
4394     # $selection and always apply them.
4395
4396     my $add_error = sub {
4397         my ($opt, $msg) = @_;
4398         $errors->{$opt} = "hotplug problem - $msg";
4399     };
4400
4401     my $changes = 0;
4402     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4403         if ($fast_plug_option->{$opt}) {
4404             $conf->{$opt} = $conf->{pending}->{$opt};
4405             delete $conf->{pending}->{$opt};
4406             $changes = 1;
4407         }
4408     }
4409
4410     if ($changes) {
4411         PVE::QemuConfig->write_config($vmid, $conf);
4412     }
4413
4414     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4415
4416     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4417     foreach my $opt (sort keys %$pending_delete_hash) {
4418         next if $selection && !$selection->{$opt};
4419         my $force = $pending_delete_hash->{$opt}->{force};
4420         eval {
4421             if ($opt eq 'hotplug') {
4422                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4423             } elsif ($opt eq 'tablet') {
4424                 die "skip\n" if !$hotplug_features->{usb};
4425                 if ($defaults->{tablet}) {
4426                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4427                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4428                         if $arch eq 'aarch64';
4429                 } else {
4430                     vm_deviceunplug($vmid, $conf, 'tablet');
4431                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4432                 }
4433             } elsif ($opt =~ m/^usb\d+/) {
4434                 die "skip\n";
4435                 # since we cannot reliably hot unplug usb devices
4436                 # we are disabling it
4437                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4438                 vm_deviceunplug($vmid, $conf, $opt);
4439             } elsif ($opt eq 'vcpus') {
4440                 die "skip\n" if !$hotplug_features->{cpu};
4441                 qemu_cpu_hotplug($vmid, $conf, undef);
4442             } elsif ($opt eq 'balloon') {
4443                 # enable balloon device is not hotpluggable
4444                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4445                 # here we reset the ballooning value to memory
4446                 my $balloon = $conf->{memory} || $defaults->{memory};
4447                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4448             } elsif ($fast_plug_option->{$opt}) {
4449                 # do nothing
4450             } elsif ($opt =~ m/^net(\d+)$/) {
4451                 die "skip\n" if !$hotplug_features->{network};
4452                 vm_deviceunplug($vmid, $conf, $opt);
4453             } elsif (is_valid_drivename($opt)) {
4454                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4455                 vm_deviceunplug($vmid, $conf, $opt);
4456                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4457             } elsif ($opt =~ m/^memory$/) {
4458                 die "skip\n" if !$hotplug_features->{memory};
4459                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4460             } elsif ($opt eq 'cpuunits') {
4461                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4462             } elsif ($opt eq 'cpulimit') {
4463                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4464             } else {
4465                 die "skip\n";
4466             }
4467         };
4468         if (my $err = $@) {
4469             &$add_error($opt, $err) if $err ne "skip\n";
4470         } else {
4471             delete $conf->{$opt};
4472             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4473         }
4474     }
4475
4476     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4477     $apply_pending_cloudinit = sub {
4478         return if $apply_pending_cloudinit_done; # once is enough
4479         $apply_pending_cloudinit_done = 1; # once is enough
4480
4481         my ($key, $value) = @_;
4482
4483         my @cloudinit_opts = keys %$confdesc_cloudinit;
4484         foreach my $opt (keys %{$conf->{pending}}) {
4485             next if !grep { $_ eq $opt } @cloudinit_opts;
4486             $conf->{$opt} = delete $conf->{pending}->{$opt};
4487         }
4488
4489         my $new_conf = { %$conf };
4490         $new_conf->{$key} = $value;
4491         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4492     };
4493
4494     foreach my $opt (keys %{$conf->{pending}}) {
4495         next if $selection && !$selection->{$opt};
4496         my $value = $conf->{pending}->{$opt};
4497         eval {
4498             if ($opt eq 'hotplug') {
4499                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4500             } elsif ($opt eq 'tablet') {
4501                 die "skip\n" if !$hotplug_features->{usb};
4502                 if ($value == 1) {
4503                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4504                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4505                         if $arch eq 'aarch64';
4506                 } elsif ($value == 0) {
4507                     vm_deviceunplug($vmid, $conf, 'tablet');
4508                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4509                 }
4510             } elsif ($opt =~ m/^usb\d+$/) {
4511                 die "skip\n";
4512                 # since we cannot reliably hot unplug usb devices
4513                 # we are disabling it
4514                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4515                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4516                 die "skip\n" if !$d;
4517                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4518             } elsif ($opt eq 'vcpus') {
4519                 die "skip\n" if !$hotplug_features->{cpu};
4520                 qemu_cpu_hotplug($vmid, $conf, $value);
4521             } elsif ($opt eq 'balloon') {
4522                 # enable/disable balloning device is not hotpluggable
4523                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4524                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4525                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4526
4527                 # allow manual ballooning if shares is set to zero
4528                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4529                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4530                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4531                 }
4532             } elsif ($opt =~ m/^net(\d+)$/) {
4533                 # some changes can be done without hotplug
4534                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4535                                     $vmid, $opt, $value, $arch, $machine_type);
4536             } elsif (is_valid_drivename($opt)) {
4537                 die "skip\n" if $opt eq 'efidisk0';
4538                 # some changes can be done without hotplug
4539                 my $drive = parse_drive($opt, $value);
4540                 if (drive_is_cloudinit($drive)) {
4541                     &$apply_pending_cloudinit($opt, $value);
4542                 }
4543                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4544                                      $vmid, $opt, $value, $arch, $machine_type);
4545             } elsif ($opt =~ m/^memory$/) { #dimms
4546                 die "skip\n" if !$hotplug_features->{memory};
4547                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4548             } elsif ($opt eq 'cpuunits') {
4549                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4550             } elsif ($opt eq 'cpulimit') {
4551                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4552                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4553             } else {
4554                 die "skip\n";  # skip non-hot-pluggable options
4555             }
4556         };
4557         if (my $err = $@) {
4558             &$add_error($opt, $err) if $err ne "skip\n";
4559         } else {
4560             $conf->{$opt} = $value;
4561             delete $conf->{pending}->{$opt};
4562         }
4563     }
4564
4565     PVE::QemuConfig->write_config($vmid, $conf);
4566 }
4567
4568 sub try_deallocate_drive {
4569     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4570
4571     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4572         my $volid = $drive->{file};
4573         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4574             my $sid = PVE::Storage::parse_volume_id($volid);
4575             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4576
4577             # check if the disk is really unused
4578             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4579                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4580             PVE::Storage::vdisk_free($storecfg, $volid);
4581             return 1;
4582         } else {
4583             # If vm is not owner of this disk remove from config
4584             return 1;
4585         }
4586     }
4587
4588     return undef;
4589 }
4590
4591 sub vmconfig_delete_or_detach_drive {
4592     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4593
4594     my $drive = parse_drive($opt, $conf->{$opt});
4595
4596     my $rpcenv = PVE::RPCEnvironment::get();
4597     my $authuser = $rpcenv->get_user();
4598
4599     if ($force) {
4600         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4601         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4602     } else {
4603         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4604     }
4605 }
4606
4607
4608
4609 sub vmconfig_apply_pending {
4610     my ($vmid, $conf, $storecfg, $errors) = @_;
4611
4612     my $add_apply_error = sub {
4613         my ($opt, $msg) = @_;
4614         my $err_msg = "unable to apply pending change $opt : $msg";
4615         $errors->{$opt} = $err_msg;
4616         warn $err_msg;
4617     };
4618
4619     # cold plug
4620
4621     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4622     foreach my $opt (sort keys %$pending_delete_hash) {
4623         my $force = $pending_delete_hash->{$opt}->{force};
4624         eval {
4625             if ($opt =~ m/^unused/) {
4626                 die "internal error";
4627             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4628                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4629             }
4630         };
4631         if (my $err = $@) {
4632             $add_apply_error->($opt, $err);
4633         } else {
4634             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4635             delete $conf->{$opt};
4636         }
4637     }
4638
4639     PVE::QemuConfig->cleanup_pending($conf);
4640
4641     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4642         next if $opt eq 'delete'; # just to be sure
4643         eval {
4644             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4645                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4646             }
4647         };
4648         if (my $err = $@) {
4649             $add_apply_error->($opt, $err);
4650         } else {
4651             $conf->{$opt} = delete $conf->{pending}->{$opt};
4652         }
4653     }
4654
4655     # write all changes at once to avoid unnecessary i/o
4656     PVE::QemuConfig->write_config($vmid, $conf);
4657 }
4658
4659 sub vmconfig_update_net {
4660     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4661
4662     my $newnet = parse_net($value);
4663
4664     if ($conf->{$opt}) {
4665         my $oldnet = parse_net($conf->{$opt});
4666
4667         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4668             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4669             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4670             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4671
4672             # for non online change, we try to hot-unplug
4673             die "skip\n" if !$hotplug;
4674             vm_deviceunplug($vmid, $conf, $opt);
4675         } else {
4676
4677             die "internal error" if $opt !~ m/net(\d+)/;
4678             my $iface = "tap${vmid}i$1";
4679
4680             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4681                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4682                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4683                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4684                 PVE::Network::tap_unplug($iface);
4685
4686                 if ($have_sdn) {
4687                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4688                 } else {
4689                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4690                 }
4691             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4692                 # Rate can be applied on its own but any change above needs to
4693                 # include the rate in tap_plug since OVS resets everything.
4694                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4695             }
4696
4697             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4698                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4699             }
4700
4701             return 1;
4702         }
4703     }
4704
4705     if ($hotplug) {
4706         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4707     } else {
4708         die "skip\n";
4709     }
4710 }
4711
4712 sub vmconfig_update_disk {
4713     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4714
4715     my $drive = parse_drive($opt, $value);
4716
4717     if ($conf->{$opt}) {
4718
4719         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4720
4721             my $media = $drive->{media} || 'disk';
4722             my $oldmedia = $old_drive->{media} || 'disk';
4723             die "unable to change media type\n" if $media ne $oldmedia;
4724
4725             if (!drive_is_cdrom($old_drive)) {
4726
4727                 if ($drive->{file} ne $old_drive->{file}) {
4728
4729                     die "skip\n" if !$hotplug;
4730
4731                     # unplug and register as unused
4732                     vm_deviceunplug($vmid, $conf, $opt);
4733                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4734
4735                 } else {
4736                     # update existing disk
4737
4738                     # skip non hotpluggable value
4739                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4740                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4741                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4742                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4743                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4744                         die "skip\n";
4745                     }
4746
4747                     # apply throttle
4748                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4749                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4750                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4751                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4752                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4753                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4754                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4755                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4756                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4757                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4758                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4759                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4760                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4761                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4762                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4763                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4764                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4765                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4766
4767                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4768                                                    ($drive->{mbps} || 0)*1024*1024,
4769                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4770                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4771                                                    $drive->{iops} || 0,
4772                                                    $drive->{iops_rd} || 0,
4773                                                    $drive->{iops_wr} || 0,
4774                                                    ($drive->{mbps_max} || 0)*1024*1024,
4775                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4776                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4777                                                    $drive->{iops_max} || 0,
4778                                                    $drive->{iops_rd_max} || 0,
4779                                                    $drive->{iops_wr_max} || 0,
4780                                                    $drive->{bps_max_length} || 1,
4781                                                    $drive->{bps_rd_max_length} || 1,
4782                                                    $drive->{bps_wr_max_length} || 1,
4783                                                    $drive->{iops_max_length} || 1,
4784                                                    $drive->{iops_rd_max_length} || 1,
4785                                                    $drive->{iops_wr_max_length} || 1);
4786
4787                     }
4788
4789                     return 1;
4790                 }
4791
4792             } else { # cdrom
4793
4794                 if ($drive->{file} eq 'none') {
4795                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4796                     if (drive_is_cloudinit($old_drive)) {
4797                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4798                     }
4799                 } else {
4800                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4801
4802                     # force eject if locked
4803                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4804
4805                     if ($path) {
4806                         mon_cmd($vmid, "blockdev-change-medium",
4807                             id => "$opt", filename => "$path");
4808                     }
4809                 }
4810
4811                 return 1;
4812             }
4813         }
4814     }
4815
4816     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4817     # hotplug new disks
4818     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4819     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4820 }
4821
4822 # called in locked context by incoming migration
4823 sub vm_migrate_get_nbd_disks {
4824     my ($storecfg, $conf, $replicated_volumes) = @_;
4825
4826     my $local_volumes = {};
4827     PVE::QemuConfig->foreach_volume($conf, sub {
4828         my ($ds, $drive) = @_;
4829
4830         return if drive_is_cdrom($drive);
4831
4832         my $volid = $drive->{file};
4833
4834         return if !$volid;
4835
4836         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4837
4838         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4839         return if $scfg->{shared};
4840
4841         # replicated disks re-use existing state via bitmap
4842         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4843         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4844     });
4845     return $local_volumes;
4846 }
4847
4848 # called in locked context by incoming migration
4849 sub vm_migrate_alloc_nbd_disks {
4850     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4851
4852     my $format = undef;
4853
4854     my $nbd = {};
4855     foreach my $opt (sort keys %$source_volumes) {
4856         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4857
4858         if ($use_existing) {
4859             $nbd->{$opt}->{drivestr} = print_drive($drive);
4860             $nbd->{$opt}->{volid} = $volid;
4861             $nbd->{$opt}->{replicated} = 1;
4862             next;
4863         }
4864
4865         # If a remote storage is specified and the format of the original
4866         # volume is not available there, fall back to the default format.
4867         # Otherwise use the same format as the original.
4868         if (!$storagemap->{identity}) {
4869             $storeid = map_storage($storagemap, $storeid);
4870             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4871             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4872             my $fileFormat = qemu_img_format($scfg, $volname);
4873             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4874         } else {
4875             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4876             $format = qemu_img_format($scfg, $volname);
4877         }
4878
4879         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4880         my $newdrive = $drive;
4881         $newdrive->{format} = $format;
4882         $newdrive->{file} = $newvolid;
4883         my $drivestr = print_drive($newdrive);
4884         $nbd->{$opt}->{drivestr} = $drivestr;
4885         $nbd->{$opt}->{volid} = $newvolid;
4886     }
4887
4888     return $nbd;
4889 }
4890
4891 # see vm_start_nolock for parameters, additionally:
4892 # migrate_opts:
4893 #   storagemap = parsed storage map for allocating NBD disks
4894 sub vm_start {
4895     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4896
4897     return PVE::QemuConfig->lock_config($vmid, sub {
4898         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4899
4900         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4901
4902         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4903
4904         PVE::QemuConfig->check_lock($conf)
4905             if !($params->{skiplock} || $params->{resume});
4906
4907         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4908
4909         if (my $storagemap = $migrate_opts->{storagemap}) {
4910             my $replicated = $migrate_opts->{replicated_volumes};
4911             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4912             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4913
4914             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4915                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4916             }
4917         }
4918
4919         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4920     });
4921 }
4922
4923
4924 # params:
4925 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4926 #   skiplock => 0/1, skip checking for config lock
4927 #   forcemachine => to force Qemu machine (rollback/migration)
4928 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4929 #   timeout => in seconds
4930 #   paused => start VM in paused state (backup)
4931 #   resume => resume from hibernation
4932 # migrate_opts:
4933 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4934 #   migratedfrom => source node
4935 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4936 #   network => CIDR of migration network
4937 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4938 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4939 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4940 sub vm_start_nolock {
4941     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4942
4943     my $statefile = $params->{statefile};
4944     my $resume = $params->{resume};
4945
4946     my $migratedfrom = $migrate_opts->{migratedfrom};
4947     my $migration_type = $migrate_opts->{type};
4948
4949     my $res = {};
4950
4951     # clean up leftover reboot request files
4952     eval { clear_reboot_request($vmid); };
4953     warn $@ if $@;
4954
4955     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4956         vmconfig_apply_pending($vmid, $conf, $storecfg);
4957         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4958     }
4959
4960     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4961
4962     my $defaults = load_defaults();
4963
4964     # set environment variable useful inside network script
4965     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4966
4967     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4968
4969     my $forcemachine = $params->{forcemachine};
4970     my $forcecpu = $params->{forcecpu};
4971     if ($resume) {
4972         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4973         $forcemachine = $conf->{runningmachine};
4974         $forcecpu = $conf->{runningcpu};
4975         print "Resuming suspended VM\n";
4976     }
4977
4978     my ($cmd, $vollist, $spice_port) =
4979         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4980
4981     my $migration_ip;
4982     my $get_migration_ip = sub {
4983         my ($nodename) = @_;
4984
4985         return $migration_ip if defined($migration_ip);
4986
4987         my $cidr = $migrate_opts->{network};
4988
4989         if (!defined($cidr)) {
4990             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4991             $cidr = $dc_conf->{migration}->{network};
4992         }
4993
4994         if (defined($cidr)) {
4995             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4996
4997             die "could not get IP: no address configured on local " .
4998                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4999
5000             die "could not get IP: multiple addresses configured on local " .
5001                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5002
5003             $migration_ip = @$ips[0];
5004         }
5005
5006         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5007             if !defined($migration_ip);
5008
5009         return $migration_ip;
5010     };
5011
5012     my $migrate_uri;
5013     if ($statefile) {
5014         if ($statefile eq 'tcp') {
5015             my $localip = "localhost";
5016             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5017             my $nodename = nodename();
5018
5019             if (!defined($migration_type)) {
5020                 if (defined($datacenterconf->{migration}->{type})) {
5021                     $migration_type = $datacenterconf->{migration}->{type};
5022                 } else {
5023                     $migration_type = 'secure';
5024                 }
5025             }
5026
5027             if ($migration_type eq 'insecure') {
5028                 $localip = $get_migration_ip->($nodename);
5029                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5030             }
5031
5032             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5033             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5034             $migrate_uri = "tcp:${localip}:${migrate_port}";
5035             push @$cmd, '-incoming', $migrate_uri;
5036             push @$cmd, '-S';
5037
5038         } elsif ($statefile eq 'unix') {
5039             # should be default for secure migrations as a ssh TCP forward
5040             # tunnel is not deterministic reliable ready and fails regurarly
5041             # to set up in time, so use UNIX socket forwards
5042             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5043             unlink $socket_addr;
5044
5045             $migrate_uri = "unix:$socket_addr";
5046
5047             push @$cmd, '-incoming', $migrate_uri;
5048             push @$cmd, '-S';
5049
5050         } elsif (-e $statefile) {
5051             push @$cmd, '-loadstate', $statefile;
5052         } else {
5053             my $statepath = PVE::Storage::path($storecfg, $statefile);
5054             push @$vollist, $statefile;
5055             push @$cmd, '-loadstate', $statepath;
5056         }
5057     } elsif ($params->{paused}) {
5058         push @$cmd, '-S';
5059     }
5060
5061     # host pci devices
5062     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5063       my $d = parse_hostpci($conf->{"hostpci$i"});
5064       next if !$d;
5065       my $pcidevices = $d->{pciid};
5066       foreach my $pcidevice (@$pcidevices) {
5067             my $pciid = $pcidevice->{id};
5068
5069             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5070             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5071             die "no pci device info for device '$pciid'\n" if !$info;
5072
5073             if ($d->{mdev}) {
5074                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5075                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5076             } else {
5077                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5078                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5079                 die "can't reset pci device '$pciid'\n"
5080                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5081             }
5082       }
5083     }
5084
5085     PVE::Storage::activate_volumes($storecfg, $vollist);
5086
5087     eval {
5088         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5089             outfunc => sub {}, errfunc => sub {});
5090     };
5091     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5092     # timeout should be more than enough here...
5093     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5094
5095     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5096                                               : $defaults->{cpuunits};
5097
5098     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5099     my %run_params = (
5100         timeout => $statefile ? undef : $start_timeout,
5101         umask => 0077,
5102         noerr => 1,
5103     );
5104
5105     # when migrating, prefix QEMU output so other side can pick up any
5106     # errors that might occur and show the user
5107     if ($migratedfrom) {
5108         $run_params{quiet} = 1;
5109         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5110     }
5111
5112     my %properties = (
5113         Slice => 'qemu.slice',
5114         KillMode => 'none',
5115         CPUShares => $cpuunits
5116     );
5117
5118     if (my $cpulimit = $conf->{cpulimit}) {
5119         $properties{CPUQuota} = int($cpulimit * 100);
5120     }
5121     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5122
5123     my $run_qemu = sub {
5124         PVE::Tools::run_fork sub {
5125             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5126
5127             my $exitcode = run_command($cmd, %run_params);
5128             die "QEMU exited with code $exitcode\n" if $exitcode;
5129         };
5130     };
5131
5132     if ($conf->{hugepages}) {
5133
5134         my $code = sub {
5135             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5136             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5137
5138             PVE::QemuServer::Memory::hugepages_mount();
5139             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5140
5141             eval { $run_qemu->() };
5142             if (my $err = $@) {
5143                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5144                 die $err;
5145             }
5146
5147             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5148         };
5149         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5150
5151     } else {
5152         eval { $run_qemu->() };
5153     }
5154
5155     if (my $err = $@) {
5156         # deactivate volumes if start fails
5157         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5158         die "start failed: $err";
5159     }
5160
5161     print "migration listens on $migrate_uri\n" if $migrate_uri;
5162     $res->{migrate_uri} = $migrate_uri;
5163
5164     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5165         eval { mon_cmd($vmid, "cont"); };
5166         warn $@ if $@;
5167     }
5168
5169     #start nbd server for storage migration
5170     if (my $nbd = $migrate_opts->{nbd}) {
5171         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5172
5173         my $migrate_storage_uri;
5174         # nbd_protocol_version > 0 for unix socket support
5175         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5176             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5177             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5178             $migrate_storage_uri = "nbd:unix:$socket_path";
5179         } else {
5180             my $nodename = nodename();
5181             my $localip = $get_migration_ip->($nodename);
5182             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5183             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5184
5185             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5186             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5187             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5188         }
5189
5190         $res->{migrate_storage_uri} = $migrate_storage_uri;
5191
5192         foreach my $opt (sort keys %$nbd) {
5193             my $drivestr = $nbd->{$opt}->{drivestr};
5194             my $volid = $nbd->{$opt}->{volid};
5195             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5196             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5197             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5198             print "re-using replicated volume: $opt - $volid\n"
5199                 if $nbd->{$opt}->{replicated};
5200
5201             $res->{drives}->{$opt} = $nbd->{$opt};
5202             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5203         }
5204     }
5205
5206     if ($migratedfrom) {
5207         eval {
5208             set_migration_caps($vmid);
5209         };
5210         warn $@ if $@;
5211
5212         if ($spice_port) {
5213             print "spice listens on port $spice_port\n";
5214             $res->{spice_port} = $spice_port;
5215             if ($migrate_opts->{spice_ticket}) {
5216                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5217                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5218             }
5219         }
5220
5221     } else {
5222         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5223             if !$statefile && $conf->{balloon};
5224
5225         foreach my $opt (keys %$conf) {
5226             next if $opt !~  m/^net\d+$/;
5227             my $nicconf = parse_net($conf->{$opt});
5228             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5229         }
5230     }
5231
5232     mon_cmd($vmid, 'qom-set',
5233                 path => "machine/peripheral/balloon0",
5234                 property => "guest-stats-polling-interval",
5235                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5236
5237     if ($resume) {
5238         print "Resumed VM, removing state\n";
5239         if (my $vmstate = $conf->{vmstate}) {
5240             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5241             PVE::Storage::vdisk_free($storecfg, $vmstate);
5242         }
5243         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5244         PVE::QemuConfig->write_config($vmid, $conf);
5245     }
5246
5247     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5248
5249     return $res;
5250 }
5251
5252 sub vm_commandline {
5253     my ($storecfg, $vmid, $snapname) = @_;
5254
5255     my $conf = PVE::QemuConfig->load_config($vmid);
5256     my $forcemachine;
5257     my $forcecpu;
5258
5259     if ($snapname) {
5260         my $snapshot = $conf->{snapshots}->{$snapname};
5261         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5262
5263         # check for machine or CPU overrides in snapshot
5264         $forcemachine = $snapshot->{runningmachine};
5265         $forcecpu = $snapshot->{runningcpu};
5266
5267         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5268
5269         $conf = $snapshot;
5270     }
5271
5272     my $defaults = load_defaults();
5273
5274     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5275         $forcemachine, $forcecpu);
5276
5277     return PVE::Tools::cmd2string($cmd);
5278 }
5279
5280 sub vm_reset {
5281     my ($vmid, $skiplock) = @_;
5282
5283     PVE::QemuConfig->lock_config($vmid, sub {
5284
5285         my $conf = PVE::QemuConfig->load_config($vmid);
5286
5287         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5288
5289         mon_cmd($vmid, "system_reset");
5290     });
5291 }
5292
5293 sub get_vm_volumes {
5294     my ($conf) = @_;
5295
5296     my $vollist = [];
5297     foreach_volid($conf, sub {
5298         my ($volid, $attr) = @_;
5299
5300         return if $volid =~ m|^/|;
5301
5302         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5303         return if !$sid;
5304
5305         push @$vollist, $volid;
5306     });
5307
5308     return $vollist;
5309 }
5310
5311 sub vm_stop_cleanup {
5312     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5313
5314     eval {
5315
5316         if (!$keepActive) {
5317             my $vollist = get_vm_volumes($conf);
5318             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5319         }
5320
5321         foreach my $ext (qw(mon qmp pid vnc qga)) {
5322             unlink "/var/run/qemu-server/${vmid}.$ext";
5323         }
5324
5325         if ($conf->{ivshmem}) {
5326             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5327             # just delete it for now, VMs which have this already open do not
5328             # are affected, but new VMs will get a separated one. If this
5329             # becomes an issue we either add some sort of ref-counting or just
5330             # add a "don't delete on stop" flag to the ivshmem format.
5331             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5332         }
5333
5334         foreach my $key (keys %$conf) {
5335             next if $key !~ m/^hostpci(\d+)$/;
5336             my $hostpciindex = $1;
5337             my $d = parse_hostpci($conf->{$key});
5338             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5339
5340             foreach my $pci (@{$d->{pciid}}) {
5341                 my $pciid = $pci->{id};
5342                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5343             }
5344         }
5345
5346         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5347     };
5348     warn $@ if $@; # avoid errors - just warn
5349 }
5350
5351 # call only in locked context
5352 sub _do_vm_stop {
5353     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5354
5355     my $pid = check_running($vmid, $nocheck);
5356     return if !$pid;
5357
5358     my $conf;
5359     if (!$nocheck) {
5360         $conf = PVE::QemuConfig->load_config($vmid);
5361         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5362         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5363             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5364             $timeout = $opts->{down} if $opts->{down};
5365         }
5366         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5367     }
5368
5369     eval {
5370         if ($shutdown) {
5371             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5372                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5373             } else {
5374                 mon_cmd($vmid, "system_powerdown");
5375             }
5376         } else {
5377             mon_cmd($vmid, "quit");
5378         }
5379     };
5380     my $err = $@;
5381
5382     if (!$err) {
5383         $timeout = 60 if !defined($timeout);
5384
5385         my $count = 0;
5386         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5387             $count++;
5388             sleep 1;
5389         }
5390
5391         if ($count >= $timeout) {
5392             if ($force) {
5393                 warn "VM still running - terminating now with SIGTERM\n";
5394                 kill 15, $pid;
5395             } else {
5396                 die "VM quit/powerdown failed - got timeout\n";
5397             }
5398         } else {
5399             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5400             return;
5401         }
5402     } else {
5403        if (!check_running($vmid, $nocheck)) {
5404             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5405             return;
5406        }
5407        if ($force) {
5408             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5409             kill 15, $pid;
5410         } else {
5411             die "VM quit/powerdown failed\n";
5412         }
5413     }
5414
5415     # wait again
5416     $timeout = 10;
5417
5418     my $count = 0;
5419     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5420         $count++;
5421         sleep 1;
5422     }
5423
5424     if ($count >= $timeout) {
5425         warn "VM still running - terminating now with SIGKILL\n";
5426         kill 9, $pid;
5427         sleep 1;
5428     }
5429
5430     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5431 }
5432
5433 # Note: use $nocheck to skip tests if VM configuration file exists.
5434 # We need that when migration VMs to other nodes (files already moved)
5435 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5436 sub vm_stop {
5437     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5438
5439     $force = 1 if !defined($force) && !$shutdown;
5440
5441     if ($migratedfrom){
5442         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5443         kill 15, $pid if $pid;
5444         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5445         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5446         return;
5447     }
5448
5449     PVE::QemuConfig->lock_config($vmid, sub {
5450         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5451    });
5452 }
5453
5454 sub vm_reboot {
5455     my ($vmid, $timeout) = @_;
5456
5457     PVE::QemuConfig->lock_config($vmid, sub {
5458         eval {
5459
5460             # only reboot if running, as qmeventd starts it again on a stop event
5461             return if !check_running($vmid);
5462
5463             create_reboot_request($vmid);
5464
5465             my $storecfg = PVE::Storage::config();
5466             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5467
5468         };
5469         if (my $err = $@) {
5470             # avoid that the next normal shutdown will be confused for a reboot
5471             clear_reboot_request($vmid);
5472             die $err;
5473         }
5474    });
5475 }
5476
5477 # note: if using the statestorage parameter, the caller has to check privileges
5478 sub vm_suspend {
5479     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5480
5481     my $conf;
5482     my $path;
5483     my $storecfg;
5484     my $vmstate;
5485
5486     PVE::QemuConfig->lock_config($vmid, sub {
5487
5488         $conf = PVE::QemuConfig->load_config($vmid);
5489
5490         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5491         PVE::QemuConfig->check_lock($conf)
5492             if !($skiplock || $is_backing_up);
5493
5494         die "cannot suspend to disk during backup\n"
5495             if $is_backing_up && $includestate;
5496
5497         if ($includestate) {
5498             $conf->{lock} = 'suspending';
5499             my $date = strftime("%Y-%m-%d", localtime(time()));
5500             $storecfg = PVE::Storage::config();
5501             if (!$statestorage) {
5502                 $statestorage = find_vmstate_storage($conf, $storecfg);
5503                 # check permissions for the storage
5504                 my $rpcenv = PVE::RPCEnvironment::get();
5505                 if ($rpcenv->{type} ne 'cli') {
5506                     my $authuser = $rpcenv->get_user();
5507                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5508                 }
5509             }
5510
5511
5512             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5513             $path = PVE::Storage::path($storecfg, $vmstate);
5514             PVE::QemuConfig->write_config($vmid, $conf);
5515         } else {
5516             mon_cmd($vmid, "stop");
5517         }
5518     });
5519
5520     if ($includestate) {
5521         # save vm state
5522         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5523
5524         eval {
5525             mon_cmd($vmid, "savevm-start", statefile => $path);
5526             for(;;) {
5527                 my $state = mon_cmd($vmid, "query-savevm");
5528                 if (!$state->{status}) {
5529                     die "savevm not active\n";
5530                 } elsif ($state->{status} eq 'active') {
5531                     sleep(1);
5532                     next;
5533                 } elsif ($state->{status} eq 'completed') {
5534                     print "State saved, quitting\n";
5535                     last;
5536                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5537                     die "query-savevm failed with error '$state->{error}'\n"
5538                 } else {
5539                     die "query-savevm returned status '$state->{status}'\n";
5540                 }
5541             }
5542         };
5543         my $err = $@;
5544
5545         PVE::QemuConfig->lock_config($vmid, sub {
5546             $conf = PVE::QemuConfig->load_config($vmid);
5547             if ($err) {
5548                 # cleanup, but leave suspending lock, to indicate something went wrong
5549                 eval {
5550                     mon_cmd($vmid, "savevm-end");
5551                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5552                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5553                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5554                     PVE::QemuConfig->write_config($vmid, $conf);
5555                 };
5556                 warn $@ if $@;
5557                 die $err;
5558             }
5559
5560             die "lock changed unexpectedly\n"
5561                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5562
5563             mon_cmd($vmid, "quit");
5564             $conf->{lock} = 'suspended';
5565             PVE::QemuConfig->write_config($vmid, $conf);
5566         });
5567     }
5568 }
5569
5570 sub vm_resume {
5571     my ($vmid, $skiplock, $nocheck) = @_;
5572
5573     PVE::QemuConfig->lock_config($vmid, sub {
5574         my $res = mon_cmd($vmid, 'query-status');
5575         my $resume_cmd = 'cont';
5576
5577         if ($res->{status} && $res->{status} eq 'suspended') {
5578             $resume_cmd = 'system_wakeup';
5579         }
5580
5581         if (!$nocheck) {
5582
5583             my $conf = PVE::QemuConfig->load_config($vmid);
5584
5585             PVE::QemuConfig->check_lock($conf)
5586                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5587         }
5588
5589         mon_cmd($vmid, $resume_cmd);
5590     });
5591 }
5592
5593 sub vm_sendkey {
5594     my ($vmid, $skiplock, $key) = @_;
5595
5596     PVE::QemuConfig->lock_config($vmid, sub {
5597
5598         my $conf = PVE::QemuConfig->load_config($vmid);
5599
5600         # there is no qmp command, so we use the human monitor command
5601         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5602         die $res if $res ne '';
5603     });
5604 }
5605
5606 # vzdump restore implementaion
5607
5608 sub tar_archive_read_firstfile {
5609     my $archive = shift;
5610
5611     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5612
5613     # try to detect archive type first
5614     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5615         die "unable to open file '$archive'\n";
5616     my $firstfile = <$fh>;
5617     kill 15, $pid;
5618     close $fh;
5619
5620     die "ERROR: archive contaions no data\n" if !$firstfile;
5621     chomp $firstfile;
5622
5623     return $firstfile;
5624 }
5625
5626 sub tar_restore_cleanup {
5627     my ($storecfg, $statfile) = @_;
5628
5629     print STDERR "starting cleanup\n";
5630
5631     if (my $fd = IO::File->new($statfile, "r")) {
5632         while (defined(my $line = <$fd>)) {
5633             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5634                 my $volid = $2;
5635                 eval {
5636                     if ($volid =~ m|^/|) {
5637                         unlink $volid || die 'unlink failed\n';
5638                     } else {
5639                         PVE::Storage::vdisk_free($storecfg, $volid);
5640                     }
5641                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5642                 };
5643                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5644             } else {
5645                 print STDERR "unable to parse line in statfile - $line";
5646             }
5647         }
5648         $fd->close();
5649     }
5650 }
5651
5652 sub restore_file_archive {
5653     my ($archive, $vmid, $user, $opts) = @_;
5654
5655     my $info = PVE::Storage::archive_info($archive);
5656     my $format = $opts->{format} // $info->{format};
5657     my $comp = $info->{compression};
5658
5659     # try to detect archive format
5660     if ($format eq 'tar') {
5661         return restore_tar_archive($archive, $vmid, $user, $opts);
5662     } else {
5663         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5664     }
5665 }
5666
5667 # hepler to remove disks that will not be used after restore
5668 my $restore_cleanup_oldconf = sub {
5669     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5670
5671     PVE::QemuConfig->foreach_volume($oldconf, sub {
5672         my ($ds, $drive) = @_;
5673
5674         return if drive_is_cdrom($drive, 1);
5675
5676         my $volid = $drive->{file};
5677         return if !$volid || $volid =~ m|^/|;
5678
5679         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5680         return if !$path || !$owner || ($owner != $vmid);
5681
5682         # Note: only delete disk we want to restore
5683         # other volumes will become unused
5684         if ($virtdev_hash->{$ds}) {
5685             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5686             if (my $err = $@) {
5687                 warn $err;
5688             }
5689         }
5690     });
5691
5692     # delete vmstate files, after the restore we have no snapshots anymore
5693     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5694         my $snap = $oldconf->{snapshots}->{$snapname};
5695         if ($snap->{vmstate}) {
5696             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5697             if (my $err = $@) {
5698                 warn $err;
5699             }
5700         }
5701     }
5702 };
5703
5704 # Helper to parse vzdump backup device hints
5705 #
5706 # $rpcenv: Environment, used to ckeck storage permissions
5707 # $user: User ID, to check storage permissions
5708 # $storecfg: Storage configuration
5709 # $fh: the file handle for reading the configuration
5710 # $devinfo: should contain device sizes for all backu-up'ed devices
5711 # $options: backup options (pool, default storage)
5712 #
5713 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5714 my $parse_backup_hints = sub {
5715     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5716
5717     my $virtdev_hash = {};
5718
5719     while (defined(my $line = <$fh>)) {
5720         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5721             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5722             die "archive does not contain data for drive '$virtdev'\n"
5723                 if !$devinfo->{$devname};
5724
5725             if (defined($options->{storage})) {
5726                 $storeid = $options->{storage} || 'local';
5727             } elsif (!$storeid) {
5728                 $storeid = 'local';
5729             }
5730             $format = 'raw' if !$format;
5731             $devinfo->{$devname}->{devname} = $devname;
5732             $devinfo->{$devname}->{virtdev} = $virtdev;
5733             $devinfo->{$devname}->{format} = $format;
5734             $devinfo->{$devname}->{storeid} = $storeid;
5735
5736             # check permission on storage
5737             my $pool = $options->{pool}; # todo: do we need that?
5738             if ($user ne 'root@pam') {
5739                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5740             }
5741
5742             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5743         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5744             my $virtdev = $1;
5745             my $drive = parse_drive($virtdev, $2);
5746             if (drive_is_cloudinit($drive)) {
5747                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5748                 $storeid = $options->{storage} if defined ($options->{storage});
5749                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5750                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5751
5752                 $virtdev_hash->{$virtdev} = {
5753                     format => $format,
5754                     storeid => $storeid,
5755                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5756                     is_cloudinit => 1,
5757                 };
5758             }
5759         }
5760     }
5761
5762     return $virtdev_hash;
5763 };
5764
5765 # Helper to allocate and activate all volumes required for a restore
5766 #
5767 # $storecfg: Storage configuration
5768 # $virtdev_hash: as returned by parse_backup_hints()
5769 #
5770 # Returns: { $virtdev => $volid }
5771 my $restore_allocate_devices = sub {
5772     my ($storecfg, $virtdev_hash, $vmid) = @_;
5773
5774     my $map = {};
5775     foreach my $virtdev (sort keys %$virtdev_hash) {
5776         my $d = $virtdev_hash->{$virtdev};
5777         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5778         my $storeid = $d->{storeid};
5779         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5780
5781         # test if requested format is supported
5782         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5783         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5784         $d->{format} = $defFormat if !$supported;
5785
5786         my $name;
5787         if ($d->{is_cloudinit}) {
5788             $name = "vm-$vmid-cloudinit";
5789             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5790         }
5791
5792         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5793
5794         print STDERR "new volume ID is '$volid'\n";
5795         $d->{volid} = $volid;
5796
5797         PVE::Storage::activate_volumes($storecfg, [$volid]);
5798
5799         $map->{$virtdev} = $volid;
5800     }
5801
5802     return $map;
5803 };
5804
5805 my $restore_update_config_line = sub {
5806     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5807
5808     return if $line =~ m/^\#qmdump\#/;
5809     return if $line =~ m/^\#vzdump\#/;
5810     return if $line =~ m/^lock:/;
5811     return if $line =~ m/^unused\d+:/;
5812     return if $line =~ m/^parent:/;
5813
5814     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5815     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5816         # try to convert old 1.X settings
5817         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5818         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5819             my ($model, $macaddr) = split(/\=/, $devconfig);
5820             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5821             my $net = {
5822                 model => $model,
5823                 bridge => "vmbr$ind",
5824                 macaddr => $macaddr,
5825             };
5826             my $netstr = print_net($net);
5827
5828             print $outfd "net$cookie->{netcount}: $netstr\n";
5829             $cookie->{netcount}++;
5830         }
5831     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5832         my ($id, $netstr) = ($1, $2);
5833         my $net = parse_net($netstr);
5834         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5835         $netstr = print_net($net);
5836         print $outfd "$id: $netstr\n";
5837     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5838         my $virtdev = $1;
5839         my $value = $3;
5840         my $di = parse_drive($virtdev, $value);
5841         if (defined($di->{backup}) && !$di->{backup}) {
5842             print $outfd "#$line";
5843         } elsif ($map->{$virtdev}) {
5844             delete $di->{format}; # format can change on restore
5845             $di->{file} = $map->{$virtdev};
5846             $value = print_drive($di);
5847             print $outfd "$virtdev: $value\n";
5848         } else {
5849             print $outfd $line;
5850         }
5851     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5852         my $vmgenid = $1;
5853         if ($vmgenid ne '0') {
5854             # always generate a new vmgenid if there was a valid one setup
5855             $vmgenid = generate_uuid();
5856         }
5857         print $outfd "vmgenid: $vmgenid\n";
5858     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5859         my ($uuid, $uuid_str);
5860         UUID::generate($uuid);
5861         UUID::unparse($uuid, $uuid_str);
5862         my $smbios1 = parse_smbios1($2);
5863         $smbios1->{uuid} = $uuid_str;
5864         print $outfd $1.print_smbios1($smbios1)."\n";
5865     } else {
5866         print $outfd $line;
5867     }
5868 };
5869
5870 my $restore_deactivate_volumes = sub {
5871     my ($storecfg, $devinfo) = @_;
5872
5873     my $vollist = [];
5874     foreach my $devname (keys %$devinfo) {
5875         my $volid = $devinfo->{$devname}->{volid};
5876         push @$vollist, $volid if $volid;
5877     }
5878
5879     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5880 };
5881
5882 my $restore_destroy_volumes = sub {
5883     my ($storecfg, $devinfo) = @_;
5884
5885     foreach my $devname (keys %$devinfo) {
5886         my $volid = $devinfo->{$devname}->{volid};
5887         next if !$volid;
5888         eval {
5889             if ($volid =~ m|^/|) {
5890                 unlink $volid || die 'unlink failed\n';
5891             } else {
5892                 PVE::Storage::vdisk_free($storecfg, $volid);
5893             }
5894             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5895         };
5896         print STDERR "unable to cleanup '$volid' - $@" if $@;
5897     }
5898 };
5899
5900 sub scan_volids {
5901     my ($cfg, $vmid) = @_;
5902
5903     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5904
5905     my $volid_hash = {};
5906     foreach my $storeid (keys %$info) {
5907         foreach my $item (@{$info->{$storeid}}) {
5908             next if !($item->{volid} && $item->{size});
5909             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5910             $volid_hash->{$item->{volid}} = $item;
5911         }
5912     }
5913
5914     return $volid_hash;
5915 }
5916
5917 sub update_disk_config {
5918     my ($vmid, $conf, $volid_hash) = @_;
5919
5920     my $changes;
5921     my $prefix = "VM $vmid:";
5922
5923     # used and unused disks
5924     my $referenced = {};
5925
5926     # Note: it is allowed to define multiple storages with same path (alias), so
5927     # we need to check both 'volid' and real 'path' (two different volid can point
5928     # to the same path).
5929
5930     my $referencedpath = {};
5931
5932     # update size info
5933     PVE::QemuConfig->foreach_volume($conf, sub {
5934         my ($opt, $drive) = @_;
5935
5936         my $volid = $drive->{file};
5937         return if !$volid;
5938
5939         # mark volid as "in-use" for next step
5940         $referenced->{$volid} = 1;
5941         if ($volid_hash->{$volid} &&
5942             (my $path = $volid_hash->{$volid}->{path})) {
5943             $referencedpath->{$path} = 1;
5944         }
5945
5946         return if drive_is_cdrom($drive);
5947         return if !$volid_hash->{$volid};
5948
5949         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5950         if (defined($updated)) {
5951             $changes = 1;
5952             $conf->{$opt} = print_drive($updated);
5953             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5954         }
5955     });
5956
5957     # remove 'unusedX' entry if volume is used
5958     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5959         my ($opt, $drive) = @_;
5960
5961         my $volid = $drive->{file};
5962         return if !$volid;
5963
5964         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5965         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5966             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5967             $changes = 1;
5968             delete $conf->{$opt};
5969         }
5970
5971         $referenced->{$volid} = 1;
5972         $referencedpath->{$path} = 1 if $path;
5973     });
5974
5975     foreach my $volid (sort keys %$volid_hash) {
5976         next if $volid =~ m/vm-$vmid-state-/;
5977         next if $referenced->{$volid};
5978         my $path = $volid_hash->{$volid}->{path};
5979         next if !$path; # just to be sure
5980         next if $referencedpath->{$path};
5981         $changes = 1;
5982         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5983         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5984         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5985     }
5986
5987     return $changes;
5988 }
5989
5990 sub rescan {
5991     my ($vmid, $nolock, $dryrun) = @_;
5992
5993     my $cfg = PVE::Storage::config();
5994
5995     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5996     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5997     foreach my $stor (keys %{$cfg->{ids}}) {
5998         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5999     }
6000
6001     print "rescan volumes...\n";
6002     my $volid_hash = scan_volids($cfg, $vmid);
6003
6004     my $updatefn =  sub {
6005         my ($vmid) = @_;
6006
6007         my $conf = PVE::QemuConfig->load_config($vmid);
6008
6009         PVE::QemuConfig->check_lock($conf);
6010
6011         my $vm_volids = {};
6012         foreach my $volid (keys %$volid_hash) {
6013             my $info = $volid_hash->{$volid};
6014             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6015         }
6016
6017         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6018
6019         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6020     };
6021
6022     if (defined($vmid)) {
6023         if ($nolock) {
6024             &$updatefn($vmid);
6025         } else {
6026             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6027         }
6028     } else {
6029         my $vmlist = config_list();
6030         foreach my $vmid (keys %$vmlist) {
6031             if ($nolock) {
6032                 &$updatefn($vmid);
6033             } else {
6034                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6035             }
6036         }
6037     }
6038 }
6039
6040 sub restore_proxmox_backup_archive {
6041     my ($archive, $vmid, $user, $options) = @_;
6042
6043     my $storecfg = PVE::Storage::config();
6044
6045     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6046     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6047
6048     my $server = $scfg->{server};
6049     my $datastore = $scfg->{datastore};
6050     my $username = $scfg->{username} // 'root@pam';
6051     my $fingerprint = $scfg->{fingerprint};
6052
6053     my $repo = "$username\@$server:$datastore";
6054     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6055     local $ENV{PBS_PASSWORD} = $password;
6056     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6057
6058     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6059         PVE::Storage::parse_volname($storecfg, $archive);
6060
6061     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6062
6063     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6064
6065     my $tmpdir = "/var/tmp/vzdumptmp$$";
6066     rmtree $tmpdir;
6067     mkpath $tmpdir;
6068
6069     my $conffile = PVE::QemuConfig->config_file($vmid);
6070     my $tmpfn = "$conffile.$$.tmp";
6071      # disable interrupts (always do cleanups)
6072     local $SIG{INT} =
6073         local $SIG{TERM} =
6074         local $SIG{QUIT} =
6075         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6076
6077     # Note: $oldconf is undef if VM does not exists
6078     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6079     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6080
6081     my $rpcenv = PVE::RPCEnvironment::get();
6082     my $devinfo = {};
6083
6084     eval {
6085         # enable interrupts
6086         local $SIG{INT} =
6087             local $SIG{TERM} =
6088             local $SIG{QUIT} =
6089             local $SIG{HUP} =
6090             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6091
6092         my $cfgfn = "$tmpdir/qemu-server.conf";
6093         my $firewall_config_fn = "$tmpdir/fw.conf";
6094         my $index_fn = "$tmpdir/index.json";
6095
6096         my $cmd = "restore";
6097
6098         my $param = [$pbs_backup_name, "index.json", $index_fn];
6099         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6100         my $index = PVE::Tools::file_get_contents($index_fn);
6101         $index = decode_json($index);
6102
6103         # print Dumper($index);
6104         foreach my $info (@{$index->{files}}) {
6105             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6106                 my $devname = $1;
6107                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6108                     $devinfo->{$devname}->{size} = $1;
6109                 } else {
6110                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6111                 }
6112             }
6113         }
6114
6115         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6116         if (!$is_qemu_server_backup) {
6117             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6118         }
6119         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6120
6121         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6122         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6123
6124         if ($has_firewall_config) {
6125             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6126             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6127
6128             my $pve_firewall_dir = '/etc/pve/firewall';
6129             mkdir $pve_firewall_dir; # make sure the dir exists
6130             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6131         }
6132
6133         my $fh = IO::File->new($cfgfn, "r") ||
6134             "unable to read qemu-server.conf - $!\n";
6135
6136         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6137
6138         # fixme: rate limit?
6139
6140         # create empty/temp config
6141         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6142
6143         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6144
6145         # allocate volumes
6146         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6147
6148         foreach my $virtdev (sort keys %$virtdev_hash) {
6149             my $d = $virtdev_hash->{$virtdev};
6150             next if $d->{is_cloudinit}; # no need to restore cloudinit
6151
6152             my $volid = $d->{volid};
6153
6154             my $path = PVE::Storage::path($storecfg, $volid);
6155
6156             my $pbs_restore_cmd = [
6157                 '/usr/bin/pbs-restore',
6158                 '--repository', $repo,
6159                 $pbs_backup_name,
6160                 "$d->{devname}.img.fidx",
6161                 $path,
6162                 '--verbose',
6163                 ];
6164
6165             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6166                 push @$pbs_restore_cmd, '--skip-zero';
6167             }
6168
6169             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6170             print "restore proxmox backup image: $dbg_cmdstring\n";
6171             run_command($pbs_restore_cmd);
6172         }
6173
6174         $fh->seek(0, 0) || die "seek failed - $!\n";
6175
6176         my $outfd = new IO::File ($tmpfn, "w") ||
6177             die "unable to write config for VM $vmid\n";
6178
6179         my $cookie = { netcount => 0 };
6180         while (defined(my $line = <$fh>)) {
6181             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6182         }
6183
6184         $fh->close();
6185         $outfd->close();
6186     };
6187     my $err = $@;
6188
6189     $restore_deactivate_volumes->($storecfg, $devinfo);
6190
6191     rmtree $tmpdir;
6192
6193     if ($err) {
6194         unlink $tmpfn;
6195         $restore_destroy_volumes->($storecfg, $devinfo);
6196         die $err;
6197     }
6198
6199     rename($tmpfn, $conffile) ||
6200         die "unable to commit configuration file '$conffile'\n";
6201
6202     PVE::Cluster::cfs_update(); # make sure we read new file
6203
6204     eval { rescan($vmid, 1); };
6205     warn $@ if $@;
6206 }
6207
6208 sub restore_vma_archive {
6209     my ($archive, $vmid, $user, $opts, $comp) = @_;
6210
6211     my $readfrom = $archive;
6212
6213     my $cfg = PVE::Storage::config();
6214     my $commands = [];
6215     my $bwlimit = $opts->{bwlimit};
6216
6217     my $dbg_cmdstring = '';
6218     my $add_pipe = sub {
6219         my ($cmd) = @_;
6220         push @$commands, $cmd;
6221         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6222         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6223         $readfrom = '-';
6224     };
6225
6226     my $input = undef;
6227     if ($archive eq '-') {
6228         $input = '<&STDIN';
6229     } else {
6230         # If we use a backup from a PVE defined storage we also consider that
6231         # storage's rate limit:
6232         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6233         if (defined($volid)) {
6234             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6235             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6236             if ($readlimit) {
6237                 print STDERR "applying read rate limit: $readlimit\n";
6238                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6239                 $add_pipe->($cstream);
6240             }
6241         }
6242     }
6243
6244     if ($comp) {
6245         my $info = PVE::Storage::decompressor_info('vma', $comp);
6246         my $cmd = $info->{decompressor};
6247         push @$cmd, $readfrom;
6248         $add_pipe->($cmd);
6249     }
6250
6251     my $tmpdir = "/var/tmp/vzdumptmp$$";
6252     rmtree $tmpdir;
6253
6254     # disable interrupts (always do cleanups)
6255     local $SIG{INT} =
6256         local $SIG{TERM} =
6257         local $SIG{QUIT} =
6258         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6259
6260     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6261     POSIX::mkfifo($mapfifo, 0600);
6262     my $fifofh;
6263
6264     my $openfifo = sub {
6265         open($fifofh, '>', $mapfifo) || die $!;
6266     };
6267
6268     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6269
6270     my $oldtimeout;
6271     my $timeout = 5;
6272
6273     my $devinfo = {};
6274
6275     my $rpcenv = PVE::RPCEnvironment::get();
6276
6277     my $conffile = PVE::QemuConfig->config_file($vmid);
6278     my $tmpfn = "$conffile.$$.tmp";
6279
6280     # Note: $oldconf is undef if VM does not exist
6281     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6282     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6283
6284     my %storage_limits;
6285
6286     my $print_devmap = sub {
6287         my $cfgfn = "$tmpdir/qemu-server.conf";
6288
6289         # we can read the config - that is already extracted
6290         my $fh = IO::File->new($cfgfn, "r") ||
6291             "unable to read qemu-server.conf - $!\n";
6292
6293         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6294         if (-f $fwcfgfn) {
6295             my $pve_firewall_dir = '/etc/pve/firewall';
6296             mkdir $pve_firewall_dir; # make sure the dir exists
6297             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6298         }
6299
6300         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6301
6302         foreach my $key (keys %storage_limits) {
6303             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6304             next if !$limit;
6305             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6306             $storage_limits{$key} = $limit * 1024;
6307         }
6308
6309         foreach my $devname (keys %$devinfo) {
6310             die "found no device mapping information for device '$devname'\n"
6311                 if !$devinfo->{$devname}->{virtdev};
6312         }
6313
6314         # create empty/temp config
6315         if ($oldconf) {
6316             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6317             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6318         }
6319
6320         # allocate volumes
6321         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6322
6323         # print restore information to $fifofh
6324         foreach my $virtdev (sort keys %$virtdev_hash) {
6325             my $d = $virtdev_hash->{$virtdev};
6326             next if $d->{is_cloudinit}; # no need to restore cloudinit
6327
6328             my $storeid = $d->{storeid};
6329             my $volid = $d->{volid};
6330
6331             my $map_opts = '';
6332             if (my $limit = $storage_limits{$storeid}) {
6333                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6334             }
6335
6336             my $write_zeros = 1;
6337             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6338                 $write_zeros = 0;
6339             }
6340
6341             my $path = PVE::Storage::path($cfg, $volid);
6342
6343             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6344
6345             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6346         }
6347
6348         $fh->seek(0, 0) || die "seek failed - $!\n";
6349
6350         my $outfd = new IO::File ($tmpfn, "w") ||
6351             die "unable to write config for VM $vmid\n";
6352
6353         my $cookie = { netcount => 0 };
6354         while (defined(my $line = <$fh>)) {
6355             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6356         }
6357
6358         $fh->close();
6359         $outfd->close();
6360     };
6361
6362     eval {
6363         # enable interrupts
6364         local $SIG{INT} =
6365             local $SIG{TERM} =
6366             local $SIG{QUIT} =
6367             local $SIG{HUP} =
6368             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6369         local $SIG{ALRM} = sub { die "got timeout\n"; };
6370
6371         $oldtimeout = alarm($timeout);
6372
6373         my $parser = sub {
6374             my $line = shift;
6375
6376             print "$line\n";
6377
6378             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6379                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6380                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6381             } elsif ($line =~ m/^CTIME: /) {
6382                 # we correctly received the vma config, so we can disable
6383                 # the timeout now for disk allocation (set to 10 minutes, so
6384                 # that we always timeout if something goes wrong)
6385                 alarm(600);
6386                 &$print_devmap();
6387                 print $fifofh "done\n";
6388                 my $tmp = $oldtimeout || 0;
6389                 $oldtimeout = undef;
6390                 alarm($tmp);
6391                 close($fifofh);
6392             }
6393         };
6394
6395         print "restore vma archive: $dbg_cmdstring\n";
6396         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6397     };
6398     my $err = $@;
6399
6400     alarm($oldtimeout) if $oldtimeout;
6401
6402     $restore_deactivate_volumes->($cfg, $devinfo);
6403
6404     unlink $mapfifo;
6405     rmtree $tmpdir;
6406
6407     if ($err) {
6408         unlink $tmpfn;
6409         $restore_destroy_volumes->($cfg, $devinfo);
6410         die $err;
6411     }
6412
6413     rename($tmpfn, $conffile) ||
6414         die "unable to commit configuration file '$conffile'\n";
6415
6416     PVE::Cluster::cfs_update(); # make sure we read new file
6417
6418     eval { rescan($vmid, 1); };
6419     warn $@ if $@;
6420 }
6421
6422 sub restore_tar_archive {
6423     my ($archive, $vmid, $user, $opts) = @_;
6424
6425     if ($archive ne '-') {
6426         my $firstfile = tar_archive_read_firstfile($archive);
6427         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6428             if $firstfile ne 'qemu-server.conf';
6429     }
6430
6431     my $storecfg = PVE::Storage::config();
6432
6433     # avoid zombie disks when restoring over an existing VM -> cleanup first
6434     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6435     # skiplock=1 because qmrestore has set the 'create' lock itself already
6436     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6437     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6438
6439     my $tocmd = "/usr/lib/qemu-server/qmextract";
6440
6441     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6442     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6443     $tocmd .= ' --prealloc' if $opts->{prealloc};
6444     $tocmd .= ' --info' if $opts->{info};
6445
6446     # tar option "xf" does not autodetect compression when read from STDIN,
6447     # so we pipe to zcat
6448     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6449         PVE::Tools::shellquote("--to-command=$tocmd");
6450
6451     my $tmpdir = "/var/tmp/vzdumptmp$$";
6452     mkpath $tmpdir;
6453
6454     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6455     local $ENV{VZDUMP_VMID} = $vmid;
6456     local $ENV{VZDUMP_USER} = $user;
6457
6458     my $conffile = PVE::QemuConfig->config_file($vmid);
6459     my $tmpfn = "$conffile.$$.tmp";
6460
6461     # disable interrupts (always do cleanups)
6462     local $SIG{INT} =
6463         local $SIG{TERM} =
6464         local $SIG{QUIT} =
6465         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6466
6467     eval {
6468         # enable interrupts
6469         local $SIG{INT} =
6470             local $SIG{TERM} =
6471             local $SIG{QUIT} =
6472             local $SIG{HUP} =
6473             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6474
6475         if ($archive eq '-') {
6476             print "extracting archive from STDIN\n";
6477             run_command($cmd, input => "<&STDIN");
6478         } else {
6479             print "extracting archive '$archive'\n";
6480             run_command($cmd);
6481         }
6482
6483         return if $opts->{info};
6484
6485         # read new mapping
6486         my $map = {};
6487         my $statfile = "$tmpdir/qmrestore.stat";
6488         if (my $fd = IO::File->new($statfile, "r")) {
6489             while (defined (my $line = <$fd>)) {
6490                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6491                     $map->{$1} = $2 if $1;
6492                 } else {
6493                     print STDERR "unable to parse line in statfile - $line\n";
6494                 }
6495             }
6496             $fd->close();
6497         }
6498
6499         my $confsrc = "$tmpdir/qemu-server.conf";
6500
6501         my $srcfd = new IO::File($confsrc, "r") ||
6502             die "unable to open file '$confsrc'\n";
6503
6504         my $outfd = new IO::File ($tmpfn, "w") ||
6505             die "unable to write config for VM $vmid\n";
6506
6507         my $cookie = { netcount => 0 };
6508         while (defined (my $line = <$srcfd>)) {
6509             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6510         }
6511
6512         $srcfd->close();
6513         $outfd->close();
6514     };
6515     if (my $err = $@) {
6516         unlink $tmpfn;
6517         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6518         die $err;
6519     }
6520
6521     rmtree $tmpdir;
6522
6523     rename $tmpfn, $conffile ||
6524         die "unable to commit configuration file '$conffile'\n";
6525
6526     PVE::Cluster::cfs_update(); # make sure we read new file
6527
6528     eval { rescan($vmid, 1); };
6529     warn $@ if $@;
6530 };
6531
6532 sub foreach_storage_used_by_vm {
6533     my ($conf, $func) = @_;
6534
6535     my $sidhash = {};
6536
6537     PVE::QemuConfig->foreach_volume($conf, sub {
6538         my ($ds, $drive) = @_;
6539         return if drive_is_cdrom($drive);
6540
6541         my $volid = $drive->{file};
6542
6543         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6544         $sidhash->{$sid} = $sid if $sid;
6545     });
6546
6547     foreach my $sid (sort keys %$sidhash) {
6548         &$func($sid);
6549     }
6550 }
6551
6552 my $qemu_snap_storage = {
6553     rbd => 1,
6554 };
6555 sub do_snapshots_with_qemu {
6556     my ($storecfg, $volid) = @_;
6557
6558     my $storage_name = PVE::Storage::parse_volume_id($volid);
6559     my $scfg = $storecfg->{ids}->{$storage_name};
6560
6561     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6562         return 1;
6563     }
6564
6565     if ($volid =~ m/\.(qcow2|qed)$/){
6566         return 1;
6567     }
6568
6569     return undef;
6570 }
6571
6572 sub qga_check_running {
6573     my ($vmid, $nowarn) = @_;
6574
6575     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6576     if ($@) {
6577         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6578         return 0;
6579     }
6580     return 1;
6581 }
6582
6583 sub template_create {
6584     my ($vmid, $conf, $disk) = @_;
6585
6586     my $storecfg = PVE::Storage::config();
6587
6588     PVE::QemuConfig->foreach_volume($conf, sub {
6589         my ($ds, $drive) = @_;
6590
6591         return if drive_is_cdrom($drive);
6592         return if $disk && $ds ne $disk;
6593
6594         my $volid = $drive->{file};
6595         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6596
6597         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6598         $drive->{file} = $voliddst;
6599         $conf->{$ds} = print_drive($drive);
6600         PVE::QemuConfig->write_config($vmid, $conf);
6601     });
6602 }
6603
6604 sub convert_iscsi_path {
6605     my ($path) = @_;
6606
6607     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6608         my $portal = $1;
6609         my $target = $2;
6610         my $lun = $3;
6611
6612         my $initiator_name = get_initiator_name();
6613
6614         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6615                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6616     }
6617
6618     die "cannot convert iscsi path '$path', unkown format\n";
6619 }
6620
6621 sub qemu_img_convert {
6622     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6623
6624     my $storecfg = PVE::Storage::config();
6625     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6626     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6627
6628     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6629
6630     my $cachemode;
6631     my $src_path;
6632     my $src_is_iscsi = 0;
6633     my $src_format;
6634
6635     if ($src_storeid) {
6636         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6637         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6638         $src_format = qemu_img_format($src_scfg, $src_volname);
6639         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6640         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6641         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6642     } elsif (-f $src_volid) {
6643         $src_path = $src_volid;
6644         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6645             $src_format = $1;
6646         }
6647     }
6648
6649     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6650
6651     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6652     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6653     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6654     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6655
6656     my $cmd = [];
6657     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6658     push @$cmd, '-l', "snapshot.name=$snapname"
6659         if $snapname && $src_format && $src_format eq "qcow2";
6660     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6661     push @$cmd, '-T', $cachemode if defined($cachemode);
6662
6663     if ($src_is_iscsi) {
6664         push @$cmd, '--image-opts';
6665         $src_path = convert_iscsi_path($src_path);
6666     } elsif ($src_format) {
6667         push @$cmd, '-f', $src_format;
6668     }
6669
6670     if ($dst_is_iscsi) {
6671         push @$cmd, '--target-image-opts';
6672         $dst_path = convert_iscsi_path($dst_path);
6673     } else {
6674         push @$cmd, '-O', $dst_format;
6675     }
6676
6677     push @$cmd, $src_path;
6678
6679     if (!$dst_is_iscsi && $is_zero_initialized) {
6680         push @$cmd, "zeroinit:$dst_path";
6681     } else {
6682         push @$cmd, $dst_path;
6683     }
6684
6685     my $parser = sub {
6686         my $line = shift;
6687         if($line =~ m/\((\S+)\/100\%\)/){
6688             my $percent = $1;
6689             my $transferred = int($size * $percent / 100);
6690             my $remaining = $size - $transferred;
6691
6692             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6693         }
6694
6695     };
6696
6697     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6698     my $err = $@;
6699     die "copy failed: $err" if $err;
6700 }
6701
6702 sub qemu_img_format {
6703     my ($scfg, $volname) = @_;
6704
6705     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6706         return $1;
6707     } else {
6708         return "raw";
6709     }
6710 }
6711
6712 sub qemu_drive_mirror {
6713     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6714
6715     $jobs = {} if !$jobs;
6716
6717     my $qemu_target;
6718     my $format;
6719     $jobs->{"drive-$drive"} = {};
6720
6721     if ($dst_volid =~ /^nbd:/) {
6722         $qemu_target = $dst_volid;
6723         $format = "nbd";
6724     } else {
6725         my $storecfg = PVE::Storage::config();
6726         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6727
6728         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6729
6730         $format = qemu_img_format($dst_scfg, $dst_volname);
6731
6732         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6733
6734         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6735     }
6736
6737     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6738     $opts->{format} = $format if $format;
6739
6740     if (defined($src_bitmap)) {
6741         $opts->{sync} = 'incremental';
6742         $opts->{bitmap} = $src_bitmap;
6743         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6744     }
6745
6746     if (defined($bwlimit)) {
6747         $opts->{speed} = $bwlimit * 1024;
6748         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6749     } else {
6750         print "drive mirror is starting for drive-$drive\n";
6751     }
6752
6753     # if a job already runs for this device we get an error, catch it for cleanup
6754     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6755     if (my $err = $@) {
6756         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6757         warn "$@\n" if $@;
6758         die "mirroring error: $err\n";
6759     }
6760
6761     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6762 }
6763
6764 # $completion can be either
6765 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6766 # 'cancel': wait until all jobs are ready, block-job-cancel them
6767 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6768 sub qemu_drive_mirror_monitor {
6769     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6770
6771     $completion //= 'complete';
6772
6773     eval {
6774         my $err_complete = 0;
6775
6776         while (1) {
6777             die "storage migration timed out\n" if $err_complete > 300;
6778
6779             my $stats = mon_cmd($vmid, "query-block-jobs");
6780
6781             my $running_mirror_jobs = {};
6782             foreach my $stat (@$stats) {
6783                 next if $stat->{type} ne 'mirror';
6784                 $running_mirror_jobs->{$stat->{device}} = $stat;
6785             }
6786
6787             my $readycounter = 0;
6788
6789             foreach my $job (keys %$jobs) {
6790
6791                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6792                     print "$job : finished\n";
6793                     delete $jobs->{$job};
6794                     next;
6795                 }
6796
6797                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6798
6799                 my $busy = $running_mirror_jobs->{$job}->{busy};
6800                 my $ready = $running_mirror_jobs->{$job}->{ready};
6801                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6802                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6803                     my $remaining = $total - $transferred;
6804                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6805
6806                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6807                 }
6808
6809                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6810             }
6811
6812             last if scalar(keys %$jobs) == 0;
6813
6814             if ($readycounter == scalar(keys %$jobs)) {
6815                 print "all mirroring jobs are ready \n";
6816                 last if $completion eq 'skip'; #do the complete later
6817
6818                 if ($vmiddst && $vmiddst != $vmid) {
6819                     my $agent_running = $qga && qga_check_running($vmid);
6820                     if ($agent_running) {
6821                         print "freeze filesystem\n";
6822                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6823                     } else {
6824                         print "suspend vm\n";
6825                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6826                     }
6827
6828                     # if we clone a disk for a new target vm, we don't switch the disk
6829                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6830
6831                     if ($agent_running) {
6832                         print "unfreeze filesystem\n";
6833                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6834                     } else {
6835                         print "resume vm\n";
6836                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6837                     }
6838
6839                     last;
6840                 } else {
6841
6842                     foreach my $job (keys %$jobs) {
6843                         # try to switch the disk if source and destination are on the same guest
6844                         print "$job: Completing block job...\n";
6845
6846                         my $op;
6847                         if ($completion eq 'complete') {
6848                             $op = 'block-job-complete';
6849                         } elsif ($completion eq 'cancel') {
6850                             $op = 'block-job-cancel';
6851                         } else {
6852                             die "invalid completion value: $completion\n";
6853                         }
6854                         eval { mon_cmd($vmid, $op, device => $job) };
6855                         if ($@ =~ m/cannot be completed/) {
6856                             print "$job: Block job cannot be completed, try again.\n";
6857                             $err_complete++;
6858                         }else {
6859                             print "$job: Completed successfully.\n";
6860                             $jobs->{$job}->{complete} = 1;
6861                         }
6862                     }
6863                 }
6864             }
6865             sleep 1;
6866         }
6867     };
6868     my $err = $@;
6869
6870     if ($err) {
6871         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6872         die "mirroring error: $err";
6873     }
6874
6875 }
6876
6877 sub qemu_blockjobs_cancel {
6878     my ($vmid, $jobs) = @_;
6879
6880     foreach my $job (keys %$jobs) {
6881         print "$job: Cancelling block job\n";
6882         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6883         $jobs->{$job}->{cancel} = 1;
6884     }
6885
6886     while (1) {
6887         my $stats = mon_cmd($vmid, "query-block-jobs");
6888
6889         my $running_jobs = {};
6890         foreach my $stat (@$stats) {
6891             $running_jobs->{$stat->{device}} = $stat;
6892         }
6893
6894         foreach my $job (keys %$jobs) {
6895
6896             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6897                 print "$job: Done.\n";
6898                 delete $jobs->{$job};
6899             }
6900         }
6901
6902         last if scalar(keys %$jobs) == 0;
6903
6904         sleep 1;
6905     }
6906 }
6907
6908 sub clone_disk {
6909     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6910         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6911
6912     my $newvolid;
6913
6914     if (!$full) {
6915         print "create linked clone of drive $drivename ($drive->{file})\n";
6916         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6917         push @$newvollist, $newvolid;
6918     } else {
6919
6920         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6921         $storeid = $storage if $storage;
6922
6923         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6924         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6925
6926         print "create full clone of drive $drivename ($drive->{file})\n";
6927         my $name = undef;
6928         if (drive_is_cloudinit($drive)) {
6929             $name = "vm-$newvmid-cloudinit";
6930             $name .= ".$dst_format" if $dst_format ne 'raw';
6931             $snapname = undef;
6932             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6933         } elsif ($drivename eq 'efidisk0') {
6934             $size = get_efivars_size($conf);
6935         }
6936         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6937         push @$newvollist, $newvolid;
6938
6939         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6940
6941         if (drive_is_cloudinit($drive)) {
6942             goto no_data_clone;
6943         }
6944
6945         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6946         if (!$running || $snapname) {
6947             # TODO: handle bwlimits
6948             if ($drivename eq 'efidisk0') {
6949                 # the relevant data on the efidisk may be smaller than the source
6950                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6951                 # that is given by the OVMF_VARS.fd
6952                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6953                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6954                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6955             } else {
6956                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6957             }
6958         } else {
6959
6960             my $kvmver = get_running_qemu_version ($vmid);
6961             if (!min_version($kvmver, 2, 7)) {
6962                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6963                     if $drive->{iothread};
6964             }
6965
6966             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6967         }
6968     }
6969
6970 no_data_clone:
6971     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6972
6973     my $disk = $drive;
6974     $disk->{format} = undef;
6975     $disk->{file} = $newvolid;
6976     $disk->{size} = $size;
6977
6978     return $disk;
6979 }
6980
6981 sub get_running_qemu_version {
6982     my ($vmid) = @_;
6983     my $res = mon_cmd($vmid, "query-version");
6984     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6985 }
6986
6987 sub qemu_use_old_bios_files {
6988     my ($machine_type) = @_;
6989
6990     return if !$machine_type;
6991
6992     my $use_old_bios_files = undef;
6993
6994     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6995         $machine_type = $1;
6996         $use_old_bios_files = 1;
6997     } else {
6998         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6999         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7000         # load new efi bios files on migration. So this hack is required to allow
7001         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7002         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7003         $use_old_bios_files = !min_version($version, 2, 4);
7004     }
7005
7006     return ($use_old_bios_files, $machine_type);
7007 }
7008
7009 sub get_efivars_size {
7010     my ($conf) = @_;
7011     my $arch = get_vm_arch($conf);
7012     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7013     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7014     return -s $ovmf_vars;
7015 }
7016
7017 sub update_efidisk_size {
7018     my ($conf) = @_;
7019
7020     return if !defined($conf->{efidisk0});
7021
7022     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7023     $disk->{size} = get_efivars_size($conf);
7024     $conf->{efidisk0} = print_drive($disk);
7025
7026     return;
7027 }
7028
7029 sub create_efidisk($$$$$) {
7030     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7031
7032     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7033     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7034
7035     my $vars_size_b = -s $ovmf_vars;
7036     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7037     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7038     PVE::Storage::activate_volumes($storecfg, [$volid]);
7039
7040     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7041     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7042
7043     return ($volid, $size/1024);
7044 }
7045
7046 sub vm_iothreads_list {
7047     my ($vmid) = @_;
7048
7049     my $res = mon_cmd($vmid, 'query-iothreads');
7050
7051     my $iothreads = {};
7052     foreach my $iothread (@$res) {
7053         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7054     }
7055
7056     return $iothreads;
7057 }
7058
7059 sub scsihw_infos {
7060     my ($conf, $drive) = @_;
7061
7062     my $maxdev = 0;
7063
7064     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7065         $maxdev = 7;
7066     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7067         $maxdev = 1;
7068     } else {
7069         $maxdev = 256;
7070     }
7071
7072     my $controller = int($drive->{index} / $maxdev);
7073     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7074
7075     return ($maxdev, $controller, $controller_prefix);
7076 }
7077
7078 sub windows_version {
7079     my ($ostype) = @_;
7080
7081     return 0 if !$ostype;
7082
7083     my $winversion = 0;
7084
7085     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7086         $winversion = 5;
7087     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7088         $winversion = 6;
7089     } elsif ($ostype =~ m/^win(\d+)$/) {
7090         $winversion = $1;
7091     }
7092
7093     return $winversion;
7094 }
7095
7096 sub resolve_dst_disk_format {
7097         my ($storecfg, $storeid, $src_volname, $format) = @_;
7098         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7099
7100         if (!$format) {
7101             # if no target format is specified, use the source disk format as hint
7102             if ($src_volname) {
7103                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7104                 $format = qemu_img_format($scfg, $src_volname);
7105             } else {
7106                 return $defFormat;
7107             }
7108         }
7109
7110         # test if requested format is supported - else use default
7111         my $supported = grep { $_ eq $format } @$validFormats;
7112         $format = $defFormat if !$supported;
7113         return $format;
7114 }
7115
7116 # NOTE: if this logic changes, please update docs & possibly gui logic
7117 sub find_vmstate_storage {
7118     my ($conf, $storecfg) = @_;
7119
7120     # first, return storage from conf if set
7121     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7122
7123     my ($target, $shared, $local);
7124
7125     foreach_storage_used_by_vm($conf, sub {
7126         my ($sid) = @_;
7127         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7128         my $dst = $scfg->{shared} ? \$shared : \$local;
7129         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7130     });
7131
7132     # second, use shared storage where VM has at least one disk
7133     # third, use local storage where VM has at least one disk
7134     # fall back to local storage
7135     $target = $shared // $local // 'local';
7136
7137     return $target;
7138 }
7139
7140 sub generate_uuid {
7141     my ($uuid, $uuid_str);
7142     UUID::generate($uuid);
7143     UUID::unparse($uuid, $uuid_str);
7144     return $uuid_str;
7145 }
7146
7147 sub generate_smbios1_uuid {
7148     return "uuid=".generate_uuid();
7149 }
7150
7151 sub nbd_stop {
7152     my ($vmid) = @_;
7153
7154     mon_cmd($vmid, 'nbd-server-stop');
7155 }
7156
7157 sub create_reboot_request {
7158     my ($vmid) = @_;
7159     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7160         or die "failed to create reboot trigger file: $!\n";
7161     close($fh);
7162 }
7163
7164 sub clear_reboot_request {
7165     my ($vmid) = @_;
7166     my $path = "/run/qemu-server/$vmid.reboot";
7167     my $res = 0;
7168
7169     $res = unlink($path);
7170     die "could not remove reboot request for $vmid: $!"
7171         if !$res && $! != POSIX::ENOENT;
7172
7173     return $res;
7174 }
7175
7176 # bash completion helper
7177
7178 sub complete_backup_archives {
7179     my ($cmdname, $pname, $cvalue) = @_;
7180
7181     my $cfg = PVE::Storage::config();
7182
7183     my $storeid;
7184
7185     if ($cvalue =~ m/^([^:]+):/) {
7186         $storeid = $1;
7187     }
7188
7189     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7190
7191     my $res = [];
7192     foreach my $id (keys %$data) {
7193         foreach my $item (@{$data->{$id}}) {
7194             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7195             push @$res, $item->{volid} if defined($item->{volid});
7196         }
7197     }
7198
7199     return $res;
7200 }
7201
7202 my $complete_vmid_full = sub {
7203     my ($running) = @_;
7204
7205     my $idlist = vmstatus();
7206
7207     my $res = [];
7208
7209     foreach my $id (keys %$idlist) {
7210         my $d = $idlist->{$id};
7211         if (defined($running)) {
7212             next if $d->{template};
7213             next if $running && $d->{status} ne 'running';
7214             next if !$running && $d->{status} eq 'running';
7215         }
7216         push @$res, $id;
7217
7218     }
7219     return $res;
7220 };
7221
7222 sub complete_vmid {
7223     return &$complete_vmid_full();
7224 }
7225
7226 sub complete_vmid_stopped {
7227     return &$complete_vmid_full(0);
7228 }
7229
7230 sub complete_vmid_running {
7231     return &$complete_vmid_full(1);
7232 }
7233
7234 sub complete_storage {
7235
7236     my $cfg = PVE::Storage::config();
7237     my $ids = $cfg->{ids};
7238
7239     my $res = [];
7240     foreach my $sid (keys %$ids) {
7241         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7242         next if !$ids->{$sid}->{content}->{images};
7243         push @$res, $sid;
7244     }
7245
7246     return $res;
7247 }
7248
7249 sub complete_migration_storage {
7250     my ($cmd, $param, $current_value, $all_args) = @_;
7251
7252     my $targetnode = @$all_args[1];
7253
7254     my $cfg = PVE::Storage::config();
7255     my $ids = $cfg->{ids};
7256
7257     my $res = [];
7258     foreach my $sid (keys %$ids) {
7259         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7260         next if !$ids->{$sid}->{content}->{images};
7261         push @$res, $sid;
7262     }
7263
7264     return $res;
7265 }
7266
7267 1;