PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_HOSTPCI_DEVICES = 16;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         pattern => '[-_.\w\d]+',
 850         optional => 1,
 851     },
 852     queues => {
 853         type => 'integer',
 854         minimum => 0, maximum => 16,
 855         description => 'Number of packet queues to be used on the device.',
 856         optional => 1,
 857     },
 858     rate => {
 859         type => 'number',
 860         minimum => 0,
 861         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 862         optional => 1,
 863     },
 864     tag => {
 865         type => 'integer',
 866         minimum => 1, maximum => 4094,
 867         description => 'VLAN tag to apply to packets on this interface.',
 868         optional => 1,
 869     },
 870     trunks => {
 871         type => 'string',
 872         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 873         description => 'VLAN trunks to pass through this interface.',
 874         format_description => 'vlanid[;vlanid...]',
 875         optional => 1,
 876     },
 877     firewall => {
 878         type => 'boolean',
 879         description => 'Whether this interface should be protected by the firewall.',
 880         optional => 1,
 881     },
 882     link_down => {
 883         type => 'boolean',
 884         description => 'Whether this interface should be disconnected (like pulling the plug).',
 885         optional => 1,
 886     },
 887 };
 888
 889 my $netdesc = {
 890     optional => 1,
 891     type => 'string', format => $net_fmt,
 892     description => "Specify network devices.",
 893 };
 894
 895 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 896
 897 my $ipconfig_fmt = {
 898     ip => {
 899         type => 'string',
 900         format => 'pve-ipv4-config',
 901         format_description => 'IPv4Format/CIDR',
 902         description => 'IPv4 address in CIDR format.',
 903         optional => 1,
 904         default => 'dhcp',
 905     },
 906     gw => {
 907         type => 'string',
 908         format => 'ipv4',
 909         format_description => 'GatewayIPv4',
 910         description => 'Default gateway for IPv4 traffic.',
 911         optional => 1,
 912         requires => 'ip',
 913     },
 914     ip6 => {
 915         type => 'string',
 916         format => 'pve-ipv6-config',
 917         format_description => 'IPv6Format/CIDR',
 918         description => 'IPv6 address in CIDR format.',
 919         optional => 1,
 920         default => 'dhcp',
 921     },
 922     gw6 => {
 923         type => 'string',
 924         format => 'ipv6',
 925         format_description => 'GatewayIPv6',
 926         description => 'Default gateway for IPv6 traffic.',
 927         optional => 1,
 928         requires => 'ip6',
 929     },
 930 };
 931 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 932 my $ipconfigdesc = {
 933     optional => 1,
 934     type => 'string', format => 'pve-qm-ipconfig',
 935     description => <<'EODESCR',
 936 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 937
 938 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 939
 940 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 941 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 942
 943 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 944 EODESCR
 945 };
 946 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 947
 948 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 949     $confdesc->{"net$i"} = $netdesc;
 950     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 951 }
 952
 953 foreach my $key (keys %$confdesc_cloudinit) {
 954     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 955 }
 956
 957 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 958 sub verify_volume_id_or_qm_path {
 959     my ($volid, $noerr) = @_;
 960
 961     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 962         return $volid;
 963     }
 964
 965     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 966     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 967     if ($@) {
 968         return undef if $noerr;
 969         die $@;
 970     }
 971     return $volid;
 972 }
 973
 974 my $usb_fmt = {
 975     host => {
 976         default_key => 1,
 977         type => 'string', format => 'pve-qm-usb-device',
 978         format_description => 'HOSTUSBDEVICE|spice',
 979         description => <<EODESCR,
 980 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 981
 982  'bus-port(.port)*' (decimal numbers) or
 983  'vendor_id:product_id' (hexadeciaml numbers) or
 984  'spice'
 985
 986 You can use the 'lsusb -t' command to list existing usb devices.
 987
 988 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 989
 990 The value 'spice' can be used to add a usb redirection devices for spice.
 991 EODESCR
 992     },
 993     usb3 => {
 994         optional => 1,
 995         type => 'boolean',
 996         description => "Specifies whether if given host option is a USB3 device or port.",
 997         default => 0,
 998     },
 999 };
1000
1001 my $usbdesc = {
1002     optional => 1,
1003     type => 'string', format => $usb_fmt,
1004     description => "Configure an USB device (n is 0 to 4).",
1005 };
1006 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1007
1008 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1009 my $hostpci_fmt = {
1010     host => {
1011         default_key => 1,
1012         type => 'string',
1013         pattern => qr/$PCIRE(;$PCIRE)*/,
1014         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1015         description => <<EODESCR,
1016 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1017 of PCI virtual functions of the host. HOSTPCIID syntax is:
1018
1019 'bus:dev.func' (hexadecimal numbers)
1020
1021 You can us the 'lspci' command to list existing PCI devices.
1022 EODESCR
1023     },
1024     rombar => {
1025         type => 'boolean',
1026         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1027         optional => 1,
1028         default => 1,
1029     },
1030     romfile => {
1031         type => 'string',
1032         pattern => '[^,;]+',
1033         format_description => 'string',
1034         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1035         optional => 1,
1036     },
1037     pcie => {
1038         type => 'boolean',
1039         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1040         optional => 1,
1041         default => 0,
1042     },
1043     'x-vga' => {
1044         type => 'boolean',
1045         description =>  "Enable vfio-vga device support.",
1046         optional => 1,
1047         default => 0,
1048     },
1049     'mdev' => {
1050         type => 'string',
1051         format_description => 'string',
1052         pattern => '[^/\.:]+',
1053         optional => 1,
1054         description => <<EODESCR
1055 The type of mediated device to use.
1056 An instance of this type will be created on startup of the VM and
1057 will be cleaned up when the VM stops.
1058 EODESCR
1059     }
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1121     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1122 }
1123
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1125     $confdesc->{"usb$i"} = $usbdesc;
1126 }
1127
1128 my $kvm_api_version = 0;
1129
1130 sub kvm_version {
1131     return $kvm_api_version if $kvm_api_version;
1132
1133     open my $fh, '<', '/dev/kvm'
1134         or return undef;
1135
1136     # 0xae00 => KVM_GET_API_VERSION
1137     $kvm_api_version = ioctl($fh, 0xae00, 0);
1138
1139     return $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version = {};
1143 my $kvm_mtime = {};
1144
1145 sub kvm_user_version {
1146     my ($binary) = @_;
1147
1148     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1149     my $st = stat($binary);
1150
1151     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1152     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1153         $cachedmtime == $st->mtime;
1154
1155     $kvm_user_version->{$binary} = 'unknown';
1156     $kvm_mtime->{$binary} = $st->mtime;
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version->{$binary} = $2;
1162         }
1163     };
1164
1165     eval { run_command([$binary, '--version'], outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version->{$binary};
1169
1170 }
1171
1172 sub kernel_has_vhost_net {
1173     return -c '/dev/vhost-net';
1174 }
1175
1176 sub option_exists {
1177     my $key = shift;
1178     return defined($confdesc->{$key});
1179 }
1180
1181 my $cdrom_path;
1182 sub get_cdrom_path {
1183
1184     return  $cdrom_path if $cdrom_path;
1185
1186     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1187     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1188     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1189 }
1190
1191 sub get_iso_path {
1192     my ($storecfg, $vmid, $cdrom) = @_;
1193
1194     if ($cdrom eq 'cdrom') {
1195         return get_cdrom_path();
1196     } elsif ($cdrom eq 'none') {
1197         return '';
1198     } elsif ($cdrom =~ m|^/|) {
1199         return $cdrom;
1200     } else {
1201         return PVE::Storage::path($storecfg, $cdrom);
1202     }
1203 }
1204
1205 # try to convert old style file names to volume IDs
1206 sub filename_to_volume_id {
1207     my ($vmid, $file, $media) = @_;
1208
1209      if (!($file eq 'none' || $file eq 'cdrom' ||
1210           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1211
1212         return undef if $file =~ m|/|;
1213
1214         if ($media && $media eq 'cdrom') {
1215             $file = "local:iso/$file";
1216         } else {
1217             $file = "local:$vmid/$file";
1218         }
1219     }
1220
1221     return $file;
1222 }
1223
1224 sub verify_media_type {
1225     my ($opt, $vtype, $media) = @_;
1226
1227     return if !$media;
1228
1229     my $etype;
1230     if ($media eq 'disk') {
1231         $etype = 'images';
1232     } elsif ($media eq 'cdrom') {
1233         $etype = 'iso';
1234     } else {
1235         die "internal error";
1236     }
1237
1238     return if ($vtype eq $etype);
1239
1240     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1241 }
1242
1243 sub cleanup_drive_path {
1244     my ($opt, $storecfg, $drive) = @_;
1245
1246     # try to convert filesystem paths to volume IDs
1247
1248     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1249         ($drive->{file} !~ m|^/dev/.+|) &&
1250         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1251         ($drive->{file} !~ m/^\d+$/)) {
1252         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1253         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1254         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1255         verify_media_type($opt, $vtype, $drive->{media});
1256         $drive->{file} = $volid;
1257     }
1258
1259     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1260 }
1261
1262 sub parse_hotplug_features {
1263     my ($data) = @_;
1264
1265     my $res = {};
1266
1267     return $res if $data eq '0';
1268
1269     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1270
1271     foreach my $feature (PVE::Tools::split_list($data)) {
1272         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1273             $res->{$1} = 1;
1274         } else {
1275             die "invalid hotplug feature '$feature'\n";
1276         }
1277     }
1278     return $res;
1279 }
1280
1281 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1282 sub pve_verify_hotplug_features {
1283     my ($value, $noerr) = @_;
1284
1285     return $value if parse_hotplug_features($value);
1286
1287     return undef if $noerr;
1288
1289     die "unable to parse hotplug option\n";
1290 }
1291
1292 sub scsi_inquiry {
1293     my($fh, $noerr) = @_;
1294
1295     my $SG_IO = 0x2285;
1296     my $SG_GET_VERSION_NUM = 0x2282;
1297
1298     my $versionbuf = "\x00" x 8;
1299     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1300     if (!$ret) {
1301         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1302         return undef;
1303     }
1304     my $version = unpack("I", $versionbuf);
1305     if ($version < 30000) {
1306         die "scsi generic interface too old\n"  if !$noerr;
1307         return undef;
1308     }
1309
1310     my $buf = "\x00" x 36;
1311     my $sensebuf = "\x00" x 8;
1312     my $cmd = pack("C x3 C x1", 0x12, 36);
1313
1314     # see /usr/include/scsi/sg.h
1315     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1316
1317     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1318                       length($sensebuf), 0, length($buf), $buf,
1319                       $cmd, $sensebuf, 6000);
1320
1321     $ret = ioctl($fh, $SG_IO, $packet);
1322     if (!$ret) {
1323         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1324         return undef;
1325     }
1326
1327     my @res = unpack($sg_io_hdr_t, $packet);
1328     if ($res[17] || $res[18]) {
1329         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1330         return undef;
1331     }
1332
1333     my $res = {};
1334     (my $byte0, my $byte1, $res->{vendor},
1335      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1336
1337     $res->{removable} = $byte1 & 128 ? 1 : 0;
1338     $res->{type} = $byte0 & 31;
1339
1340     return $res;
1341 }
1342
1343 sub path_is_scsi {
1344     my ($path) = @_;
1345
1346     my $fh = IO::File->new("+<$path") || return undef;
1347     my $res = scsi_inquiry($fh, 1);
1348     close($fh);
1349
1350     return $res;
1351 }
1352
1353 sub print_tabletdevice_full {
1354     my ($conf, $arch) = @_;
1355
1356     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1357
1358     # we use uhci for old VMs because tablet driver was buggy in older qemu
1359     my $usbbus;
1360     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1361         $usbbus = 'ehci';
1362     } else {
1363         $usbbus = 'uhci';
1364     }
1365
1366     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1367 }
1368
1369 sub print_keyboarddevice_full {
1370     my ($conf, $arch, $machine) = @_;
1371
1372     return undef if $arch ne 'aarch64';
1373
1374     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1375 }
1376
1377 sub print_drivedevice_full {
1378     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1379
1380     my $device = '';
1381     my $maxdev = 0;
1382
1383     if ($drive->{interface} eq 'virtio') {
1384         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1385         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1386         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1387     } elsif ($drive->{interface} eq 'scsi') {
1388
1389         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1390         my $unit = $drive->{index} % $maxdev;
1391         my $devicetype = 'hd';
1392         my $path = '';
1393         if (drive_is_cdrom($drive)) {
1394             $devicetype = 'cd';
1395         } else {
1396             if ($drive->{file} =~ m|^/|) {
1397                 $path = $drive->{file};
1398                 if (my $info = path_is_scsi($path)) {
1399                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1400                         $devicetype = 'block';
1401                     } elsif ($info->{type} == 1) { # tape
1402                         $devicetype = 'generic';
1403                     }
1404                 }
1405             } else {
1406                  $path = PVE::Storage::path($storecfg, $drive->{file});
1407             }
1408
1409             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1410             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1411             if ($path =~ m/^iscsi\:\/\// &&
1412                !min_version($version, 4, 1)) {
1413                 $devicetype = 'generic';
1414             }
1415         }
1416
1417         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1421         }
1422
1423         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1424             $device .= ",rotation_rate=1";
1425         }
1426         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1427
1428     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1429         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1430         my $controller = int($drive->{index} / $maxdev);
1431         my $unit = $drive->{index} % $maxdev;
1432         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1433
1434         $device = "ide-$devicetype";
1435         if ($drive->{interface} eq 'ide') {
1436             $device .= ",bus=ide.$controller,unit=$unit";
1437         } else {
1438             $device .= ",bus=ahci$controller.$unit";
1439         }
1440         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1441
1442         if ($devicetype eq 'hd') {
1443             if (my $model = $drive->{model}) {
1444                 $model = URI::Escape::uri_unescape($model);
1445                 $device .= ",model=$model";
1446             }
1447             if ($drive->{ssd}) {
1448                 $device .= ",rotation_rate=1";
1449             }
1450         }
1451         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1452     } elsif ($drive->{interface} eq 'usb') {
1453         die "implement me";
1454         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1455     } else {
1456         die "unsupported interface type";
1457     }
1458
1459     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1460
1461     if (my $serial = $drive->{serial}) {
1462         $serial = URI::Escape::uri_unescape($serial);
1463         $device .= ",serial=$serial";
1464     }
1465
1466
1467     return $device;
1468 }
1469
1470 sub get_initiator_name {
1471     my $initiator;
1472
1473     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1474     while (defined(my $line = <$fh>)) {
1475         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1476         $initiator = $1;
1477         last;
1478     }
1479     $fh->close();
1480
1481     return $initiator;
1482 }
1483
1484 sub print_drive_commandline_full {
1485     my ($storecfg, $vmid, $drive) = @_;
1486
1487     my $path;
1488     my $volid = $drive->{file};
1489     my $format;
1490
1491     if (drive_is_cdrom($drive)) {
1492         $path = get_iso_path($storecfg, $vmid, $volid);
1493     } else {
1494         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1495         if ($storeid) {
1496             $path = PVE::Storage::path($storecfg, $volid);
1497             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1498             $format = qemu_img_format($scfg, $volname);
1499         } else {
1500             $path = $volid;
1501             $format = "raw";
1502         }
1503    }
1504
1505     my $opts = '';
1506     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1507     foreach my $o (@qemu_drive_options) {
1508         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1509     }
1510
1511     # snapshot only accepts on|off
1512     if (defined($drive->{snapshot})) {
1513         my $v = $drive->{snapshot} ? 'on' : 'off';
1514         $opts .= ",snapshot=$v";
1515     }
1516
1517     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1518         my ($dir, $qmpname) = @$type;
1519         if (my $v = $drive->{"mbps$dir"}) {
1520             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1521         }
1522         if (my $v = $drive->{"mbps${dir}_max"}) {
1523             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1524         }
1525         if (my $v = $drive->{"bps${dir}_max_length"}) {
1526             $opts .= ",throttling.bps$qmpname-max-length=$v";
1527         }
1528         if (my $v = $drive->{"iops${dir}"}) {
1529             $opts .= ",throttling.iops$qmpname=$v";
1530         }
1531         if (my $v = $drive->{"iops${dir}_max"}) {
1532             $opts .= ",throttling.iops$qmpname-max=$v";
1533         }
1534         if (my $v = $drive->{"iops${dir}_max_length"}) {
1535             $opts .= ",throttling.iops$qmpname-max-length=$v";
1536         }
1537     }
1538
1539     $opts .= ",format=$format" if $format && !$drive->{format};
1540
1541     my $cache_direct = 0;
1542
1543     if (my $cache = $drive->{cache}) {
1544         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1545     } elsif (!drive_is_cdrom($drive)) {
1546         $opts .= ",cache=none";
1547         $cache_direct = 1;
1548     }
1549
1550     # aio native works only with O_DIRECT
1551     if (!$drive->{aio}) {
1552         if($cache_direct) {
1553             $opts .= ",aio=native";
1554         } else {
1555             $opts .= ",aio=threads";
1556         }
1557     }
1558
1559     if (!drive_is_cdrom($drive)) {
1560         my $detectzeroes;
1561         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1562             $detectzeroes = 'off';
1563         } elsif ($drive->{discard}) {
1564             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1565         } else {
1566             # This used to be our default with discard not being specified:
1567             $detectzeroes = 'on';
1568         }
1569         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1570     }
1571
1572     my $pathinfo = $path ? "file=$path," : '';
1573
1574     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1575 }
1576
1577 sub print_netdevice_full {
1578     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1579
1580     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1581
1582     my $device = $net->{model};
1583     if ($net->{model} eq 'virtio') {
1584          $device = 'virtio-net-pci';
1585      };
1586
1587     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1588     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1589     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1590         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1591         my $vectors = $net->{queues} * 2 + 2;
1592         $tmpstr .= ",vectors=$vectors,mq=on";
1593     }
1594     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1595
1596     if ($use_old_bios_files) {
1597         my $romfile;
1598         if ($device eq 'virtio-net-pci') {
1599             $romfile = 'pxe-virtio.rom';
1600         } elsif ($device eq 'e1000') {
1601             $romfile = 'pxe-e1000.rom';
1602         } elsif ($device eq 'ne2k') {
1603             $romfile = 'pxe-ne2k_pci.rom';
1604         } elsif ($device eq 'pcnet') {
1605             $romfile = 'pxe-pcnet.rom';
1606         } elsif ($device eq 'rtl8139') {
1607             $romfile = 'pxe-rtl8139.rom';
1608         }
1609         $tmpstr .= ",romfile=$romfile" if $romfile;
1610     }
1611
1612     return $tmpstr;
1613 }
1614
1615 sub print_netdev_full {
1616     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1617
1618     my $i = '';
1619     if ($netid =~ m/^net(\d+)$/) {
1620         $i = int($1);
1621     }
1622
1623     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1624
1625     my $ifname = "tap${vmid}i$i";
1626
1627     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1628     die "interface name '$ifname' is too long (max 15 character)\n"
1629         if length($ifname) >= 16;
1630
1631     my $vhostparam = '';
1632     if (is_native($arch)) {
1633         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1634     }
1635
1636     my $vmname = $conf->{name} || "vm$vmid";
1637
1638     my $netdev = "";
1639     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1640
1641     if ($net->{bridge}) {
1642         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1643     } else {
1644         $netdev = "type=user,id=$netid,hostname=$vmname";
1645     }
1646
1647     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1648
1649     return $netdev;
1650 }
1651
1652 my $vga_map = {
1653     'cirrus' => 'cirrus-vga',
1654     'std' => 'VGA',
1655     'vmware' => 'vmware-svga',
1656     'virtio' => 'virtio-vga',
1657 };
1658
1659 sub print_vga_device {
1660     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1661
1662     my $type = $vga_map->{$vga->{type}};
1663     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1664         $type = 'virtio-gpu';
1665     }
1666     my $vgamem_mb = $vga->{memory};
1667
1668     my $max_outputs = '';
1669     if ($qxlnum) {
1670         $type = $id ? 'qxl' : 'qxl-vga';
1671
1672         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1673             # set max outputs so linux can have up to 4 qxl displays with one device
1674             if (min_version($machine_version, 4, 1)) {
1675                 $max_outputs = ",max_outputs=4";
1676             }
1677         }
1678     }
1679
1680     die "no devicetype for $vga->{type}\n" if !$type;
1681
1682     my $memory = "";
1683     if ($vgamem_mb) {
1684         if ($vga->{type} eq 'virtio') {
1685             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1686             $memory = ",max_hostmem=$bytes";
1687         } elsif ($qxlnum) {
1688             # from https://www.spice-space.org/multiple-monitors.html
1689             $memory = ",vgamem_mb=$vga->{memory}";
1690             my $ram = $vgamem_mb * 4;
1691             my $vram = $vgamem_mb * 2;
1692             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1693         } else {
1694             $memory = ",vgamem_mb=$vga->{memory}";
1695         }
1696     } elsif ($qxlnum && $id) {
1697         $memory = ",ram_size=67108864,vram_size=33554432";
1698     }
1699
1700     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1701     my $vgaid = "vga" . ($id // '');
1702     my $pciaddr;
1703
1704     if ($q35 && $vgaid eq 'vga') {
1705         # the first display uses pcie.0 bus on q35 machines
1706         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1707     } else {
1708         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1709     }
1710
1711     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1712 }
1713
1714 sub parse_number_sets {
1715     my ($set) = @_;
1716     my $res = [];
1717     foreach my $part (split(/;/, $set)) {
1718         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1719             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1720             push @$res, [ $1, $2 ];
1721         } else {
1722             die "invalid range: $part\n";
1723         }
1724     }
1725     return $res;
1726 }
1727
1728 sub parse_numa {
1729     my ($data) = @_;
1730
1731     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1732     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1733     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1734     return $res;
1735 }
1736
1737 sub parse_hostpci {
1738     my ($value) = @_;
1739
1740     return undef if !$value;
1741
1742     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1743
1744     my @idlist = split(/;/, $res->{host});
1745     delete $res->{host};
1746     foreach my $id (@idlist) {
1747         my $devs = PVE::SysFSTools::lspci($id);
1748         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1749         push @{$res->{pciid}}, @$devs;
1750     }
1751     return $res;
1752 }
1753
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1755 sub parse_net {
1756     my ($data) = @_;
1757
1758     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1759     if ($@) {
1760         warn $@;
1761         return undef;
1762     }
1763     if (!defined($res->{macaddr})) {
1764         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1765         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1766     }
1767     return $res;
1768 }
1769
1770 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1771 sub parse_ipconfig {
1772     my ($data) = @_;
1773
1774     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1775     if ($@) {
1776         warn $@;
1777         return undef;
1778     }
1779
1780     if ($res->{gw} && !$res->{ip}) {
1781         warn 'gateway specified without specifying an IP address';
1782         return undef;
1783     }
1784     if ($res->{gw6} && !$res->{ip6}) {
1785         warn 'IPv6 gateway specified without specifying an IPv6 address';
1786         return undef;
1787     }
1788     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1789         warn 'gateway specified together with DHCP';
1790         return undef;
1791     }
1792     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1793         # gw6 + auto/dhcp
1794         warn "IPv6 gateway specified together with $res->{ip6} address";
1795         return undef;
1796     }
1797
1798     if (!$res->{ip} && !$res->{ip6}) {
1799         return { ip => 'dhcp', ip6 => 'dhcp' };
1800     }
1801
1802     return $res;
1803 }
1804
1805 sub print_net {
1806     my $net = shift;
1807
1808     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1809 }
1810
1811 sub add_random_macs {
1812     my ($settings) = @_;
1813
1814     foreach my $opt (keys %$settings) {
1815         next if $opt !~ m/^net(\d+)$/;
1816         my $net = parse_net($settings->{$opt});
1817         next if !$net;
1818         $settings->{$opt} = print_net($net);
1819     }
1820 }
1821
1822 sub vm_is_volid_owner {
1823     my ($storecfg, $vmid, $volid) = @_;
1824
1825     if ($volid !~  m|^/|) {
1826         my ($path, $owner);
1827         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1828         if ($owner && ($owner == $vmid)) {
1829             return 1;
1830         }
1831     }
1832
1833     return undef;
1834 }
1835
1836 sub vmconfig_register_unused_drive {
1837     my ($storecfg, $vmid, $conf, $drive) = @_;
1838
1839     if (drive_is_cloudinit($drive)) {
1840         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1841         warn $@ if $@;
1842     } elsif (!drive_is_cdrom($drive)) {
1843         my $volid = $drive->{file};
1844         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1845             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1846         }
1847     }
1848 }
1849
1850 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1851 my $smbios1_fmt = {
1852     uuid => {
1853         type => 'string',
1854         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1855         format_description => 'UUID',
1856         description => "Set SMBIOS1 UUID.",
1857         optional => 1,
1858     },
1859     version => {
1860         type => 'string',
1861         pattern => '[A-Za-z0-9+\/]+={0,2}',
1862         format_description => 'Base64 encoded string',
1863         description => "Set SMBIOS1 version.",
1864         optional => 1,
1865     },
1866     serial => {
1867         type => 'string',
1868         pattern => '[A-Za-z0-9+\/]+={0,2}',
1869         format_description => 'Base64 encoded string',
1870         description => "Set SMBIOS1 serial number.",
1871         optional => 1,
1872     },
1873     manufacturer => {
1874         type => 'string',
1875         pattern => '[A-Za-z0-9+\/]+={0,2}',
1876         format_description => 'Base64 encoded string',
1877         description => "Set SMBIOS1 manufacturer.",
1878         optional => 1,
1879     },
1880     product => {
1881         type => 'string',
1882         pattern => '[A-Za-z0-9+\/]+={0,2}',
1883         format_description => 'Base64 encoded string',
1884         description => "Set SMBIOS1 product ID.",
1885         optional => 1,
1886     },
1887     sku => {
1888         type => 'string',
1889         pattern => '[A-Za-z0-9+\/]+={0,2}',
1890         format_description => 'Base64 encoded string',
1891         description => "Set SMBIOS1 SKU string.",
1892         optional => 1,
1893     },
1894     family => {
1895         type => 'string',
1896         pattern => '[A-Za-z0-9+\/]+={0,2}',
1897         format_description => 'Base64 encoded string',
1898         description => "Set SMBIOS1 family string.",
1899         optional => 1,
1900     },
1901     base64 => {
1902         type => 'boolean',
1903         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1904         optional => 1,
1905     },
1906 };
1907
1908 sub parse_smbios1 {
1909     my ($data) = @_;
1910
1911     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1912     warn $@ if $@;
1913     return $res;
1914 }
1915
1916 sub print_smbios1 {
1917     my ($smbios1) = @_;
1918     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1919 }
1920
1921 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1922
1923 sub parse_watchdog {
1924     my ($value) = @_;
1925
1926     return undef if !$value;
1927
1928     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1929     warn $@ if $@;
1930     return $res;
1931 }
1932
1933 sub parse_guest_agent {
1934     my ($value) = @_;
1935
1936     return {} if !defined($value->{agent});
1937
1938     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1939     warn $@ if $@;
1940
1941     # if the agent is disabled ignore the other potentially set properties
1942     return {} if !$res->{enabled};
1943     return $res;
1944 }
1945
1946 sub parse_vga {
1947     my ($value) = @_;
1948
1949     return {} if !$value;
1950     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1951     warn $@ if $@;
1952     return $res;
1953 }
1954
1955 sub parse_rng {
1956     my ($value) = @_;
1957
1958     return undef if !$value;
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1961     warn $@ if $@;
1962     return $res;
1963 }
1964
1965 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1966 sub verify_usb_device {
1967     my ($value, $noerr) = @_;
1968
1969     return $value if parse_usb_device($value);
1970
1971     return undef if $noerr;
1972
1973     die "unable to parse usb device\n";
1974 }
1975
1976 # add JSON properties for create and set function
1977 sub json_config_properties {
1978     my $prop = shift;
1979
1980     foreach my $opt (keys %$confdesc) {
1981         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1982             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1983         $prop->{$opt} = $confdesc->{$opt};
1984     }
1985
1986     return $prop;
1987 }
1988
1989 # return copy of $confdesc_cloudinit to generate documentation
1990 sub cloudinit_config_properties {
1991
1992     return dclone($confdesc_cloudinit);
1993 }
1994
1995 sub check_type {
1996     my ($key, $value) = @_;
1997
1998     die "unknown setting '$key'\n" if !$confdesc->{$key};
1999
2000     my $type = $confdesc->{$key}->{type};
2001
2002     if (!defined($value)) {
2003         die "got undefined value\n";
2004     }
2005
2006     if ($value =~ m/[\n\r]/) {
2007         die "property contains a line feed\n";
2008     }
2009
2010     if ($type eq 'boolean') {
2011         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2012         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2013         die "type check ('boolean') failed - got '$value'\n";
2014     } elsif ($type eq 'integer') {
2015         return int($1) if $value =~ m/^(\d+)$/;
2016         die "type check ('integer') failed - got '$value'\n";
2017     } elsif ($type eq 'number') {
2018         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2019         die "type check ('number') failed - got '$value'\n";
2020     } elsif ($type eq 'string') {
2021         if (my $fmt = $confdesc->{$key}->{format}) {
2022             PVE::JSONSchema::check_format($fmt, $value);
2023             return $value;
2024         }
2025         $value =~ s/^\"(.*)\"$/$1/;
2026         return $value;
2027     } else {
2028         die "internal error"
2029     }
2030 }
2031
2032 sub destroy_vm {
2033     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2034
2035     my $conf = PVE::QemuConfig->load_config($vmid);
2036
2037     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2038
2039     if ($conf->{template}) {
2040         # check if any base image is still used by a linked clone
2041         PVE::QemuConfig->foreach_volume($conf, sub {
2042                 my ($ds, $drive) = @_;
2043                 return if drive_is_cdrom($drive);
2044
2045                 my $volid = $drive->{file};
2046                 return if !$volid || $volid =~ m|^/|;
2047
2048                 die "base volume '$volid' is still in use by linked cloned\n"
2049                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2050
2051         });
2052     }
2053
2054     # only remove disks owned by this VM
2055     PVE::QemuConfig->foreach_volume($conf, sub {
2056         my ($ds, $drive) = @_;
2057         return if drive_is_cdrom($drive, 1);
2058
2059         my $volid = $drive->{file};
2060         return if !$volid || $volid =~ m|^/|;
2061
2062         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2063         return if !$path || !$owner || ($owner != $vmid);
2064
2065         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2066         warn "Could not remove disk '$volid', check manually: $@" if $@;
2067     });
2068
2069     # also remove unused disk
2070     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2071     PVE::Storage::foreach_volid($vmdisks, sub {
2072         my ($volid, $sid, $volname, $d) = @_;
2073         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2074         warn $@ if $@;
2075     });
2076
2077     if (defined $replacement_conf) {
2078         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2079     } else {
2080         PVE::QemuConfig->destroy_config($vmid);
2081     }
2082 }
2083
2084 sub parse_vm_config {
2085     my ($filename, $raw) = @_;
2086
2087     return undef if !defined($raw);
2088
2089     my $res = {
2090         digest => Digest::SHA::sha1_hex($raw),
2091         snapshots => {},
2092         pending => {},
2093     };
2094
2095     $filename =~ m|/qemu-server/(\d+)\.conf$|
2096         || die "got strange filename '$filename'";
2097
2098     my $vmid = $1;
2099
2100     my $conf = $res;
2101     my $descr;
2102     my $section = '';
2103
2104     my @lines = split(/\n/, $raw);
2105     foreach my $line (@lines) {
2106         next if $line =~ m/^\s*$/;
2107
2108         if ($line =~ m/^\[PENDING\]\s*$/i) {
2109             $section = 'pending';
2110             if (defined($descr)) {
2111                 $descr =~ s/\s+$//;
2112                 $conf->{description} = $descr;
2113             }
2114             $descr = undef;
2115             $conf = $res->{$section} = {};
2116             next;
2117
2118         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2119             $section = $1;
2120             if (defined($descr)) {
2121                 $descr =~ s/\s+$//;
2122                 $conf->{description} = $descr;
2123             }
2124             $descr = undef;
2125             $conf = $res->{snapshots}->{$section} = {};
2126             next;
2127         }
2128
2129         if ($line =~ m/^\#(.*)\s*$/) {
2130             $descr = '' if !defined($descr);
2131             $descr .= PVE::Tools::decode_text($1) . "\n";
2132             next;
2133         }
2134
2135         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2136             $descr = '' if !defined($descr);
2137             $descr .= PVE::Tools::decode_text($2);
2138         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2139             $conf->{snapstate} = $1;
2140         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2141             my $key = $1;
2142             my $value = $2;
2143             $conf->{$key} = $value;
2144         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2145             my $value = $1;
2146             if ($section eq 'pending') {
2147                 $conf->{delete} = $value; # we parse this later
2148             } else {
2149                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2150             }
2151         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2152             my $key = $1;
2153             my $value = $2;
2154             eval { $value = check_type($key, $value); };
2155             if ($@) {
2156                 warn "vm $vmid - unable to parse value of '$key' - $@";
2157             } else {
2158                 $key = 'ide2' if $key eq 'cdrom';
2159                 my $fmt = $confdesc->{$key}->{format};
2160                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2161                     my $v = parse_drive($key, $value);
2162                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2163                         $v->{file} = $volid;
2164                         $value = print_drive($v);
2165                     } else {
2166                         warn "vm $vmid - unable to parse value of '$key'\n";
2167                         next;
2168                     }
2169                 }
2170
2171                 $conf->{$key} = $value;
2172             }
2173         }
2174     }
2175
2176     if (defined($descr)) {
2177         $descr =~ s/\s+$//;
2178         $conf->{description} = $descr;
2179     }
2180     delete $res->{snapstate}; # just to be sure
2181
2182     return $res;
2183 }
2184
2185 sub write_vm_config {
2186     my ($filename, $conf) = @_;
2187
2188     delete $conf->{snapstate}; # just to be sure
2189
2190     if ($conf->{cdrom}) {
2191         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2192         $conf->{ide2} = $conf->{cdrom};
2193         delete $conf->{cdrom};
2194     }
2195
2196     # we do not use 'smp' any longer
2197     if ($conf->{sockets}) {
2198         delete $conf->{smp};
2199     } elsif ($conf->{smp}) {
2200         $conf->{sockets} = $conf->{smp};
2201         delete $conf->{cores};
2202         delete $conf->{smp};
2203     }
2204
2205     my $used_volids = {};
2206
2207     my $cleanup_config = sub {
2208         my ($cref, $pending, $snapname) = @_;
2209
2210         foreach my $key (keys %$cref) {
2211             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2212                 $key eq 'snapstate' || $key eq 'pending';
2213             my $value = $cref->{$key};
2214             if ($key eq 'delete') {
2215                 die "propertry 'delete' is only allowed in [PENDING]\n"
2216                     if !$pending;
2217                 # fixme: check syntax?
2218                 next;
2219             }
2220             eval { $value = check_type($key, $value); };
2221             die "unable to parse value of '$key' - $@" if $@;
2222
2223             $cref->{$key} = $value;
2224
2225             if (!$snapname && is_valid_drivename($key)) {
2226                 my $drive = parse_drive($key, $value);
2227                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2228             }
2229         }
2230     };
2231
2232     &$cleanup_config($conf);
2233
2234     &$cleanup_config($conf->{pending}, 1);
2235
2236     foreach my $snapname (keys %{$conf->{snapshots}}) {
2237         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2238         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2239     }
2240
2241     # remove 'unusedX' settings if we re-add a volume
2242     foreach my $key (keys %$conf) {
2243         my $value = $conf->{$key};
2244         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2245             delete $conf->{$key};
2246         }
2247     }
2248
2249     my $generate_raw_config = sub {
2250         my ($conf, $pending) = @_;
2251
2252         my $raw = '';
2253
2254         # add description as comment to top of file
2255         if (defined(my $descr = $conf->{description})) {
2256             if ($descr) {
2257                 foreach my $cl (split(/\n/, $descr)) {
2258                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2259                 }
2260             } else {
2261                 $raw .= "#\n" if $pending;
2262             }
2263         }
2264
2265         foreach my $key (sort keys %$conf) {
2266             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2267             $raw .= "$key: $conf->{$key}\n";
2268         }
2269         return $raw;
2270     };
2271
2272     my $raw = &$generate_raw_config($conf);
2273
2274     if (scalar(keys %{$conf->{pending}})){
2275         $raw .= "\n[PENDING]\n";
2276         $raw .= &$generate_raw_config($conf->{pending}, 1);
2277     }
2278
2279     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2280         $raw .= "\n[$snapname]\n";
2281         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2282     }
2283
2284     return $raw;
2285 }
2286
2287 sub load_defaults {
2288
2289     my $res = {};
2290
2291     # we use static defaults from our JSON schema configuration
2292     foreach my $key (keys %$confdesc) {
2293         if (defined(my $default = $confdesc->{$key}->{default})) {
2294             $res->{$key} = $default;
2295         }
2296     }
2297
2298     return $res;
2299 }
2300
2301 sub config_list {
2302     my $vmlist = PVE::Cluster::get_vmlist();
2303     my $res = {};
2304     return $res if !$vmlist || !$vmlist->{ids};
2305     my $ids = $vmlist->{ids};
2306     my $nodename = nodename();
2307
2308     foreach my $vmid (keys %$ids) {
2309         my $d = $ids->{$vmid};
2310         next if !$d->{node} || $d->{node} ne $nodename;
2311         next if !$d->{type} || $d->{type} ne 'qemu';
2312         $res->{$vmid}->{exists} = 1;
2313     }
2314     return $res;
2315 }
2316
2317 # test if VM uses local resources (to prevent migration)
2318 sub check_local_resources {
2319     my ($conf, $noerr) = @_;
2320
2321     my @loc_res = ();
2322
2323     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2324     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2325
2326     push @loc_res, "ivshmem" if $conf->{ivshmem};
2327
2328     foreach my $k (keys %$conf) {
2329         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2330         # sockets are safe: they will recreated be on the target side post-migrate
2331         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2332         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2333     }
2334
2335     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2336
2337     return \@loc_res;
2338 }
2339
2340 # check if used storages are available on all nodes (use by migrate)
2341 sub check_storage_availability {
2342     my ($storecfg, $conf, $node) = @_;
2343
2344     PVE::QemuConfig->foreach_volume($conf, sub {
2345         my ($ds, $drive) = @_;
2346
2347         my $volid = $drive->{file};
2348         return if !$volid;
2349
2350         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2351         return if !$sid;
2352
2353         # check if storage is available on both nodes
2354         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2355         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2356    });
2357 }
2358
2359 # list nodes where all VM images are available (used by has_feature API)
2360 sub shared_nodes {
2361     my ($conf, $storecfg) = @_;
2362
2363     my $nodelist = PVE::Cluster::get_nodelist();
2364     my $nodehash = { map { $_ => 1 } @$nodelist };
2365     my $nodename = nodename();
2366
2367     PVE::QemuConfig->foreach_volume($conf, sub {
2368         my ($ds, $drive) = @_;
2369
2370         my $volid = $drive->{file};
2371         return if !$volid;
2372
2373         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2374         if ($storeid) {
2375             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2376             if ($scfg->{disable}) {
2377                 $nodehash = {};
2378             } elsif (my $avail = $scfg->{nodes}) {
2379                 foreach my $node (keys %$nodehash) {
2380                     delete $nodehash->{$node} if !$avail->{$node};
2381                 }
2382             } elsif (!$scfg->{shared}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     delete $nodehash->{$node} if $node ne $nodename
2385                 }
2386             }
2387         }
2388     });
2389
2390     return $nodehash
2391 }
2392
2393 sub check_local_storage_availability {
2394     my ($conf, $storecfg) = @_;
2395
2396     my $nodelist = PVE::Cluster::get_nodelist();
2397     my $nodehash = { map { $_ => {} } @$nodelist };
2398
2399     PVE::QemuConfig->foreach_volume($conf, sub {
2400         my ($ds, $drive) = @_;
2401
2402         my $volid = $drive->{file};
2403         return if !$volid;
2404
2405         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2406         if ($storeid) {
2407             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2408
2409             if ($scfg->{disable}) {
2410                 foreach my $node (keys %$nodehash) {
2411                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2412                 }
2413             } elsif (my $avail = $scfg->{nodes}) {
2414                 foreach my $node (keys %$nodehash) {
2415                     if (!$avail->{$node}) {
2416                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2417                     }
2418                 }
2419             }
2420         }
2421     });
2422
2423     foreach my $node (values %$nodehash) {
2424         if (my $unavail = $node->{unavailable_storages}) {
2425             $node->{unavailable_storages} = [ sort keys %$unavail ];
2426         }
2427     }
2428
2429     return $nodehash
2430 }
2431
2432 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2433 sub check_running {
2434     my ($vmid, $nocheck, $node) = @_;
2435
2436     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2437     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2438 }
2439
2440 sub vzlist {
2441
2442     my $vzlist = config_list();
2443
2444     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2445
2446     while (defined(my $de = $fd->read)) {
2447         next if $de !~ m/^(\d+)\.pid$/;
2448         my $vmid = $1;
2449         next if !defined($vzlist->{$vmid});
2450         if (my $pid = check_running($vmid)) {
2451             $vzlist->{$vmid}->{pid} = $pid;
2452         }
2453     }
2454
2455     return $vzlist;
2456 }
2457
2458 our $vmstatus_return_properties = {
2459     vmid => get_standard_option('pve-vmid'),
2460     status => {
2461         description => "Qemu process status.",
2462         type => 'string',
2463         enum => ['stopped', 'running'],
2464     },
2465     maxmem => {
2466         description => "Maximum memory in bytes.",
2467         type => 'integer',
2468         optional => 1,
2469         renderer => 'bytes',
2470     },
2471     maxdisk => {
2472         description => "Root disk size in bytes.",
2473         type => 'integer',
2474         optional => 1,
2475         renderer => 'bytes',
2476     },
2477     name => {
2478         description => "VM name.",
2479         type => 'string',
2480         optional => 1,
2481     },
2482     qmpstatus => {
2483         description => "Qemu QMP agent status.",
2484         type => 'string',
2485         optional => 1,
2486     },
2487     pid => {
2488         description => "PID of running qemu process.",
2489         type => 'integer',
2490         optional => 1,
2491     },
2492     uptime => {
2493         description => "Uptime.",
2494         type => 'integer',
2495         optional => 1,
2496         renderer => 'duration',
2497     },
2498     cpus => {
2499         description => "Maximum usable CPUs.",
2500         type => 'number',
2501         optional => 1,
2502     },
2503     lock => {
2504         description => "The current config lock, if any.",
2505         type => 'string',
2506         optional => 1,
2507     },
2508     tags => {
2509         description  => "The current configured tags, if any",
2510         type => 'string',
2511         optional => 1,
2512     },
2513 };
2514
2515 my $last_proc_pid_stat;
2516
2517 # get VM status information
2518 # This must be fast and should not block ($full == false)
2519 # We only query KVM using QMP if $full == true (this can be slow)
2520 sub vmstatus {
2521     my ($opt_vmid, $full) = @_;
2522
2523     my $res = {};
2524
2525     my $storecfg = PVE::Storage::config();
2526
2527     my $list = vzlist();
2528     my $defaults = load_defaults();
2529
2530     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2531
2532     my $cpucount = $cpuinfo->{cpus} || 1;
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536
2537         my $conf = PVE::QemuConfig->load_config($vmid);
2538
2539         my $d = { vmid => $vmid };
2540         $d->{pid} = $list->{$vmid}->{pid};
2541
2542         # fixme: better status?
2543         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2544
2545         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2546         if (defined($size)) {
2547             $d->{disk} = 0; # no info available
2548             $d->{maxdisk} = $size;
2549         } else {
2550             $d->{disk} = 0;
2551             $d->{maxdisk} = 0;
2552         }
2553
2554         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2555             * ($conf->{cores} || $defaults->{cores});
2556         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2557         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2558
2559         $d->{name} = $conf->{name} || "VM $vmid";
2560         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2561             : $defaults->{memory}*(1024*1024);
2562
2563         if ($conf->{balloon}) {
2564             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2565             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2566                 : $defaults->{shares};
2567         }
2568
2569         $d->{uptime} = 0;
2570         $d->{cpu} = 0;
2571         $d->{mem} = 0;
2572
2573         $d->{netout} = 0;
2574         $d->{netin} = 0;
2575
2576         $d->{diskread} = 0;
2577         $d->{diskwrite} = 0;
2578
2579         $d->{template} = PVE::QemuConfig->is_template($conf);
2580
2581         $d->{serial} = 1 if conf_has_serial($conf);
2582         $d->{lock} = $conf->{lock} if $conf->{lock};
2583         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2584
2585         $res->{$vmid} = $d;
2586     }
2587
2588     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2589     foreach my $dev (keys %$netdev) {
2590         next if $dev !~ m/^tap([1-9]\d*)i/;
2591         my $vmid = $1;
2592         my $d = $res->{$vmid};
2593         next if !$d;
2594
2595         $d->{netout} += $netdev->{$dev}->{receive};
2596         $d->{netin} += $netdev->{$dev}->{transmit};
2597
2598         if ($full) {
2599             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2600             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2601         }
2602
2603     }
2604
2605     my $ctime = gettimeofday;
2606
2607     foreach my $vmid (keys %$list) {
2608
2609         my $d = $res->{$vmid};
2610         my $pid = $d->{pid};
2611         next if !$pid;
2612
2613         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2614         next if !$pstat; # not running
2615
2616         my $used = $pstat->{utime} + $pstat->{stime};
2617
2618         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2619
2620         if ($pstat->{vsize}) {
2621             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2622         }
2623
2624         my $old = $last_proc_pid_stat->{$pid};
2625         if (!$old) {
2626             $last_proc_pid_stat->{$pid} = {
2627                 time => $ctime,
2628                 used => $used,
2629                 cpu => 0,
2630             };
2631             next;
2632         }
2633
2634         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2635
2636         if ($dtime > 1000) {
2637             my $dutime = $used -  $old->{used};
2638
2639             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2640             $last_proc_pid_stat->{$pid} = {
2641                 time => $ctime,
2642                 used => $used,
2643                 cpu => $d->{cpu},
2644             };
2645         } else {
2646             $d->{cpu} = $old->{cpu};
2647         }
2648     }
2649
2650     return $res if !$full;
2651
2652     my $qmpclient = PVE::QMPClient->new();
2653
2654     my $ballooncb = sub {
2655         my ($vmid, $resp) = @_;
2656
2657         my $info = $resp->{'return'};
2658         return if !$info->{max_mem};
2659
2660         my $d = $res->{$vmid};
2661
2662         # use memory assigned to VM
2663         $d->{maxmem} = $info->{max_mem};
2664         $d->{balloon} = $info->{actual};
2665
2666         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2667             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2668             $d->{freemem} = $info->{free_mem};
2669         }
2670
2671         $d->{ballooninfo} = $info;
2672     };
2673
2674     my $blockstatscb = sub {
2675         my ($vmid, $resp) = @_;
2676         my $data = $resp->{'return'} || [];
2677         my $totalrdbytes = 0;
2678         my $totalwrbytes = 0;
2679
2680         for my $blockstat (@$data) {
2681             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2682             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2683
2684             $blockstat->{device} =~ s/drive-//;
2685             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2686         }
2687         $res->{$vmid}->{diskread} = $totalrdbytes;
2688         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2689     };
2690
2691     my $statuscb = sub {
2692         my ($vmid, $resp) = @_;
2693
2694         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2695         # this fails if ballon driver is not loaded, so this must be
2696         # the last commnand (following command are aborted if this fails).
2697         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2698
2699         my $status = 'unknown';
2700         if (!defined($status = $resp->{'return'}->{status})) {
2701             warn "unable to get VM status\n";
2702             return;
2703         }
2704
2705         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2706     };
2707
2708     foreach my $vmid (keys %$list) {
2709         next if $opt_vmid && ($vmid ne $opt_vmid);
2710         next if !$res->{$vmid}->{pid}; # not running
2711         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2712     }
2713
2714     $qmpclient->queue_execute(undef, 2);
2715
2716     foreach my $vmid (keys %$list) {
2717         next if $opt_vmid && ($vmid ne $opt_vmid);
2718         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2719     }
2720
2721     return $res;
2722 }
2723
2724 sub conf_has_serial {
2725     my ($conf) = @_;
2726
2727     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2728         if ($conf->{"serial$i"}) {
2729             return 1;
2730         }
2731     }
2732
2733     return 0;
2734 }
2735
2736 sub conf_has_audio {
2737     my ($conf, $id) = @_;
2738
2739     $id //= 0;
2740     my $audio = $conf->{"audio$id"};
2741     return undef if !defined($audio);
2742
2743     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2744     my $audiodriver = $audioproperties->{driver} // 'spice';
2745
2746     return {
2747         dev => $audioproperties->{device},
2748         dev_id => "audiodev$id",
2749         backend => $audiodriver,
2750         backend_id => "$audiodriver-backend${id}",
2751     };
2752 }
2753
2754 sub vga_conf_has_spice {
2755     my ($vga) = @_;
2756
2757     my $vgaconf = parse_vga($vga);
2758     my $vgatype = $vgaconf->{type};
2759     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2760
2761     return $1 || 1;
2762 }
2763
2764 sub is_native($) {
2765     my ($arch) = @_;
2766     return get_host_arch() eq $arch;
2767 }
2768
2769 sub get_vm_arch {
2770     my ($conf) = @_;
2771     return $conf->{arch} // get_host_arch();
2772 }
2773
2774 my $default_machines = {
2775     x86_64 => 'pc',
2776     aarch64 => 'virt',
2777 };
2778
2779 sub get_vm_machine {
2780     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2781
2782     my $machine = $forcemachine || $conf->{machine};
2783
2784     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2785         $arch //= 'x86_64';
2786         $machine ||= $default_machines->{$arch};
2787         if ($add_pve_version) {
2788             $kvmversion //= kvm_user_version();
2789             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2790             $machine .= "+pve$pvever";
2791         }
2792     }
2793
2794     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2795         # for version-pinned machines that do not include a pve-version (e.g.
2796         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2797         $machine .= '+pve0';
2798     }
2799
2800     return $machine;
2801 }
2802
2803 sub get_ovmf_files($) {
2804     my ($arch) = @_;
2805
2806     my $ovmf = $OVMF->{$arch}
2807         or die "no OVMF images known for architecture '$arch'\n";
2808
2809     return @$ovmf;
2810 }
2811
2812 my $Arch2Qemu = {
2813     aarch64 => '/usr/bin/qemu-system-aarch64',
2814     x86_64 => '/usr/bin/qemu-system-x86_64',
2815 };
2816 sub get_command_for_arch($) {
2817     my ($arch) = @_;
2818     return '/usr/bin/kvm' if is_native($arch);
2819
2820     my $cmd = $Arch2Qemu->{$arch}
2821         or die "don't know how to emulate architecture '$arch'\n";
2822     return $cmd;
2823 }
2824
2825 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2826 # to use in a QEMU command line (-cpu element), first array_intersect the result
2827 # of query_supported_ with query_understood_. This is necessary because:
2828 #
2829 # a) query_understood_ returns flags the host cannot use and
2830 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2831 #    flags, but CPU settings - with most of them being flags. Those settings
2832 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2833 #
2834 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2835 # expensive. If you need the value returned from this, you can get it much
2836 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2837 # $accel being 'kvm' or 'tcg'.
2838 #
2839 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2840 # changes, automatically populating pmxcfs.
2841 #
2842 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2843 # since kvm and tcg machines support different flags
2844 #
2845 sub query_supported_cpu_flags {
2846     my ($arch) = @_;
2847
2848     $arch //= get_host_arch();
2849     my $default_machine = $default_machines->{$arch};
2850
2851     my $flags = {};
2852
2853     # FIXME: Once this is merged, the code below should work for ARM as well:
2854     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2855     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2856         $arch eq "aarch64";
2857
2858     my $kvm_supported = defined(kvm_version());
2859     my $qemu_cmd = get_command_for_arch($arch);
2860     my $fakevmid = -1;
2861     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2862
2863     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2864     my $query_supported_run_qemu = sub {
2865         my ($kvm) = @_;
2866
2867         my $flags = {};
2868         my $cmd = [
2869             $qemu_cmd,
2870             '-machine', $default_machine,
2871             '-display', 'none',
2872             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2873             '-mon', 'chardev=qmp,mode=control',
2874             '-pidfile', $pidfile,
2875             '-S', '-daemonize'
2876         ];
2877
2878         if (!$kvm) {
2879             push @$cmd, '-accel', 'tcg';
2880         }
2881
2882         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2883         die "QEMU flag querying VM exited with code " . $rc if $rc;
2884
2885         eval {
2886             my $cmd_result = mon_cmd(
2887                 $fakevmid,
2888                 'query-cpu-model-expansion',
2889                 type => 'full',
2890                 model => { name => 'host' }
2891             );
2892
2893             my $props = $cmd_result->{model}->{props};
2894             foreach my $prop (keys %$props) {
2895                 next if $props->{$prop} ne '1';
2896                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2897                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2898                 # We only keep those with underscores, to match /proc/cpuinfo
2899                 $prop =~ s/\.|-/_/g;
2900                 $flags->{$prop} = 1;
2901             }
2902         };
2903         my $err = $@;
2904
2905         # force stop with 10 sec timeout and 'nocheck'
2906         # always stop, even if QMP failed
2907         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2908
2909         die $err if $err;
2910
2911         return [ sort keys %$flags ];
2912     };
2913
2914     # We need to query QEMU twice, since KVM and TCG have different supported flags
2915     PVE::QemuConfig->lock_config($fakevmid, sub {
2916         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2917         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2918
2919         if ($kvm_supported) {
2920             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2921             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2922         }
2923     });
2924
2925     return $flags;
2926 }
2927
2928 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2929 my $understood_cpu_flag_dir = "/usr/share/kvm";
2930 sub query_understood_cpu_flags {
2931     my $arch = get_host_arch();
2932     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2933
2934     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2935         if ! -e $filepath;
2936
2937     my $raw = file_get_contents($filepath);
2938     $raw =~ s/^\s+|\s+$//g;
2939     my @flags = split(/\s+/, $raw);
2940
2941     return \@flags;
2942 }
2943
2944 sub config_to_command {
2945     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2946
2947     my $cmd = [];
2948     my $globalFlags = [];
2949     my $machineFlags = [];
2950     my $rtcFlags = [];
2951     my $devices = [];
2952     my $pciaddr = '';
2953     my $bridges = {};
2954     my $ostype = $conf->{ostype};
2955     my $winversion = windows_version($ostype);
2956     my $kvm = $conf->{kvm};
2957     my $nodename = nodename();
2958
2959     my $arch = get_vm_arch($conf);
2960     my $kvm_binary = get_command_for_arch($arch);
2961     my $kvmver = kvm_user_version($kvm_binary);
2962
2963     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2964         $kvmver //= "undefined";
2965         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2966     }
2967
2968     my $add_pve_version = min_version($kvmver, 4, 1);
2969
2970     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2971     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2972     $kvm //= 1 if is_native($arch);
2973
2974     $machine_version =~ m/(\d+)\.(\d+)/;
2975     my ($machine_major, $machine_minor) = ($1, $2);
2976
2977     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2978         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2979     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2980         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2981     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2982         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2983         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2984             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2985     }
2986
2987     # if a specific +pve version is required for a feature, use $version_guard
2988     # instead of min_version to allow machines to be run with the minimum
2989     # required version
2990     my $required_pve_version = 0;
2991     my $version_guard = sub {
2992         my ($major, $minor, $pve) = @_;
2993         return 0 if !min_version($machine_version, $major, $minor, $pve);
2994         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2995         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2996         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2997         return 1;
2998     };
2999
3000     if ($kvm) {
3001         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3002             if !defined kvm_version();
3003     }
3004
3005     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3006     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3007     my $use_old_bios_files = undef;
3008     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3009
3010     my $cpuunits = defined($conf->{cpuunits}) ?
3011             $conf->{cpuunits} : $defaults->{cpuunits};
3012
3013     push @$cmd, $kvm_binary;
3014
3015     push @$cmd, '-id', $vmid;
3016
3017     my $vmname = $conf->{name} || "vm$vmid";
3018
3019     push @$cmd, '-name', $vmname;
3020
3021     my $use_virtio = 0;
3022
3023     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3024     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3025     push @$cmd, '-mon', "chardev=qmp,mode=control";
3026
3027     if (min_version($machine_version, 2, 12)) {
3028         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3029         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3030     }
3031
3032     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3033
3034     push @$cmd, '-daemonize';
3035
3036     if ($conf->{smbios1}) {
3037         my $smbios_conf = parse_smbios1($conf->{smbios1});
3038         if ($smbios_conf->{base64}) {
3039             # Do not pass base64 flag to qemu
3040             delete $smbios_conf->{base64};
3041             my $smbios_string = "";
3042             foreach my $key (keys %$smbios_conf) {
3043                 my $value;
3044                 if ($key eq "uuid") {
3045                     $value = $smbios_conf->{uuid}
3046                 } else {
3047                     $value = decode_base64($smbios_conf->{$key});
3048                 }
3049                 # qemu accepts any binary data, only commas need escaping by double comma
3050                 $value =~ s/,/,,/g;
3051                 $smbios_string .= "," . $key . "=" . $value if $value;
3052             }
3053             push @$cmd, '-smbios', "type=1" . $smbios_string;
3054         } else {
3055             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3056         }
3057     }
3058
3059     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3060     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3061         die "uefi base image not found\n" if ! -f $ovmf_code;
3062
3063         my $path;
3064         my $format;
3065         if (my $efidisk = $conf->{efidisk0}) {
3066             my $d = parse_drive('efidisk0', $efidisk);
3067             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3068             $format = $d->{format};
3069             if ($storeid) {
3070                 $path = PVE::Storage::path($storecfg, $d->{file});
3071                 if (!defined($format)) {
3072                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3073                     $format = qemu_img_format($scfg, $volname);
3074                 }
3075             } else {
3076                 $path = $d->{file};
3077                 die "efidisk format must be specified\n"
3078                     if !defined($format);
3079             }
3080         } else {
3081             warn "no efidisk configured! Using temporary efivars disk.\n";
3082             $path = "/tmp/$vmid-ovmf.fd";
3083             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3084             $format = 'raw';
3085         }
3086
3087         my $size_str = "";
3088
3089         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3090             $size_str = ",size=" . (-s $ovmf_vars);
3091         }
3092
3093         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3094         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3095     }
3096
3097     # load q35 config
3098     if ($q35) {
3099         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3100         if (min_version($machine_version, 4, 0)) {
3101             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3102         } else {
3103             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3104         }
3105     }
3106
3107     if ($conf->{vmgenid}) {
3108         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3109     }
3110
3111     # add usb controllers
3112     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3113     push @$devices, @usbcontrollers if @usbcontrollers;
3114     my $vga = parse_vga($conf->{vga});
3115
3116     my $qxlnum = vga_conf_has_spice($conf->{vga});
3117     $vga->{type} = 'qxl' if $qxlnum;
3118
3119     if (!$vga->{type}) {
3120         if ($arch eq 'aarch64') {
3121             $vga->{type} = 'virtio';
3122         } elsif (min_version($machine_version, 2, 9)) {
3123             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3124         } else {
3125             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3126         }
3127     }
3128
3129     # enable absolute mouse coordinates (needed by vnc)
3130     my $tablet;
3131     if (defined($conf->{tablet})) {
3132         $tablet = $conf->{tablet};
3133     } else {
3134         $tablet = $defaults->{tablet};
3135         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3136         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3137     }
3138
3139     if ($tablet) {
3140         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3141         my $kbd = print_keyboarddevice_full($conf, $arch);
3142         push @$devices, '-device', $kbd if defined($kbd);
3143     }
3144
3145     my $kvm_off = 0;
3146     my $gpu_passthrough;
3147
3148     # host pci devices
3149     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3150         my $id = "hostpci$i";
3151         my $d = parse_hostpci($conf->{$id});
3152         next if !$d;
3153
3154         if (my $pcie = $d->{pcie}) {
3155             die "q35 machine model is not enabled" if !$q35;
3156             # win7 wants to have the pcie devices directly on the pcie bus
3157             # instead of in the root port
3158             if ($winversion == 7) {
3159                 $pciaddr = print_pcie_addr("${id}bus0");
3160             } else {
3161                 # add more root ports if needed, 4 are present by default
3162                 # by pve-q35 cfgs, rest added here on demand.
3163                 if ($i > 3) {
3164                     push @$devices, '-device', print_pcie_root_port($i);
3165                 }
3166                 $pciaddr = print_pcie_addr($id);
3167             }
3168         } else {
3169             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3170         }
3171
3172         my $xvga = '';
3173         if ($d->{'x-vga'}) {
3174             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3175             $kvm_off = 1;
3176             $vga->{type} = 'none' if !defined($conf->{vga});
3177             $gpu_passthrough = 1;
3178         }
3179
3180         my $pcidevices = $d->{pciid};
3181         my $multifunction = 1 if @$pcidevices > 1;
3182
3183         my $sysfspath;
3184         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3185             my $pci_id = $pcidevices->[0]->{id};
3186             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3187             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3188         } elsif ($d->{mdev}) {
3189             warn "ignoring mediated device '$id' with multifunction device\n";
3190         }
3191
3192         my $j=0;
3193         foreach my $pcidevice (@$pcidevices) {
3194             my $devicestr = "vfio-pci";
3195
3196             if ($sysfspath) {
3197                 $devicestr .= ",sysfsdev=$sysfspath";
3198             } else {
3199                 $devicestr .= ",host=$pcidevice->{id}";
3200             }
3201
3202             my $mf_addr = $multifunction ? ".$j" : '';
3203             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3204
3205             if ($j == 0) {
3206                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3207                 $devicestr .= "$xvga";
3208                 $devicestr .= ",multifunction=on" if $multifunction;
3209                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3210             }
3211
3212             push @$devices, '-device', $devicestr;
3213             $j++;
3214         }
3215     }
3216
3217     # usb devices
3218     my $usb_dev_features = {};
3219     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3220
3221     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3222     push @$devices, @usbdevices if @usbdevices;
3223     # serial devices
3224     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3225         if (my $path = $conf->{"serial$i"}) {
3226             if ($path eq 'socket') {
3227                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3228                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3229                 # On aarch64, serial0 is the UART device. Qemu only allows
3230                 # connecting UART devices via the '-serial' command line, as
3231                 # the device has a fixed slot on the hardware...
3232                 if ($arch eq 'aarch64' && $i == 0) {
3233                     push @$devices, '-serial', "chardev:serial$i";
3234                 } else {
3235                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3236                 }
3237             } else {
3238                 die "no such serial device\n" if ! -c $path;
3239                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3240                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3241             }
3242         }
3243     }
3244
3245     # parallel devices
3246     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3247         if (my $path = $conf->{"parallel$i"}) {
3248             die "no such parallel device\n" if ! -c $path;
3249             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3250             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3251             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3252         }
3253     }
3254
3255     if (my $audio = conf_has_audio($conf)) {
3256
3257         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3258
3259         my $id = $audio->{dev_id};
3260         if ($audio->{dev} eq 'AC97') {
3261             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3262         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3263             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3264             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3265             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3266         } else {
3267             die "unkown audio device '$audio->{dev}', implement me!";
3268         }
3269
3270         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3271     }
3272
3273     my $sockets = 1;
3274     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3275     $sockets = $conf->{sockets} if  $conf->{sockets};
3276
3277     my $cores = $conf->{cores} || 1;
3278
3279     my $maxcpus = $sockets * $cores;
3280
3281     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3282
3283     my $allowed_vcpus = $cpuinfo->{cpus};
3284
3285     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3286         if ($allowed_vcpus < $maxcpus);
3287
3288     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3289
3290         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3291         for (my $i = 2; $i <= $vcpus; $i++)  {
3292             my $cpustr = print_cpu_device($conf,$i);
3293             push @$cmd, '-device', $cpustr;
3294         }
3295
3296     } else {
3297
3298         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3299     }
3300     push @$cmd, '-nodefaults';
3301
3302     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3303
3304     my $bootindex_hash = {};
3305     my $i = 1;
3306     foreach my $o (split(//, $bootorder)) {
3307         $bootindex_hash->{$o} = $i*100;
3308         $i++;
3309     }
3310
3311     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3312
3313     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3314
3315     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3316
3317     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3318         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3319         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3320         push @$cmd,  '-vnc', "unix:$socket,password";
3321     } else {
3322         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3323         push @$cmd, '-nographic';
3324     }
3325
3326     # time drift fix
3327     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3328     my $useLocaltime = $conf->{localtime};
3329
3330     if ($winversion >= 5) { # windows
3331         $useLocaltime = 1 if !defined($conf->{localtime});
3332
3333         # use time drift fix when acpi is enabled
3334         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3335             $tdf = 1 if !defined($conf->{tdf});
3336         }
3337     }
3338
3339     if ($winversion >= 6) {
3340         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3341         push @$cmd, '-no-hpet';
3342     }
3343
3344     push @$rtcFlags, 'driftfix=slew' if $tdf;
3345
3346     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3347         push @$rtcFlags, "base=$conf->{startdate}";
3348     } elsif ($useLocaltime) {
3349         push @$rtcFlags, 'base=localtime';
3350     }
3351
3352     if ($forcecpu) {
3353         push @$cmd, '-cpu', $forcecpu;
3354     } else {
3355         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3356     }
3357
3358     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3359
3360     push @$cmd, '-S' if $conf->{freeze};
3361
3362     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3363
3364     my $guest_agent = parse_guest_agent($conf);
3365
3366     if ($guest_agent->{enabled}) {
3367         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3368         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3369
3370         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3371             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3372             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3373             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3374         } elsif ($guest_agent->{type} eq 'isa') {
3375             push @$devices, '-device', "isa-serial,chardev=qga0";
3376         }
3377     }
3378
3379     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3380     if ($rng && &$version_guard(4, 1, 2)) {
3381         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3382         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3383
3384         my $limiter_str = "";
3385         if ($max_bytes) {
3386             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3387         }
3388
3389         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3390         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3391             if ! -e $rng->{source};
3392
3393         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3394
3395         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3396         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3397     }
3398
3399     my $spice_port;
3400
3401     if ($qxlnum) {
3402         if ($qxlnum > 1) {
3403             if ($winversion){
3404                 for (my $i = 1; $i < $qxlnum; $i++){
3405                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3406                 }
3407             } else {
3408                 # assume other OS works like Linux
3409                 my ($ram, $vram) = ("134217728", "67108864");
3410                 if ($vga->{memory}) {
3411                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3412                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3413                 }
3414                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3415                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3416             }
3417         }
3418
3419         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3420
3421         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3422         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3423         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3424
3425         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3426         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3427         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3428
3429         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3430         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3431
3432         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3433         if ($spice_enhancement->{foldersharing}) {
3434             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3435             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3436         }
3437
3438         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3439         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3440         push @$devices, '-spice', "$spice_opts";
3441     }
3442
3443     # enable balloon by default, unless explicitly disabled
3444     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3445         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3446         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3447     }
3448
3449     if ($conf->{watchdog}) {
3450         my $wdopts = parse_watchdog($conf->{watchdog});
3451         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3452         my $watchdog = $wdopts->{model} || 'i6300esb';
3453         push @$devices, '-device', "$watchdog$pciaddr";
3454         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3455     }
3456
3457     my $vollist = [];
3458     my $scsicontroller = {};
3459     my $ahcicontroller = {};
3460     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3461
3462     # Add iscsi initiator name if available
3463     if (my $initiator = get_initiator_name()) {
3464         push @$devices, '-iscsi', "initiator-name=$initiator";
3465     }
3466
3467     PVE::QemuConfig->foreach_volume($conf, sub {
3468         my ($ds, $drive) = @_;
3469
3470         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3471             push @$vollist, $drive->{file};
3472         }
3473
3474         # ignore efidisk here, already added in bios/fw handling code above
3475         return if $drive->{interface} eq 'efidisk';
3476
3477         $use_virtio = 1 if $ds =~ m/^virtio/;
3478
3479         if (drive_is_cdrom ($drive)) {
3480             if ($bootindex_hash->{d}) {
3481                 $drive->{bootindex} = $bootindex_hash->{d};
3482                 $bootindex_hash->{d} += 1;
3483             }
3484         } else {
3485             if ($bootindex_hash->{c}) {
3486                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3487                 $bootindex_hash->{c} += 1;
3488             }
3489         }
3490
3491         if ($drive->{interface} eq 'virtio'){
3492            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3493         }
3494
3495         if ($drive->{interface} eq 'scsi') {
3496
3497             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3498
3499             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3500                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3501
3502             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3503             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3504
3505             my $iothread = '';
3506             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3507                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3508                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3509             } elsif ($drive->{iothread}) {
3510                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3511             }
3512
3513             my $queues = '';
3514             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3515                 $queues = ",num_queues=$drive->{queues}";
3516             }
3517
3518             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3519             $scsicontroller->{$controller}=1;
3520         }
3521
3522         if ($drive->{interface} eq 'sata') {
3523             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3524             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3525             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3526             $ahcicontroller->{$controller}=1;
3527         }
3528
3529         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3530         push @$devices, '-drive',$drive_cmd;
3531         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3532     });
3533
3534     for (my $i = 0; $i < $MAX_NETS; $i++) {
3535          next if !$conf->{"net$i"};
3536          my $d = parse_net($conf->{"net$i"});
3537          next if !$d;
3538
3539          $use_virtio = 1 if $d->{model} eq 'virtio';
3540
3541          if ($bootindex_hash->{n}) {
3542             $d->{bootindex} = $bootindex_hash->{n};
3543             $bootindex_hash->{n} += 1;
3544          }
3545
3546          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3547          push @$devices, '-netdev', $netdevfull;
3548
3549          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3550          push @$devices, '-device', $netdevicefull;
3551     }
3552
3553     if ($conf->{ivshmem}) {
3554         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3555
3556         my $bus;
3557         if ($q35) {
3558             $bus = print_pcie_addr("ivshmem");
3559         } else {
3560             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3561         }
3562
3563         my $ivshmem_name = $ivshmem->{name} // $vmid;
3564         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3565
3566         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3567         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3568     }
3569
3570     # pci.4 is nested in pci.1
3571     $bridges->{1} = 1 if $bridges->{4};
3572
3573     if (!$q35) {
3574         # add pci bridges
3575         if (min_version($machine_version, 2, 3)) {
3576            $bridges->{1} = 1;
3577            $bridges->{2} = 1;
3578         }
3579
3580         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3581
3582     }
3583
3584     for my $k (sort {$b cmp $a} keys %$bridges) {
3585         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3586         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3587         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3588         if ($q35) {
3589             # add after -readconfig pve-q35.cfg
3590             splice @$devices, 2, 0, '-device', $devstr;
3591         } else {
3592             unshift @$devices, '-device', $devstr if $k > 0;
3593         }
3594     }
3595
3596     if (!$kvm) {
3597         push @$machineFlags, 'accel=tcg';
3598     }
3599
3600     my $machine_type_min = $machine_type;
3601     if ($add_pve_version) {
3602         $machine_type_min =~ s/\+pve\d+$//;
3603         $machine_type_min .= "+pve$required_pve_version";
3604     }
3605     push @$machineFlags, "type=${machine_type_min}";
3606
3607     push @$cmd, @$devices;
3608     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3609     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3610     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3611
3612     if (my $vmstate = $conf->{vmstate}) {
3613         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3614         push @$vollist, $vmstate;
3615         push @$cmd, '-loadstate', $statepath;
3616         print "activating and using '$vmstate' as vmstate\n";
3617     }
3618
3619     # add custom args
3620     if ($conf->{args}) {
3621         my $aa = PVE::Tools::split_args($conf->{args});
3622         push @$cmd, @$aa;
3623     }
3624
3625     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3626 }
3627
3628 sub spice_port {
3629     my ($vmid) = @_;
3630
3631     my $res = mon_cmd($vmid, 'query-spice');
3632
3633     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3634 }
3635
3636 sub vm_devices_list {
3637     my ($vmid) = @_;
3638
3639     my $res = mon_cmd($vmid, 'query-pci');
3640     my $devices_to_check = [];
3641     my $devices = {};
3642     foreach my $pcibus (@$res) {
3643         push @$devices_to_check, @{$pcibus->{devices}},
3644     }
3645
3646     while (@$devices_to_check) {
3647         my $to_check = [];
3648         for my $d (@$devices_to_check) {
3649             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3650             next if !$d->{'pci_bridge'};
3651
3652             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3653             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3654         }
3655         $devices_to_check = $to_check;
3656     }
3657
3658     my $resblock = mon_cmd($vmid, 'query-block');
3659     foreach my $block (@$resblock) {
3660         if($block->{device} =~ m/^drive-(\S+)/){
3661                 $devices->{$1} = 1;
3662         }
3663     }
3664
3665     my $resmice = mon_cmd($vmid, 'query-mice');
3666     foreach my $mice (@$resmice) {
3667         if ($mice->{name} eq 'QEMU HID Tablet') {
3668             $devices->{tablet} = 1;
3669             last;
3670         }
3671     }
3672
3673     # for usb devices there is no query-usb
3674     # but we can iterate over the entries in
3675     # qom-list path=/machine/peripheral
3676     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3677     foreach my $per (@$resperipheral) {
3678         if ($per->{name} =~ m/^usb\d+$/) {
3679             $devices->{$per->{name}} = 1;
3680         }
3681     }
3682
3683     return $devices;
3684 }
3685
3686 sub vm_deviceplug {
3687     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3688
3689     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3690
3691     my $devices_list = vm_devices_list($vmid);
3692     return 1 if defined($devices_list->{$deviceid});
3693
3694     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3695
3696     if ($deviceid eq 'tablet') {
3697
3698         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3699
3700     } elsif ($deviceid eq 'keyboard') {
3701
3702         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3703
3704     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3705
3706         die "usb hotplug currently not reliable\n";
3707         # since we can't reliably hot unplug all added usb devices
3708         # and usb passthrough disables live migration
3709         # we disable usb hotplugging for now
3710         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3711
3712     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3713
3714         qemu_iothread_add($vmid, $deviceid, $device);
3715
3716         qemu_driveadd($storecfg, $vmid, $device);
3717         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3718
3719         qemu_deviceadd($vmid, $devicefull);
3720         eval { qemu_deviceaddverify($vmid, $deviceid); };
3721         if (my $err = $@) {
3722             eval { qemu_drivedel($vmid, $deviceid); };
3723             warn $@ if $@;
3724             die $err;
3725         }
3726
3727     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3728
3729
3730         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3731         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3732         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3733
3734         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3735
3736         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3737             qemu_iothread_add($vmid, $deviceid, $device);
3738             $devicefull .= ",iothread=iothread-$deviceid";
3739         }
3740
3741         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3742             $devicefull .= ",num_queues=$device->{queues}";
3743         }
3744
3745         qemu_deviceadd($vmid, $devicefull);
3746         qemu_deviceaddverify($vmid, $deviceid);
3747
3748     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3749
3750         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3751         qemu_driveadd($storecfg, $vmid, $device);
3752
3753         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3754         eval { qemu_deviceadd($vmid, $devicefull); };
3755         if (my $err = $@) {
3756             eval { qemu_drivedel($vmid, $deviceid); };
3757             warn $@ if $@;
3758             die $err;
3759         }
3760
3761     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3762
3763         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3764
3765         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3766         my $use_old_bios_files = undef;
3767         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3768
3769         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3770         qemu_deviceadd($vmid, $netdevicefull);
3771         eval {
3772             qemu_deviceaddverify($vmid, $deviceid);
3773             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3774         };
3775         if (my $err = $@) {
3776             eval { qemu_netdevdel($vmid, $deviceid); };
3777             warn $@ if $@;
3778             die $err;
3779         }
3780
3781     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3782
3783         my $bridgeid = $2;
3784         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3785         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3786
3787         qemu_deviceadd($vmid, $devicefull);
3788         qemu_deviceaddverify($vmid, $deviceid);
3789
3790     } else {
3791         die "can't hotplug device '$deviceid'\n";
3792     }
3793
3794     return 1;
3795 }
3796
3797 # fixme: this should raise exceptions on error!
3798 sub vm_deviceunplug {
3799     my ($vmid, $conf, $deviceid) = @_;
3800
3801     my $devices_list = vm_devices_list($vmid);
3802     return 1 if !defined($devices_list->{$deviceid});
3803
3804     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3805
3806     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3807
3808         qemu_devicedel($vmid, $deviceid);
3809
3810     } elsif ($deviceid =~ m/^usb\d+$/) {
3811
3812         die "usb hotplug currently not reliable\n";
3813         # when unplugging usb devices this way,
3814         # there may be remaining usb controllers/hubs
3815         # so we disable it for now
3816         qemu_devicedel($vmid, $deviceid);
3817         qemu_devicedelverify($vmid, $deviceid);
3818
3819     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3820
3821         qemu_devicedel($vmid, $deviceid);
3822         qemu_devicedelverify($vmid, $deviceid);
3823         qemu_drivedel($vmid, $deviceid);
3824         qemu_iothread_del($conf, $vmid, $deviceid);
3825
3826     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3827
3828         qemu_devicedel($vmid, $deviceid);
3829         qemu_devicedelverify($vmid, $deviceid);
3830         qemu_iothread_del($conf, $vmid, $deviceid);
3831
3832     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3833
3834         qemu_devicedel($vmid, $deviceid);
3835         qemu_drivedel($vmid, $deviceid);
3836         qemu_deletescsihw($conf, $vmid, $deviceid);
3837
3838     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3839
3840         qemu_devicedel($vmid, $deviceid);
3841         qemu_devicedelverify($vmid, $deviceid);
3842         qemu_netdevdel($vmid, $deviceid);
3843
3844     } else {
3845         die "can't unplug device '$deviceid'\n";
3846     }
3847
3848     return 1;
3849 }
3850
3851 sub qemu_deviceadd {
3852     my ($vmid, $devicefull) = @_;
3853
3854     $devicefull = "driver=".$devicefull;
3855     my %options =  split(/[=,]/, $devicefull);
3856
3857     mon_cmd($vmid, "device_add" , %options);
3858 }
3859
3860 sub qemu_devicedel {
3861     my ($vmid, $deviceid) = @_;
3862
3863     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3864 }
3865
3866 sub qemu_iothread_add {
3867     my($vmid, $deviceid, $device) = @_;
3868
3869     if ($device->{iothread}) {
3870         my $iothreads = vm_iothreads_list($vmid);
3871         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3872     }
3873 }
3874
3875 sub qemu_iothread_del {
3876     my($conf, $vmid, $deviceid) = @_;
3877
3878     my $confid = $deviceid;
3879     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3880         $confid = 'scsi' . $1;
3881     }
3882     my $device = parse_drive($confid, $conf->{$confid});
3883     if ($device->{iothread}) {
3884         my $iothreads = vm_iothreads_list($vmid);
3885         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3886     }
3887 }
3888
3889 sub qemu_objectadd {
3890     my($vmid, $objectid, $qomtype) = @_;
3891
3892     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3893
3894     return 1;
3895 }
3896
3897 sub qemu_objectdel {
3898     my($vmid, $objectid) = @_;
3899
3900     mon_cmd($vmid, "object-del", id => $objectid);
3901
3902     return 1;
3903 }
3904
3905 sub qemu_driveadd {
3906     my ($storecfg, $vmid, $device) = @_;
3907
3908     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3909     $drive =~ s/\\/\\\\/g;
3910     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3911
3912     # If the command succeeds qemu prints: "OK"
3913     return 1 if $ret =~ m/OK/s;
3914
3915     die "adding drive failed: $ret\n";
3916 }
3917
3918 sub qemu_drivedel {
3919     my($vmid, $deviceid) = @_;
3920
3921     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3922     $ret =~ s/^\s+//;
3923
3924     return 1 if $ret eq "";
3925
3926     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3927     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3928
3929     die "deleting drive $deviceid failed : $ret\n";
3930 }
3931
3932 sub qemu_deviceaddverify {
3933     my ($vmid, $deviceid) = @_;
3934
3935     for (my $i = 0; $i <= 5; $i++) {
3936          my $devices_list = vm_devices_list($vmid);
3937          return 1 if defined($devices_list->{$deviceid});
3938          sleep 1;
3939     }
3940
3941     die "error on hotplug device '$deviceid'\n";
3942 }
3943
3944
3945 sub qemu_devicedelverify {
3946     my ($vmid, $deviceid) = @_;
3947
3948     # need to verify that the device is correctly removed as device_del
3949     # is async and empty return is not reliable
3950
3951     for (my $i = 0; $i <= 5; $i++) {
3952          my $devices_list = vm_devices_list($vmid);
3953          return 1 if !defined($devices_list->{$deviceid});
3954          sleep 1;
3955     }
3956
3957     die "error on hot-unplugging device '$deviceid'\n";
3958 }
3959
3960 sub qemu_findorcreatescsihw {
3961     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3962
3963     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3964
3965     my $scsihwid="$controller_prefix$controller";
3966     my $devices_list = vm_devices_list($vmid);
3967
3968     if(!defined($devices_list->{$scsihwid})) {
3969         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3970     }
3971
3972     return 1;
3973 }
3974
3975 sub qemu_deletescsihw {
3976     my ($conf, $vmid, $opt) = @_;
3977
3978     my $device = parse_drive($opt, $conf->{$opt});
3979
3980     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3981         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3982         return 1;
3983     }
3984
3985     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3986
3987     my $devices_list = vm_devices_list($vmid);
3988     foreach my $opt (keys %{$devices_list}) {
3989         if (is_valid_drivename($opt)) {
3990             my $drive = parse_drive($opt, $conf->{$opt});
3991             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3992                 return 1;
3993             }
3994         }
3995     }
3996
3997     my $scsihwid="scsihw$controller";
3998
3999     vm_deviceunplug($vmid, $conf, $scsihwid);
4000
4001     return 1;
4002 }
4003
4004 sub qemu_add_pci_bridge {
4005     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4006
4007     my $bridges = {};
4008
4009     my $bridgeid;
4010
4011     print_pci_addr($device, $bridges, $arch, $machine_type);
4012
4013     while (my ($k, $v) = each %$bridges) {
4014         $bridgeid = $k;
4015     }
4016     return 1 if !defined($bridgeid) || $bridgeid < 1;
4017
4018     my $bridge = "pci.$bridgeid";
4019     my $devices_list = vm_devices_list($vmid);
4020
4021     if (!defined($devices_list->{$bridge})) {
4022         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4023     }
4024
4025     return 1;
4026 }
4027
4028 sub qemu_set_link_status {
4029     my ($vmid, $device, $up) = @_;
4030
4031     mon_cmd($vmid, "set_link", name => $device,
4032                up => $up ? JSON::true : JSON::false);
4033 }
4034
4035 sub qemu_netdevadd {
4036     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4037
4038     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4039     my %options =  split(/[=,]/, $netdev);
4040
4041     mon_cmd($vmid, "netdev_add",  %options);
4042     return 1;
4043 }
4044
4045 sub qemu_netdevdel {
4046     my ($vmid, $deviceid) = @_;
4047
4048     mon_cmd($vmid, "netdev_del", id => $deviceid);
4049 }
4050
4051 sub qemu_usb_hotplug {
4052     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4053
4054     return if !$device;
4055
4056     # remove the old one first
4057     vm_deviceunplug($vmid, $conf, $deviceid);
4058
4059     # check if xhci controller is necessary and available
4060     if ($device->{usb3}) {
4061
4062         my $devicelist = vm_devices_list($vmid);
4063
4064         if (!$devicelist->{xhci}) {
4065             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4066             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4067         }
4068     }
4069     my $d = parse_usb_device($device->{host});
4070     $d->{usb3} = $device->{usb3};
4071
4072     # add the new one
4073     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4074 }
4075
4076 sub qemu_cpu_hotplug {
4077     my ($vmid, $conf, $vcpus) = @_;
4078
4079     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4080
4081     my $sockets = 1;
4082     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4083     $sockets = $conf->{sockets} if  $conf->{sockets};
4084     my $cores = $conf->{cores} || 1;
4085     my $maxcpus = $sockets * $cores;
4086
4087     $vcpus = $maxcpus if !$vcpus;
4088
4089     die "you can't add more vcpus than maxcpus\n"
4090         if $vcpus > $maxcpus;
4091
4092     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4093
4094     if ($vcpus < $currentvcpus) {
4095
4096         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4097
4098             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4099                 qemu_devicedel($vmid, "cpu$i");
4100                 my $retry = 0;
4101                 my $currentrunningvcpus = undef;
4102                 while (1) {
4103                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4104                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4105                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4106                     $retry++;
4107                     sleep 1;
4108                 }
4109                 #update conf after each succesfull cpu unplug
4110                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4111                 PVE::QemuConfig->write_config($vmid, $conf);
4112             }
4113         } else {
4114             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4115         }
4116
4117         return;
4118     }
4119
4120     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4121     die "vcpus in running vm does not match its configuration\n"
4122         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4123
4124     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4125
4126         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4127             my $cpustr = print_cpu_device($conf, $i);
4128             qemu_deviceadd($vmid, $cpustr);
4129
4130             my $retry = 0;
4131             my $currentrunningvcpus = undef;
4132             while (1) {
4133                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4134                 last if scalar(@{$currentrunningvcpus}) == $i;
4135                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4136                 sleep 1;
4137                 $retry++;
4138             }
4139             #update conf after each succesfull cpu hotplug
4140             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4141             PVE::QemuConfig->write_config($vmid, $conf);
4142         }
4143     } else {
4144
4145         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4146             mon_cmd($vmid, "cpu-add", id => int($i));
4147         }
4148     }
4149 }
4150
4151 sub qemu_block_set_io_throttle {
4152     my ($vmid, $deviceid,
4153         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4154         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4155         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4156         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4157
4158     return if !check_running($vmid) ;
4159
4160     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4161         bps => int($bps),
4162         bps_rd => int($bps_rd),
4163         bps_wr => int($bps_wr),
4164         iops => int($iops),
4165         iops_rd => int($iops_rd),
4166         iops_wr => int($iops_wr),
4167         bps_max => int($bps_max),
4168         bps_rd_max => int($bps_rd_max),
4169         bps_wr_max => int($bps_wr_max),
4170         iops_max => int($iops_max),
4171         iops_rd_max => int($iops_rd_max),
4172         iops_wr_max => int($iops_wr_max),
4173         bps_max_length => int($bps_max_length),
4174         bps_rd_max_length => int($bps_rd_max_length),
4175         bps_wr_max_length => int($bps_wr_max_length),
4176         iops_max_length => int($iops_max_length),
4177         iops_rd_max_length => int($iops_rd_max_length),
4178         iops_wr_max_length => int($iops_wr_max_length),
4179     );
4180
4181 }
4182
4183 # old code, only used to shutdown old VM after update
4184 sub __read_avail {
4185     my ($fh, $timeout) = @_;
4186
4187     my $sel = new IO::Select;
4188     $sel->add($fh);
4189
4190     my $res = '';
4191     my $buf;
4192
4193     my @ready;
4194     while (scalar (@ready = $sel->can_read($timeout))) {
4195         my $count;
4196         if ($count = $fh->sysread($buf, 8192)) {
4197             if ($buf =~ /^(.*)\(qemu\) $/s) {
4198                 $res .= $1;
4199                 last;
4200             } else {
4201                 $res .= $buf;
4202             }
4203         } else {
4204             if (!defined($count)) {
4205                 die "$!\n";
4206             }
4207             last;
4208         }
4209     }
4210
4211     die "monitor read timeout\n" if !scalar(@ready);
4212
4213     return $res;
4214 }
4215
4216 sub qemu_block_resize {
4217     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4218
4219     my $running = check_running($vmid);
4220
4221     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4222
4223     return if !$running;
4224
4225     my $padding = (1024 - $size % 1024) % 1024;
4226     $size = $size + $padding;
4227
4228     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4229
4230 }
4231
4232 sub qemu_volume_snapshot {
4233     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4234
4235     my $running = check_running($vmid);
4236
4237     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4238         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4239     } else {
4240         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4241     }
4242 }
4243
4244 sub qemu_volume_snapshot_delete {
4245     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4246
4247     my $running = check_running($vmid);
4248
4249     if($running) {
4250
4251         $running = undef;
4252         my $conf = PVE::QemuConfig->load_config($vmid);
4253         PVE::QemuConfig->foreach_volume($conf, sub {
4254             my ($ds, $drive) = @_;
4255             $running = 1 if $drive->{file} eq $volid;
4256         });
4257     }
4258
4259     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4260         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4261     } else {
4262         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4263     }
4264 }
4265
4266 sub set_migration_caps {
4267     my ($vmid) = @_;
4268
4269     my $cap_ref = [];
4270
4271     my $enabled_cap = {
4272         "auto-converge" => 1,
4273         "xbzrle" => 1,
4274         "x-rdma-pin-all" => 0,
4275         "zero-blocks" => 0,
4276         "compress" => 0
4277     };
4278
4279     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4280
4281     for my $supported_capability (@$supported_capabilities) {
4282         push @$cap_ref, {
4283             capability => $supported_capability->{capability},
4284             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4285         };
4286     }
4287
4288     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4289 }
4290
4291 sub foreach_volid {
4292     my ($conf, $func, @param) = @_;
4293
4294     my $volhash = {};
4295
4296     my $test_volid = sub {
4297         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
4298
4299         return if !$volid;
4300
4301         $volhash->{$volid}->{cdrom} //= 1;
4302         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
4303
4304         $volhash->{$volid}->{replicate} //= 0;
4305         $volhash->{$volid}->{replicate} = 1 if $replicate;
4306
4307         $volhash->{$volid}->{shared} //= 0;
4308         $volhash->{$volid}->{shared} = 1 if $shared;
4309
4310         $volhash->{$volid}->{referenced_in_config} //= 0;
4311         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4312
4313         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4314             if defined($snapname);
4315         $volhash->{$volid}->{size} = $size if $size;
4316     };
4317
4318     PVE::QemuConfig->foreach_volume($conf, sub {
4319         my ($ds, $drive) = @_;
4320         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
4321     });
4322
4323     foreach my $snapname (keys %{$conf->{snapshots}}) {
4324         my $snap = $conf->{snapshots}->{$snapname};
4325         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
4326         $volhash->{$snap->{vmstate}}->{is_vmstate} = 1 if $snap->{vmstate};
4327         PVE::QemuConfig->foreach_volume($snap, sub {
4328             my ($ds, $drive) = @_;
4329             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
4330         });
4331     }
4332
4333     foreach my $volid (keys %$volhash) {
4334         &$func($volid, $volhash->{$volid}, @param);
4335     }
4336 }
4337
4338 my $fast_plug_option = {
4339     'lock' => 1,
4340     'name' => 1,
4341     'onboot' => 1,
4342     'shares' => 1,
4343     'startup' => 1,
4344     'description' => 1,
4345     'protection' => 1,
4346     'vmstatestorage' => 1,
4347     'hookscript' => 1,
4348     'tags' => 1,
4349 };
4350
4351 # hotplug changes in [PENDING]
4352 # $selection hash can be used to only apply specified options, for
4353 # example: { cores => 1 } (only apply changed 'cores')
4354 # $errors ref is used to return error messages
4355 sub vmconfig_hotplug_pending {
4356     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4357
4358     my $defaults = load_defaults();
4359     my $arch = get_vm_arch($conf);
4360     my $machine_type = get_vm_machine($conf, undef, $arch);
4361
4362     # commit values which do not have any impact on running VM first
4363     # Note: those option cannot raise errors, we we do not care about
4364     # $selection and always apply them.
4365
4366     my $add_error = sub {
4367         my ($opt, $msg) = @_;
4368         $errors->{$opt} = "hotplug problem - $msg";
4369     };
4370
4371     my $changes = 0;
4372     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4373         if ($fast_plug_option->{$opt}) {
4374             $conf->{$opt} = $conf->{pending}->{$opt};
4375             delete $conf->{pending}->{$opt};
4376             $changes = 1;
4377         }
4378     }
4379
4380     if ($changes) {
4381         PVE::QemuConfig->write_config($vmid, $conf);
4382     }
4383
4384     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4385
4386     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4387     foreach my $opt (sort keys %$pending_delete_hash) {
4388         next if $selection && !$selection->{$opt};
4389         my $force = $pending_delete_hash->{$opt}->{force};
4390         eval {
4391             if ($opt eq 'hotplug') {
4392                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4393             } elsif ($opt eq 'tablet') {
4394                 die "skip\n" if !$hotplug_features->{usb};
4395                 if ($defaults->{tablet}) {
4396                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4397                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4398                         if $arch eq 'aarch64';
4399                 } else {
4400                     vm_deviceunplug($vmid, $conf, 'tablet');
4401                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4402                 }
4403             } elsif ($opt =~ m/^usb\d+/) {
4404                 die "skip\n";
4405                 # since we cannot reliably hot unplug usb devices
4406                 # we are disabling it
4407                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4408                 vm_deviceunplug($vmid, $conf, $opt);
4409             } elsif ($opt eq 'vcpus') {
4410                 die "skip\n" if !$hotplug_features->{cpu};
4411                 qemu_cpu_hotplug($vmid, $conf, undef);
4412             } elsif ($opt eq 'balloon') {
4413                 # enable balloon device is not hotpluggable
4414                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4415                 # here we reset the ballooning value to memory
4416                 my $balloon = $conf->{memory} || $defaults->{memory};
4417                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4418             } elsif ($fast_plug_option->{$opt}) {
4419                 # do nothing
4420             } elsif ($opt =~ m/^net(\d+)$/) {
4421                 die "skip\n" if !$hotplug_features->{network};
4422                 vm_deviceunplug($vmid, $conf, $opt);
4423             } elsif (is_valid_drivename($opt)) {
4424                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4425                 vm_deviceunplug($vmid, $conf, $opt);
4426                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4427             } elsif ($opt =~ m/^memory$/) {
4428                 die "skip\n" if !$hotplug_features->{memory};
4429                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4430             } elsif ($opt eq 'cpuunits') {
4431                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4432             } elsif ($opt eq 'cpulimit') {
4433                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4434             } else {
4435                 die "skip\n";
4436             }
4437         };
4438         if (my $err = $@) {
4439             &$add_error($opt, $err) if $err ne "skip\n";
4440         } else {
4441             delete $conf->{$opt};
4442             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4443         }
4444     }
4445
4446     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4447     $apply_pending_cloudinit = sub {
4448         return if $apply_pending_cloudinit_done; # once is enough
4449         $apply_pending_cloudinit_done = 1; # once is enough
4450
4451         my ($key, $value) = @_;
4452
4453         my @cloudinit_opts = keys %$confdesc_cloudinit;
4454         foreach my $opt (keys %{$conf->{pending}}) {
4455             next if !grep { $_ eq $opt } @cloudinit_opts;
4456             $conf->{$opt} = delete $conf->{pending}->{$opt};
4457         }
4458
4459         my $new_conf = { %$conf };
4460         $new_conf->{$key} = $value;
4461         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4462     };
4463
4464     foreach my $opt (keys %{$conf->{pending}}) {
4465         next if $selection && !$selection->{$opt};
4466         my $value = $conf->{pending}->{$opt};
4467         eval {
4468             if ($opt eq 'hotplug') {
4469                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4470             } elsif ($opt eq 'tablet') {
4471                 die "skip\n" if !$hotplug_features->{usb};
4472                 if ($value == 1) {
4473                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4474                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4475                         if $arch eq 'aarch64';
4476                 } elsif ($value == 0) {
4477                     vm_deviceunplug($vmid, $conf, 'tablet');
4478                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4479                 }
4480             } elsif ($opt =~ m/^usb\d+$/) {
4481                 die "skip\n";
4482                 # since we cannot reliably hot unplug usb devices
4483                 # we are disabling it
4484                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4485                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4486                 die "skip\n" if !$d;
4487                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4488             } elsif ($opt eq 'vcpus') {
4489                 die "skip\n" if !$hotplug_features->{cpu};
4490                 qemu_cpu_hotplug($vmid, $conf, $value);
4491             } elsif ($opt eq 'balloon') {
4492                 # enable/disable balloning device is not hotpluggable
4493                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4494                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4495                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4496
4497                 # allow manual ballooning if shares is set to zero
4498                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4499                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4500                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4501                 }
4502             } elsif ($opt =~ m/^net(\d+)$/) {
4503                 # some changes can be done without hotplug
4504                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4505                                     $vmid, $opt, $value, $arch, $machine_type);
4506             } elsif (is_valid_drivename($opt)) {
4507                 die "skip\n" if $opt eq 'efidisk0';
4508                 # some changes can be done without hotplug
4509                 my $drive = parse_drive($opt, $value);
4510                 if (drive_is_cloudinit($drive)) {
4511                     &$apply_pending_cloudinit($opt, $value);
4512                 }
4513                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4514                                      $vmid, $opt, $value, $arch, $machine_type);
4515             } elsif ($opt =~ m/^memory$/) { #dimms
4516                 die "skip\n" if !$hotplug_features->{memory};
4517                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4518             } elsif ($opt eq 'cpuunits') {
4519                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4520             } elsif ($opt eq 'cpulimit') {
4521                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4522                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4523             } else {
4524                 die "skip\n";  # skip non-hot-pluggable options
4525             }
4526         };
4527         if (my $err = $@) {
4528             &$add_error($opt, $err) if $err ne "skip\n";
4529         } else {
4530             $conf->{$opt} = $value;
4531             delete $conf->{pending}->{$opt};
4532         }
4533     }
4534
4535     PVE::QemuConfig->write_config($vmid, $conf);
4536 }
4537
4538 sub try_deallocate_drive {
4539     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4540
4541     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4542         my $volid = $drive->{file};
4543         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4544             my $sid = PVE::Storage::parse_volume_id($volid);
4545             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4546
4547             # check if the disk is really unused
4548             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4549                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4550             PVE::Storage::vdisk_free($storecfg, $volid);
4551             return 1;
4552         } else {
4553             # If vm is not owner of this disk remove from config
4554             return 1;
4555         }
4556     }
4557
4558     return undef;
4559 }
4560
4561 sub vmconfig_delete_or_detach_drive {
4562     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4563
4564     my $drive = parse_drive($opt, $conf->{$opt});
4565
4566     my $rpcenv = PVE::RPCEnvironment::get();
4567     my $authuser = $rpcenv->get_user();
4568
4569     if ($force) {
4570         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4571         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4572     } else {
4573         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4574     }
4575 }
4576
4577
4578
4579 sub vmconfig_apply_pending {
4580     my ($vmid, $conf, $storecfg, $errors) = @_;
4581
4582     my $add_apply_error = sub {
4583         my ($opt, $msg) = @_;
4584         my $err_msg = "unable to apply pending change $opt : $msg";
4585         $errors->{$opt} = $err_msg;
4586         warn $err_msg;
4587     };
4588
4589     # cold plug
4590
4591     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4592     foreach my $opt (sort keys %$pending_delete_hash) {
4593         my $force = $pending_delete_hash->{$opt}->{force};
4594         eval {
4595             if ($opt =~ m/^unused/) {
4596                 die "internal error";
4597             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4598                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4599             }
4600         };
4601         if (my $err = $@) {
4602             $add_apply_error->($opt, $err);
4603         } else {
4604             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4605             delete $conf->{$opt};
4606         }
4607     }
4608
4609     PVE::QemuConfig->cleanup_pending($conf);
4610
4611     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4612         next if $opt eq 'delete'; # just to be sure
4613         eval {
4614             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4615                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4616             }
4617         };
4618         if (my $err = $@) {
4619             $add_apply_error->($opt, $err);
4620         } else {
4621             $conf->{$opt} = delete $conf->{pending}->{$opt};
4622         }
4623     }
4624
4625     # write all changes at once to avoid unnecessary i/o
4626     PVE::QemuConfig->write_config($vmid, $conf);
4627 }
4628
4629 sub vmconfig_update_net {
4630     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4631
4632     my $newnet = parse_net($value);
4633
4634     if ($conf->{$opt}) {
4635         my $oldnet = parse_net($conf->{$opt});
4636
4637         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4638             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4639             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4640             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4641
4642             # for non online change, we try to hot-unplug
4643             die "skip\n" if !$hotplug;
4644             vm_deviceunplug($vmid, $conf, $opt);
4645         } else {
4646
4647             die "internal error" if $opt !~ m/net(\d+)/;
4648             my $iface = "tap${vmid}i$1";
4649
4650             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4651                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4652                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4653                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4654                 PVE::Network::tap_unplug($iface);
4655
4656                 if ($have_sdn) {
4657                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4658                 } else {
4659                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4660                 }
4661             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4662                 # Rate can be applied on its own but any change above needs to
4663                 # include the rate in tap_plug since OVS resets everything.
4664                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4665             }
4666
4667             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4668                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4669             }
4670
4671             return 1;
4672         }
4673     }
4674
4675     if ($hotplug) {
4676         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4677     } else {
4678         die "skip\n";
4679     }
4680 }
4681
4682 sub vmconfig_update_disk {
4683     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4684
4685     my $drive = parse_drive($opt, $value);
4686
4687     if ($conf->{$opt}) {
4688
4689         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4690
4691             my $media = $drive->{media} || 'disk';
4692             my $oldmedia = $old_drive->{media} || 'disk';
4693             die "unable to change media type\n" if $media ne $oldmedia;
4694
4695             if (!drive_is_cdrom($old_drive)) {
4696
4697                 if ($drive->{file} ne $old_drive->{file}) {
4698
4699                     die "skip\n" if !$hotplug;
4700
4701                     # unplug and register as unused
4702                     vm_deviceunplug($vmid, $conf, $opt);
4703                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4704
4705                 } else {
4706                     # update existing disk
4707
4708                     # skip non hotpluggable value
4709                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4710                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4711                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4712                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4713                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4714                         die "skip\n";
4715                     }
4716
4717                     # apply throttle
4718                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4719                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4720                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4721                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4722                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4723                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4724                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4725                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4726                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4727                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4728                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4729                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4730                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4731                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4732                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4733                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4734                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4735                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4736
4737                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4738                                                    ($drive->{mbps} || 0)*1024*1024,
4739                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4740                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4741                                                    $drive->{iops} || 0,
4742                                                    $drive->{iops_rd} || 0,
4743                                                    $drive->{iops_wr} || 0,
4744                                                    ($drive->{mbps_max} || 0)*1024*1024,
4745                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4746                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4747                                                    $drive->{iops_max} || 0,
4748                                                    $drive->{iops_rd_max} || 0,
4749                                                    $drive->{iops_wr_max} || 0,
4750                                                    $drive->{bps_max_length} || 1,
4751                                                    $drive->{bps_rd_max_length} || 1,
4752                                                    $drive->{bps_wr_max_length} || 1,
4753                                                    $drive->{iops_max_length} || 1,
4754                                                    $drive->{iops_rd_max_length} || 1,
4755                                                    $drive->{iops_wr_max_length} || 1);
4756
4757                     }
4758
4759                     return 1;
4760                 }
4761
4762             } else { # cdrom
4763
4764                 if ($drive->{file} eq 'none') {
4765                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4766                     if (drive_is_cloudinit($old_drive)) {
4767                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4768                     }
4769                 } else {
4770                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4771
4772                     # force eject if locked
4773                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4774
4775                     if ($path) {
4776                         mon_cmd($vmid, "blockdev-change-medium",
4777                             id => "$opt", filename => "$path");
4778                     }
4779                 }
4780
4781                 return 1;
4782             }
4783         }
4784     }
4785
4786     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4787     # hotplug new disks
4788     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4789     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4790 }
4791
4792 # called in locked context by incoming migration
4793 sub vm_migrate_get_nbd_disks {
4794     my ($storecfg, $conf, $replicated_volumes) = @_;
4795
4796     my $local_volumes = {};
4797     PVE::QemuConfig->foreach_volume($conf, sub {
4798         my ($ds, $drive) = @_;
4799
4800         return if drive_is_cdrom($drive);
4801
4802         my $volid = $drive->{file};
4803
4804         return if !$volid;
4805
4806         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4807
4808         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4809         return if $scfg->{shared};
4810
4811         # replicated disks re-use existing state via bitmap
4812         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4813         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4814     });
4815     return $local_volumes;
4816 }
4817
4818 # called in locked context by incoming migration
4819 sub vm_migrate_alloc_nbd_disks {
4820     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4821
4822     my $format = undef;
4823
4824     my $nbd = {};
4825     foreach my $opt (sort keys %$source_volumes) {
4826         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4827
4828         if ($use_existing) {
4829             $nbd->{$opt}->{drivestr} = print_drive($drive);
4830             $nbd->{$opt}->{volid} = $volid;
4831             $nbd->{$opt}->{replicated} = 1;
4832             next;
4833         }
4834
4835         # If a remote storage is specified and the format of the original
4836         # volume is not available there, fall back to the default format.
4837         # Otherwise use the same format as the original.
4838         if (!$storagemap->{identity}) {
4839             $storeid = map_storage($storagemap, $storeid);
4840             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4841             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4842             my $fileFormat = qemu_img_format($scfg, $volname);
4843             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4844         } else {
4845             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4846             $format = qemu_img_format($scfg, $volname);
4847         }
4848
4849         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4850         my $newdrive = $drive;
4851         $newdrive->{format} = $format;
4852         $newdrive->{file} = $newvolid;
4853         my $drivestr = print_drive($newdrive);
4854         $nbd->{$opt}->{drivestr} = $drivestr;
4855         $nbd->{$opt}->{volid} = $newvolid;
4856     }
4857
4858     return $nbd;
4859 }
4860
4861 # see vm_start_nolock for parameters, additionally:
4862 # migrate_opts:
4863 #   storagemap = parsed storage map for allocating NBD disks
4864 sub vm_start {
4865     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4866
4867     return PVE::QemuConfig->lock_config($vmid, sub {
4868         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4869
4870         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4871
4872         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4873
4874         PVE::QemuConfig->check_lock($conf)
4875             if !($params->{skiplock} || $params->{resume});
4876
4877         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4878
4879         if (my $storagemap = $migrate_opts->{storagemap}) {
4880             my $replicated = $migrate_opts->{replicated_volumes};
4881             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4882             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4883
4884             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4885                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4886             }
4887         }
4888
4889         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4890     });
4891 }
4892
4893
4894 # params:
4895 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4896 #   skiplock => 0/1, skip checking for config lock
4897 #   forcemachine => to force Qemu machine (rollback/migration)
4898 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4899 #   timeout => in seconds
4900 #   paused => start VM in paused state (backup)
4901 #   resume => resume from hibernation
4902 # migrate_opts:
4903 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4904 #   migratedfrom => source node
4905 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4906 #   network => CIDR of migration network
4907 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4908 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4909 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4910 sub vm_start_nolock {
4911     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4912
4913     my $statefile = $params->{statefile};
4914     my $resume = $params->{resume};
4915
4916     my $migratedfrom = $migrate_opts->{migratedfrom};
4917     my $migration_type = $migrate_opts->{type};
4918
4919     my $res = {};
4920
4921     # clean up leftover reboot request files
4922     eval { clear_reboot_request($vmid); };
4923     warn $@ if $@;
4924
4925     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4926         vmconfig_apply_pending($vmid, $conf, $storecfg);
4927         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4928     }
4929
4930     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4931
4932     my $defaults = load_defaults();
4933
4934     # set environment variable useful inside network script
4935     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4936
4937     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4938
4939     my $forcemachine = $params->{forcemachine};
4940     my $forcecpu = $params->{forcecpu};
4941     if ($resume) {
4942         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4943         $forcemachine = $conf->{runningmachine};
4944         $forcecpu = $conf->{runningcpu};
4945         print "Resuming suspended VM\n";
4946     }
4947
4948     my ($cmd, $vollist, $spice_port) =
4949         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4950
4951     my $migration_ip;
4952     my $get_migration_ip = sub {
4953         my ($nodename) = @_;
4954
4955         return $migration_ip if defined($migration_ip);
4956
4957         my $cidr = $migrate_opts->{network};
4958
4959         if (!defined($cidr)) {
4960             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4961             $cidr = $dc_conf->{migration}->{network};
4962         }
4963
4964         if (defined($cidr)) {
4965             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4966
4967             die "could not get IP: no address configured on local " .
4968                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4969
4970             die "could not get IP: multiple addresses configured on local " .
4971                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4972
4973             $migration_ip = @$ips[0];
4974         }
4975
4976         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4977             if !defined($migration_ip);
4978
4979         return $migration_ip;
4980     };
4981
4982     my $migrate_uri;
4983     if ($statefile) {
4984         if ($statefile eq 'tcp') {
4985             my $localip = "localhost";
4986             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4987             my $nodename = nodename();
4988
4989             if (!defined($migration_type)) {
4990                 if (defined($datacenterconf->{migration}->{type})) {
4991                     $migration_type = $datacenterconf->{migration}->{type};
4992                 } else {
4993                     $migration_type = 'secure';
4994                 }
4995             }
4996
4997             if ($migration_type eq 'insecure') {
4998                 $localip = $get_migration_ip->($nodename);
4999                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5000             }
5001
5002             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5003             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5004             $migrate_uri = "tcp:${localip}:${migrate_port}";
5005             push @$cmd, '-incoming', $migrate_uri;
5006             push @$cmd, '-S';
5007
5008         } elsif ($statefile eq 'unix') {
5009             # should be default for secure migrations as a ssh TCP forward
5010             # tunnel is not deterministic reliable ready and fails regurarly
5011             # to set up in time, so use UNIX socket forwards
5012             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5013             unlink $socket_addr;
5014
5015             $migrate_uri = "unix:$socket_addr";
5016
5017             push @$cmd, '-incoming', $migrate_uri;
5018             push @$cmd, '-S';
5019
5020         } elsif (-e $statefile) {
5021             push @$cmd, '-loadstate', $statefile;
5022         } else {
5023             my $statepath = PVE::Storage::path($storecfg, $statefile);
5024             push @$vollist, $statefile;
5025             push @$cmd, '-loadstate', $statepath;
5026         }
5027     } elsif ($params->{paused}) {
5028         push @$cmd, '-S';
5029     }
5030
5031     # host pci devices
5032     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5033       my $d = parse_hostpci($conf->{"hostpci$i"});
5034       next if !$d;
5035       my $pcidevices = $d->{pciid};
5036       foreach my $pcidevice (@$pcidevices) {
5037             my $pciid = $pcidevice->{id};
5038
5039             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5040             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5041             die "no pci device info for device '$pciid'\n" if !$info;
5042
5043             if ($d->{mdev}) {
5044                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5045                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5046             } else {
5047                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5048                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5049                 die "can't reset pci device '$pciid'\n"
5050                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5051             }
5052       }
5053     }
5054
5055     PVE::Storage::activate_volumes($storecfg, $vollist);
5056
5057     eval {
5058         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5059             outfunc => sub {}, errfunc => sub {});
5060     };
5061     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5062     # timeout should be more than enough here...
5063     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5064
5065     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5066                                               : $defaults->{cpuunits};
5067
5068     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5069     my %run_params = (
5070         timeout => $statefile ? undef : $start_timeout,
5071         umask => 0077,
5072         noerr => 1,
5073     );
5074
5075     # when migrating, prefix QEMU output so other side can pick up any
5076     # errors that might occur and show the user
5077     if ($migratedfrom) {
5078         $run_params{quiet} = 1;
5079         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5080     }
5081
5082     my %properties = (
5083         Slice => 'qemu.slice',
5084         KillMode => 'none',
5085         CPUShares => $cpuunits
5086     );
5087
5088     if (my $cpulimit = $conf->{cpulimit}) {
5089         $properties{CPUQuota} = int($cpulimit * 100);
5090     }
5091     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5092
5093     my $run_qemu = sub {
5094         PVE::Tools::run_fork sub {
5095             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5096
5097             my $exitcode = run_command($cmd, %run_params);
5098             die "QEMU exited with code $exitcode\n" if $exitcode;
5099         };
5100     };
5101
5102     if ($conf->{hugepages}) {
5103
5104         my $code = sub {
5105             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5106             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5107
5108             PVE::QemuServer::Memory::hugepages_mount();
5109             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5110
5111             eval { $run_qemu->() };
5112             if (my $err = $@) {
5113                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5114                 die $err;
5115             }
5116
5117             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5118         };
5119         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5120
5121     } else {
5122         eval { $run_qemu->() };
5123     }
5124
5125     if (my $err = $@) {
5126         # deactivate volumes if start fails
5127         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5128         die "start failed: $err";
5129     }
5130
5131     print "migration listens on $migrate_uri\n" if $migrate_uri;
5132     $res->{migrate_uri} = $migrate_uri;
5133
5134     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5135         eval { mon_cmd($vmid, "cont"); };
5136         warn $@ if $@;
5137     }
5138
5139     #start nbd server for storage migration
5140     if (my $nbd = $migrate_opts->{nbd}) {
5141         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5142
5143         my $migrate_storage_uri;
5144         # nbd_protocol_version > 0 for unix socket support
5145         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5146             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5147             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5148             $migrate_storage_uri = "nbd:unix:$socket_path";
5149         } else {
5150             my $nodename = nodename();
5151             my $localip = $get_migration_ip->($nodename);
5152             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5153             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5154
5155             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5156             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5157             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5158         }
5159
5160         $res->{migrate_storage_uri} = $migrate_storage_uri;
5161
5162         foreach my $opt (sort keys %$nbd) {
5163             my $drivestr = $nbd->{$opt}->{drivestr};
5164             my $volid = $nbd->{$opt}->{volid};
5165             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5166             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5167             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5168             print "re-using replicated volume: $opt - $volid\n"
5169                 if $nbd->{$opt}->{replicated};
5170
5171             $res->{drives}->{$opt} = $nbd->{$opt};
5172             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5173         }
5174     }
5175
5176     if ($migratedfrom) {
5177         eval {
5178             set_migration_caps($vmid);
5179         };
5180         warn $@ if $@;
5181
5182         if ($spice_port) {
5183             print "spice listens on port $spice_port\n";
5184             $res->{spice_port} = $spice_port;
5185             if ($migrate_opts->{spice_ticket}) {
5186                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5187                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5188             }
5189         }
5190
5191     } else {
5192         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5193             if !$statefile && $conf->{balloon};
5194
5195         foreach my $opt (keys %$conf) {
5196             next if $opt !~  m/^net\d+$/;
5197             my $nicconf = parse_net($conf->{$opt});
5198             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5199         }
5200     }
5201
5202     mon_cmd($vmid, 'qom-set',
5203                 path => "machine/peripheral/balloon0",
5204                 property => "guest-stats-polling-interval",
5205                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5206
5207     if ($resume) {
5208         print "Resumed VM, removing state\n";
5209         if (my $vmstate = $conf->{vmstate}) {
5210             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5211             PVE::Storage::vdisk_free($storecfg, $vmstate);
5212         }
5213         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5214         PVE::QemuConfig->write_config($vmid, $conf);
5215     }
5216
5217     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5218
5219     return $res;
5220 }
5221
5222 sub vm_commandline {
5223     my ($storecfg, $vmid, $snapname) = @_;
5224
5225     my $conf = PVE::QemuConfig->load_config($vmid);
5226     my $forcemachine;
5227     my $forcecpu;
5228
5229     if ($snapname) {
5230         my $snapshot = $conf->{snapshots}->{$snapname};
5231         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5232
5233         # check for machine or CPU overrides in snapshot
5234         $forcemachine = $snapshot->{runningmachine};
5235         $forcecpu = $snapshot->{runningcpu};
5236
5237         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5238
5239         $conf = $snapshot;
5240     }
5241
5242     my $defaults = load_defaults();
5243
5244     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5245         $forcemachine, $forcecpu);
5246
5247     return PVE::Tools::cmd2string($cmd);
5248 }
5249
5250 sub vm_reset {
5251     my ($vmid, $skiplock) = @_;
5252
5253     PVE::QemuConfig->lock_config($vmid, sub {
5254
5255         my $conf = PVE::QemuConfig->load_config($vmid);
5256
5257         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5258
5259         mon_cmd($vmid, "system_reset");
5260     });
5261 }
5262
5263 sub get_vm_volumes {
5264     my ($conf) = @_;
5265
5266     my $vollist = [];
5267     foreach_volid($conf, sub {
5268         my ($volid, $attr) = @_;
5269
5270         return if $volid =~ m|^/|;
5271
5272         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5273         return if !$sid;
5274
5275         push @$vollist, $volid;
5276     });
5277
5278     return $vollist;
5279 }
5280
5281 sub vm_stop_cleanup {
5282     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5283
5284     eval {
5285
5286         if (!$keepActive) {
5287             my $vollist = get_vm_volumes($conf);
5288             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5289         }
5290
5291         foreach my $ext (qw(mon qmp pid vnc qga)) {
5292             unlink "/var/run/qemu-server/${vmid}.$ext";
5293         }
5294
5295         if ($conf->{ivshmem}) {
5296             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5297             # just delete it for now, VMs which have this already open do not
5298             # are affected, but new VMs will get a separated one. If this
5299             # becomes an issue we either add some sort of ref-counting or just
5300             # add a "don't delete on stop" flag to the ivshmem format.
5301             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5302         }
5303
5304         foreach my $key (keys %$conf) {
5305             next if $key !~ m/^hostpci(\d+)$/;
5306             my $hostpciindex = $1;
5307             my $d = parse_hostpci($conf->{$key});
5308             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5309
5310             foreach my $pci (@{$d->{pciid}}) {
5311                 my $pciid = $pci->{id};
5312                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5313             }
5314         }
5315
5316         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5317     };
5318     warn $@ if $@; # avoid errors - just warn
5319 }
5320
5321 # call only in locked context
5322 sub _do_vm_stop {
5323     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5324
5325     my $pid = check_running($vmid, $nocheck);
5326     return if !$pid;
5327
5328     my $conf;
5329     if (!$nocheck) {
5330         $conf = PVE::QemuConfig->load_config($vmid);
5331         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5332         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5333             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5334             $timeout = $opts->{down} if $opts->{down};
5335         }
5336         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5337     }
5338
5339     eval {
5340         if ($shutdown) {
5341             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5342                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5343             } else {
5344                 mon_cmd($vmid, "system_powerdown");
5345             }
5346         } else {
5347             mon_cmd($vmid, "quit");
5348         }
5349     };
5350     my $err = $@;
5351
5352     if (!$err) {
5353         $timeout = 60 if !defined($timeout);
5354
5355         my $count = 0;
5356         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5357             $count++;
5358             sleep 1;
5359         }
5360
5361         if ($count >= $timeout) {
5362             if ($force) {
5363                 warn "VM still running - terminating now with SIGTERM\n";
5364                 kill 15, $pid;
5365             } else {
5366                 die "VM quit/powerdown failed - got timeout\n";
5367             }
5368         } else {
5369             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5370             return;
5371         }
5372     } else {
5373         if ($force) {
5374             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5375             kill 15, $pid;
5376         } else {
5377             die "VM quit/powerdown failed\n";
5378         }
5379     }
5380
5381     # wait again
5382     $timeout = 10;
5383
5384     my $count = 0;
5385     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5386         $count++;
5387         sleep 1;
5388     }
5389
5390     if ($count >= $timeout) {
5391         warn "VM still running - terminating now with SIGKILL\n";
5392         kill 9, $pid;
5393         sleep 1;
5394     }
5395
5396     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5397 }
5398
5399 # Note: use $nocheck to skip tests if VM configuration file exists.
5400 # We need that when migration VMs to other nodes (files already moved)
5401 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5402 sub vm_stop {
5403     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5404
5405     $force = 1 if !defined($force) && !$shutdown;
5406
5407     if ($migratedfrom){
5408         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5409         kill 15, $pid if $pid;
5410         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5411         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5412         return;
5413     }
5414
5415     PVE::QemuConfig->lock_config($vmid, sub {
5416         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5417    });
5418 }
5419
5420 sub vm_reboot {
5421     my ($vmid, $timeout) = @_;
5422
5423     PVE::QemuConfig->lock_config($vmid, sub {
5424         eval {
5425
5426             # only reboot if running, as qmeventd starts it again on a stop event
5427             return if !check_running($vmid);
5428
5429             create_reboot_request($vmid);
5430
5431             my $storecfg = PVE::Storage::config();
5432             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5433
5434         };
5435         if (my $err = $@) {
5436             # avoid that the next normal shutdown will be confused for a reboot
5437             clear_reboot_request($vmid);
5438             die $err;
5439         }
5440    });
5441 }
5442
5443 # note: if using the statestorage parameter, the caller has to check privileges
5444 sub vm_suspend {
5445     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5446
5447     my $conf;
5448     my $path;
5449     my $storecfg;
5450     my $vmstate;
5451
5452     PVE::QemuConfig->lock_config($vmid, sub {
5453
5454         $conf = PVE::QemuConfig->load_config($vmid);
5455
5456         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5457         PVE::QemuConfig->check_lock($conf)
5458             if !($skiplock || $is_backing_up);
5459
5460         die "cannot suspend to disk during backup\n"
5461             if $is_backing_up && $includestate;
5462
5463         if ($includestate) {
5464             $conf->{lock} = 'suspending';
5465             my $date = strftime("%Y-%m-%d", localtime(time()));
5466             $storecfg = PVE::Storage::config();
5467             if (!$statestorage) {
5468                 $statestorage = find_vmstate_storage($conf, $storecfg);
5469                 # check permissions for the storage
5470                 my $rpcenv = PVE::RPCEnvironment::get();
5471                 if ($rpcenv->{type} ne 'cli') {
5472                     my $authuser = $rpcenv->get_user();
5473                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5474                 }
5475             }
5476
5477
5478             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5479             $path = PVE::Storage::path($storecfg, $vmstate);
5480             PVE::QemuConfig->write_config($vmid, $conf);
5481         } else {
5482             mon_cmd($vmid, "stop");
5483         }
5484     });
5485
5486     if ($includestate) {
5487         # save vm state
5488         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5489
5490         eval {
5491             mon_cmd($vmid, "savevm-start", statefile => $path);
5492             for(;;) {
5493                 my $state = mon_cmd($vmid, "query-savevm");
5494                 if (!$state->{status}) {
5495                     die "savevm not active\n";
5496                 } elsif ($state->{status} eq 'active') {
5497                     sleep(1);
5498                     next;
5499                 } elsif ($state->{status} eq 'completed') {
5500                     print "State saved, quitting\n";
5501                     last;
5502                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5503                     die "query-savevm failed with error '$state->{error}'\n"
5504                 } else {
5505                     die "query-savevm returned status '$state->{status}'\n";
5506                 }
5507             }
5508         };
5509         my $err = $@;
5510
5511         PVE::QemuConfig->lock_config($vmid, sub {
5512             $conf = PVE::QemuConfig->load_config($vmid);
5513             if ($err) {
5514                 # cleanup, but leave suspending lock, to indicate something went wrong
5515                 eval {
5516                     mon_cmd($vmid, "savevm-end");
5517                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5518                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5519                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5520                     PVE::QemuConfig->write_config($vmid, $conf);
5521                 };
5522                 warn $@ if $@;
5523                 die $err;
5524             }
5525
5526             die "lock changed unexpectedly\n"
5527                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5528
5529             mon_cmd($vmid, "quit");
5530             $conf->{lock} = 'suspended';
5531             PVE::QemuConfig->write_config($vmid, $conf);
5532         });
5533     }
5534 }
5535
5536 sub vm_resume {
5537     my ($vmid, $skiplock, $nocheck) = @_;
5538
5539     PVE::QemuConfig->lock_config($vmid, sub {
5540         my $res = mon_cmd($vmid, 'query-status');
5541         my $resume_cmd = 'cont';
5542
5543         if ($res->{status} && $res->{status} eq 'suspended') {
5544             $resume_cmd = 'system_wakeup';
5545         }
5546
5547         if (!$nocheck) {
5548
5549             my $conf = PVE::QemuConfig->load_config($vmid);
5550
5551             PVE::QemuConfig->check_lock($conf)
5552                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5553         }
5554
5555         mon_cmd($vmid, $resume_cmd);
5556     });
5557 }
5558
5559 sub vm_sendkey {
5560     my ($vmid, $skiplock, $key) = @_;
5561
5562     PVE::QemuConfig->lock_config($vmid, sub {
5563
5564         my $conf = PVE::QemuConfig->load_config($vmid);
5565
5566         # there is no qmp command, so we use the human monitor command
5567         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5568         die $res if $res ne '';
5569     });
5570 }
5571
5572 # vzdump restore implementaion
5573
5574 sub tar_archive_read_firstfile {
5575     my $archive = shift;
5576
5577     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5578
5579     # try to detect archive type first
5580     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5581         die "unable to open file '$archive'\n";
5582     my $firstfile = <$fh>;
5583     kill 15, $pid;
5584     close $fh;
5585
5586     die "ERROR: archive contaions no data\n" if !$firstfile;
5587     chomp $firstfile;
5588
5589     return $firstfile;
5590 }
5591
5592 sub tar_restore_cleanup {
5593     my ($storecfg, $statfile) = @_;
5594
5595     print STDERR "starting cleanup\n";
5596
5597     if (my $fd = IO::File->new($statfile, "r")) {
5598         while (defined(my $line = <$fd>)) {
5599             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5600                 my $volid = $2;
5601                 eval {
5602                     if ($volid =~ m|^/|) {
5603                         unlink $volid || die 'unlink failed\n';
5604                     } else {
5605                         PVE::Storage::vdisk_free($storecfg, $volid);
5606                     }
5607                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5608                 };
5609                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5610             } else {
5611                 print STDERR "unable to parse line in statfile - $line";
5612             }
5613         }
5614         $fd->close();
5615     }
5616 }
5617
5618 sub restore_file_archive {
5619     my ($archive, $vmid, $user, $opts) = @_;
5620
5621     my $format = $opts->{format};
5622     my $comp;
5623
5624     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5625         $format = 'tar' if !$format;
5626         $comp = 'gzip';
5627     } elsif ($archive =~ m/\.tar$/) {
5628         $format = 'tar' if !$format;
5629     } elsif ($archive =~ m/.tar.lzo$/) {
5630         $format = 'tar' if !$format;
5631         $comp = 'lzop';
5632     } elsif ($archive =~ m/\.vma$/) {
5633         $format = 'vma' if !$format;
5634     } elsif ($archive =~ m/\.vma\.gz$/) {
5635         $format = 'vma' if !$format;
5636         $comp = 'gzip';
5637     } elsif ($archive =~ m/\.vma\.lzo$/) {
5638         $format = 'vma' if !$format;
5639         $comp = 'lzop';
5640     } else {
5641         $format = 'vma' if !$format; # default
5642     }
5643
5644     # try to detect archive format
5645     if ($format eq 'tar') {
5646         return restore_tar_archive($archive, $vmid, $user, $opts);
5647     } else {
5648         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5649     }
5650 }
5651
5652 # hepler to remove disks that will not be used after restore
5653 my $restore_cleanup_oldconf = sub {
5654     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5655
5656     PVE::QemuConfig->foreach_volume($oldconf, sub {
5657         my ($ds, $drive) = @_;
5658
5659         return if drive_is_cdrom($drive, 1);
5660
5661         my $volid = $drive->{file};
5662         return if !$volid || $volid =~ m|^/|;
5663
5664         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5665         return if !$path || !$owner || ($owner != $vmid);
5666
5667         # Note: only delete disk we want to restore
5668         # other volumes will become unused
5669         if ($virtdev_hash->{$ds}) {
5670             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5671             if (my $err = $@) {
5672                 warn $err;
5673             }
5674         }
5675     });
5676
5677     # delete vmstate files, after the restore we have no snapshots anymore
5678     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5679         my $snap = $oldconf->{snapshots}->{$snapname};
5680         if ($snap->{vmstate}) {
5681             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5682             if (my $err = $@) {
5683                 warn $err;
5684             }
5685         }
5686     }
5687 };
5688
5689 # Helper to parse vzdump backup device hints
5690 #
5691 # $rpcenv: Environment, used to ckeck storage permissions
5692 # $user: User ID, to check storage permissions
5693 # $storecfg: Storage configuration
5694 # $fh: the file handle for reading the configuration
5695 # $devinfo: should contain device sizes for all backu-up'ed devices
5696 # $options: backup options (pool, default storage)
5697 #
5698 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5699 my $parse_backup_hints = sub {
5700     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5701
5702     my $virtdev_hash = {};
5703
5704     while (defined(my $line = <$fh>)) {
5705         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5706             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5707             die "archive does not contain data for drive '$virtdev'\n"
5708                 if !$devinfo->{$devname};
5709
5710             if (defined($options->{storage})) {
5711                 $storeid = $options->{storage} || 'local';
5712             } elsif (!$storeid) {
5713                 $storeid = 'local';
5714             }
5715             $format = 'raw' if !$format;
5716             $devinfo->{$devname}->{devname} = $devname;
5717             $devinfo->{$devname}->{virtdev} = $virtdev;
5718             $devinfo->{$devname}->{format} = $format;
5719             $devinfo->{$devname}->{storeid} = $storeid;
5720
5721             # check permission on storage
5722             my $pool = $options->{pool}; # todo: do we need that?
5723             if ($user ne 'root@pam') {
5724                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5725             }
5726
5727             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5728         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5729             my $virtdev = $1;
5730             my $drive = parse_drive($virtdev, $2);
5731             if (drive_is_cloudinit($drive)) {
5732                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5733                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5734                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5735
5736                 $virtdev_hash->{$virtdev} = {
5737                     format => $format,
5738                     storeid => $options->{storage} // $storeid,
5739                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5740                     is_cloudinit => 1,
5741                 };
5742             }
5743         }
5744     }
5745
5746     return $virtdev_hash;
5747 };
5748
5749 # Helper to allocate and activate all volumes required for a restore
5750 #
5751 # $storecfg: Storage configuration
5752 # $virtdev_hash: as returned by parse_backup_hints()
5753 #
5754 # Returns: { $virtdev => $volid }
5755 my $restore_allocate_devices = sub {
5756     my ($storecfg, $virtdev_hash, $vmid) = @_;
5757
5758     my $map = {};
5759     foreach my $virtdev (sort keys %$virtdev_hash) {
5760         my $d = $virtdev_hash->{$virtdev};
5761         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5762         my $storeid = $d->{storeid};
5763         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5764
5765         # test if requested format is supported
5766         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5767         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5768         $d->{format} = $defFormat if !$supported;
5769
5770         my $name;
5771         if ($d->{is_cloudinit}) {
5772             $name = "vm-$vmid-cloudinit";
5773             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5774         }
5775
5776         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5777
5778         print STDERR "new volume ID is '$volid'\n";
5779         $d->{volid} = $volid;
5780
5781         PVE::Storage::activate_volumes($storecfg, [$volid]);
5782
5783         $map->{$virtdev} = $volid;
5784     }
5785
5786     return $map;
5787 };
5788
5789 my $restore_update_config_line = sub {
5790     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5791
5792     return if $line =~ m/^\#qmdump\#/;
5793     return if $line =~ m/^\#vzdump\#/;
5794     return if $line =~ m/^lock:/;
5795     return if $line =~ m/^unused\d+:/;
5796     return if $line =~ m/^parent:/;
5797
5798     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5799     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5800         # try to convert old 1.X settings
5801         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5802         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5803             my ($model, $macaddr) = split(/\=/, $devconfig);
5804             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5805             my $net = {
5806                 model => $model,
5807                 bridge => "vmbr$ind",
5808                 macaddr => $macaddr,
5809             };
5810             my $netstr = print_net($net);
5811
5812             print $outfd "net$cookie->{netcount}: $netstr\n";
5813             $cookie->{netcount}++;
5814         }
5815     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5816         my ($id, $netstr) = ($1, $2);
5817         my $net = parse_net($netstr);
5818         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5819         $netstr = print_net($net);
5820         print $outfd "$id: $netstr\n";
5821     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5822         my $virtdev = $1;
5823         my $value = $3;
5824         my $di = parse_drive($virtdev, $value);
5825         if (defined($di->{backup}) && !$di->{backup}) {
5826             print $outfd "#$line";
5827         } elsif ($map->{$virtdev}) {
5828             delete $di->{format}; # format can change on restore
5829             $di->{file} = $map->{$virtdev};
5830             $value = print_drive($di);
5831             print $outfd "$virtdev: $value\n";
5832         } else {
5833             print $outfd $line;
5834         }
5835     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5836         my $vmgenid = $1;
5837         if ($vmgenid ne '0') {
5838             # always generate a new vmgenid if there was a valid one setup
5839             $vmgenid = generate_uuid();
5840         }
5841         print $outfd "vmgenid: $vmgenid\n";
5842     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5843         my ($uuid, $uuid_str);
5844         UUID::generate($uuid);
5845         UUID::unparse($uuid, $uuid_str);
5846         my $smbios1 = parse_smbios1($2);
5847         $smbios1->{uuid} = $uuid_str;
5848         print $outfd $1.print_smbios1($smbios1)."\n";
5849     } else {
5850         print $outfd $line;
5851     }
5852 };
5853
5854 my $restore_deactivate_volumes = sub {
5855     my ($storecfg, $devinfo) = @_;
5856
5857     my $vollist = [];
5858     foreach my $devname (keys %$devinfo) {
5859         my $volid = $devinfo->{$devname}->{volid};
5860         push @$vollist, $volid if $volid;
5861     }
5862
5863     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5864 };
5865
5866 my $restore_destroy_volumes = sub {
5867     my ($storecfg, $devinfo) = @_;
5868
5869     foreach my $devname (keys %$devinfo) {
5870         my $volid = $devinfo->{$devname}->{volid};
5871         next if !$volid;
5872         eval {
5873             if ($volid =~ m|^/|) {
5874                 unlink $volid || die 'unlink failed\n';
5875             } else {
5876                 PVE::Storage::vdisk_free($storecfg, $volid);
5877             }
5878             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5879         };
5880         print STDERR "unable to cleanup '$volid' - $@" if $@;
5881     }
5882 };
5883
5884 sub scan_volids {
5885     my ($cfg, $vmid) = @_;
5886
5887     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5888
5889     my $volid_hash = {};
5890     foreach my $storeid (keys %$info) {
5891         foreach my $item (@{$info->{$storeid}}) {
5892             next if !($item->{volid} && $item->{size});
5893             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5894             $volid_hash->{$item->{volid}} = $item;
5895         }
5896     }
5897
5898     return $volid_hash;
5899 }
5900
5901 sub update_disk_config {
5902     my ($vmid, $conf, $volid_hash) = @_;
5903
5904     my $changes;
5905     my $prefix = "VM $vmid:";
5906
5907     # used and unused disks
5908     my $referenced = {};
5909
5910     # Note: it is allowed to define multiple storages with same path (alias), so
5911     # we need to check both 'volid' and real 'path' (two different volid can point
5912     # to the same path).
5913
5914     my $referencedpath = {};
5915
5916     # update size info
5917     PVE::QemuConfig->foreach_volume($conf, sub {
5918         my ($opt, $drive) = @_;
5919
5920         my $volid = $drive->{file};
5921         return if !$volid;
5922
5923         # mark volid as "in-use" for next step
5924         $referenced->{$volid} = 1;
5925         if ($volid_hash->{$volid} &&
5926             (my $path = $volid_hash->{$volid}->{path})) {
5927             $referencedpath->{$path} = 1;
5928         }
5929
5930         return if drive_is_cdrom($drive);
5931         return if !$volid_hash->{$volid};
5932
5933         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5934         if (defined($updated)) {
5935             $changes = 1;
5936             $conf->{$opt} = print_drive($updated);
5937             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5938         }
5939     });
5940
5941     # remove 'unusedX' entry if volume is used
5942     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5943         my ($opt, $drive) = @_;
5944
5945         my $volid = $drive->{file};
5946         return if !$volid;
5947
5948         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5949         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5950             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5951             $changes = 1;
5952             delete $conf->{$opt};
5953         }
5954
5955         $referenced->{$volid} = 1;
5956         $referencedpath->{$path} = 1 if $path;
5957     });
5958
5959     foreach my $volid (sort keys %$volid_hash) {
5960         next if $volid =~ m/vm-$vmid-state-/;
5961         next if $referenced->{$volid};
5962         my $path = $volid_hash->{$volid}->{path};
5963         next if !$path; # just to be sure
5964         next if $referencedpath->{$path};
5965         $changes = 1;
5966         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5967         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5968         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5969     }
5970
5971     return $changes;
5972 }
5973
5974 sub rescan {
5975     my ($vmid, $nolock, $dryrun) = @_;
5976
5977     my $cfg = PVE::Storage::config();
5978
5979     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5980     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5981     foreach my $stor (keys %{$cfg->{ids}}) {
5982         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5983     }
5984
5985     print "rescan volumes...\n";
5986     my $volid_hash = scan_volids($cfg, $vmid);
5987
5988     my $updatefn =  sub {
5989         my ($vmid) = @_;
5990
5991         my $conf = PVE::QemuConfig->load_config($vmid);
5992
5993         PVE::QemuConfig->check_lock($conf);
5994
5995         my $vm_volids = {};
5996         foreach my $volid (keys %$volid_hash) {
5997             my $info = $volid_hash->{$volid};
5998             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5999         }
6000
6001         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6002
6003         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6004     };
6005
6006     if (defined($vmid)) {
6007         if ($nolock) {
6008             &$updatefn($vmid);
6009         } else {
6010             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6011         }
6012     } else {
6013         my $vmlist = config_list();
6014         foreach my $vmid (keys %$vmlist) {
6015             if ($nolock) {
6016                 &$updatefn($vmid);
6017             } else {
6018                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6019             }
6020         }
6021     }
6022 }
6023
6024 sub restore_proxmox_backup_archive {
6025     my ($archive, $vmid, $user, $options) = @_;
6026
6027     my $storecfg = PVE::Storage::config();
6028
6029     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6030     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6031
6032     my $server = $scfg->{server};
6033     my $datastore = $scfg->{datastore};
6034     my $username = $scfg->{username} // 'root@pam';
6035     my $fingerprint = $scfg->{fingerprint};
6036
6037     my $repo = "$username\@$server:$datastore";
6038     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6039     local $ENV{PBS_PASSWORD} = $password;
6040     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6041
6042     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6043         PVE::Storage::parse_volname($storecfg, $archive);
6044
6045     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6046
6047     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6048
6049     my $tmpdir = "/var/tmp/vzdumptmp$$";
6050     rmtree $tmpdir;
6051     mkpath $tmpdir;
6052
6053     my $conffile = PVE::QemuConfig->config_file($vmid);
6054     my $tmpfn = "$conffile.$$.tmp";
6055      # disable interrupts (always do cleanups)
6056     local $SIG{INT} =
6057         local $SIG{TERM} =
6058         local $SIG{QUIT} =
6059         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6060
6061     # Note: $oldconf is undef if VM does not exists
6062     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6063     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6064
6065     my $rpcenv = PVE::RPCEnvironment::get();
6066     my $devinfo = {};
6067
6068     eval {
6069         # enable interrupts
6070         local $SIG{INT} =
6071             local $SIG{TERM} =
6072             local $SIG{QUIT} =
6073             local $SIG{HUP} =
6074             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6075
6076         my $cfgfn = "$tmpdir/qemu-server.conf";
6077         my $firewall_config_fn = "$tmpdir/fw.conf";
6078         my $index_fn = "$tmpdir/index.json";
6079
6080         my $cmd = "restore";
6081
6082         my $param = [$pbs_backup_name, "index.json", $index_fn];
6083         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6084         my $index = PVE::Tools::file_get_contents($index_fn);
6085         $index = decode_json($index);
6086
6087         # print Dumper($index);
6088         foreach my $info (@{$index->{files}}) {
6089             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6090                 my $devname = $1;
6091                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6092                     $devinfo->{$devname}->{size} = $1;
6093                 } else {
6094                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6095                 }
6096             }
6097         }
6098
6099         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6100         if (!$is_qemu_server_backup) {
6101             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6102         }
6103         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6104
6105         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6106         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6107
6108         if ($has_firewall_config) {
6109             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6110             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6111
6112             my $pve_firewall_dir = '/etc/pve/firewall';
6113             mkdir $pve_firewall_dir; # make sure the dir exists
6114             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6115         }
6116
6117         my $fh = IO::File->new($cfgfn, "r") ||
6118             "unable to read qemu-server.conf - $!\n";
6119
6120         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6121
6122         # fixme: rate limit?
6123
6124         # create empty/temp config
6125         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6126
6127         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6128
6129         # allocate volumes
6130         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6131
6132         foreach my $virtdev (sort keys %$virtdev_hash) {
6133             my $d = $virtdev_hash->{$virtdev};
6134             next if $d->{is_cloudinit}; # no need to restore cloudinit
6135
6136             my $volid = $d->{volid};
6137
6138             my $path = PVE::Storage::path($storecfg, $volid);
6139
6140             my $pbs_restore_cmd = [
6141                 '/usr/bin/pbs-restore',
6142                 '--repository', $repo,
6143                 $pbs_backup_name,
6144                 "$d->{devname}.img.fidx",
6145                 $path,
6146                 '--verbose',
6147                 ];
6148
6149             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6150                 push @$pbs_restore_cmd, '--skip-zero';
6151             }
6152
6153             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6154             print "restore proxmox backup image: $dbg_cmdstring\n";
6155             run_command($pbs_restore_cmd);
6156         }
6157
6158         $fh->seek(0, 0) || die "seek failed - $!\n";
6159
6160         my $outfd = new IO::File ($tmpfn, "w") ||
6161             die "unable to write config for VM $vmid\n";
6162
6163         my $cookie = { netcount => 0 };
6164         while (defined(my $line = <$fh>)) {
6165             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6166         }
6167
6168         $fh->close();
6169         $outfd->close();
6170     };
6171     my $err = $@;
6172
6173     $restore_deactivate_volumes->($storecfg, $devinfo);
6174
6175     rmtree $tmpdir;
6176
6177     if ($err) {
6178         unlink $tmpfn;
6179         $restore_destroy_volumes->($storecfg, $devinfo);
6180         die $err;
6181     }
6182
6183     rename($tmpfn, $conffile) ||
6184         die "unable to commit configuration file '$conffile'\n";
6185
6186     PVE::Cluster::cfs_update(); # make sure we read new file
6187
6188     eval { rescan($vmid, 1); };
6189     warn $@ if $@;
6190 }
6191
6192 sub restore_vma_archive {
6193     my ($archive, $vmid, $user, $opts, $comp) = @_;
6194
6195     my $readfrom = $archive;
6196
6197     my $cfg = PVE::Storage::config();
6198     my $commands = [];
6199     my $bwlimit = $opts->{bwlimit};
6200
6201     my $dbg_cmdstring = '';
6202     my $add_pipe = sub {
6203         my ($cmd) = @_;
6204         push @$commands, $cmd;
6205         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6206         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6207         $readfrom = '-';
6208     };
6209
6210     my $input = undef;
6211     if ($archive eq '-') {
6212         $input = '<&STDIN';
6213     } else {
6214         # If we use a backup from a PVE defined storage we also consider that
6215         # storage's rate limit:
6216         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6217         if (defined($volid)) {
6218             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6219             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6220             if ($readlimit) {
6221                 print STDERR "applying read rate limit: $readlimit\n";
6222                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6223                 $add_pipe->($cstream);
6224             }
6225         }
6226     }
6227
6228     if ($comp) {
6229         my $cmd;
6230         if ($comp eq 'gzip') {
6231             $cmd = ['zcat', $readfrom];
6232         } elsif ($comp eq 'lzop') {
6233             $cmd = ['lzop', '-d', '-c', $readfrom];
6234         } else {
6235             die "unknown compression method '$comp'\n";
6236         }
6237         $add_pipe->($cmd);
6238     }
6239
6240     my $tmpdir = "/var/tmp/vzdumptmp$$";
6241     rmtree $tmpdir;
6242
6243     # disable interrupts (always do cleanups)
6244     local $SIG{INT} =
6245         local $SIG{TERM} =
6246         local $SIG{QUIT} =
6247         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6248
6249     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6250     POSIX::mkfifo($mapfifo, 0600);
6251     my $fifofh;
6252
6253     my $openfifo = sub {
6254         open($fifofh, '>', $mapfifo) || die $!;
6255     };
6256
6257     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6258
6259     my $oldtimeout;
6260     my $timeout = 5;
6261
6262     my $devinfo = {};
6263
6264     my $rpcenv = PVE::RPCEnvironment::get();
6265
6266     my $conffile = PVE::QemuConfig->config_file($vmid);
6267     my $tmpfn = "$conffile.$$.tmp";
6268
6269     # Note: $oldconf is undef if VM does not exist
6270     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6271     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6272
6273     my %storage_limits;
6274
6275     my $print_devmap = sub {
6276         my $cfgfn = "$tmpdir/qemu-server.conf";
6277
6278         # we can read the config - that is already extracted
6279         my $fh = IO::File->new($cfgfn, "r") ||
6280             "unable to read qemu-server.conf - $!\n";
6281
6282         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6283         if (-f $fwcfgfn) {
6284             my $pve_firewall_dir = '/etc/pve/firewall';
6285             mkdir $pve_firewall_dir; # make sure the dir exists
6286             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6287         }
6288
6289         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6290
6291         foreach my $key (keys %storage_limits) {
6292             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6293             next if !$limit;
6294             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6295             $storage_limits{$key} = $limit * 1024;
6296         }
6297
6298         foreach my $devname (keys %$devinfo) {
6299             die "found no device mapping information for device '$devname'\n"
6300                 if !$devinfo->{$devname}->{virtdev};
6301         }
6302
6303         # create empty/temp config
6304         if ($oldconf) {
6305             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6306             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6307         }
6308
6309         # allocate volumes
6310         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6311
6312         # print restore information to $fifofh
6313         foreach my $virtdev (sort keys %$virtdev_hash) {
6314             my $d = $virtdev_hash->{$virtdev};
6315             next if $d->{is_cloudinit}; # no need to restore cloudinit
6316
6317             my $storeid = $d->{storeid};
6318             my $volid = $d->{volid};
6319
6320             my $map_opts = '';
6321             if (my $limit = $storage_limits{$storeid}) {
6322                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6323             }
6324
6325             my $write_zeros = 1;
6326             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6327                 $write_zeros = 0;
6328             }
6329
6330             my $path = PVE::Storage::path($cfg, $volid);
6331
6332             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6333
6334             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6335         }
6336
6337         $fh->seek(0, 0) || die "seek failed - $!\n";
6338
6339         my $outfd = new IO::File ($tmpfn, "w") ||
6340             die "unable to write config for VM $vmid\n";
6341
6342         my $cookie = { netcount => 0 };
6343         while (defined(my $line = <$fh>)) {
6344             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6345         }
6346
6347         $fh->close();
6348         $outfd->close();
6349     };
6350
6351     eval {
6352         # enable interrupts
6353         local $SIG{INT} =
6354             local $SIG{TERM} =
6355             local $SIG{QUIT} =
6356             local $SIG{HUP} =
6357             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6358         local $SIG{ALRM} = sub { die "got timeout\n"; };
6359
6360         $oldtimeout = alarm($timeout);
6361
6362         my $parser = sub {
6363             my $line = shift;
6364
6365             print "$line\n";
6366
6367             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6368                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6369                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6370             } elsif ($line =~ m/^CTIME: /) {
6371                 # we correctly received the vma config, so we can disable
6372                 # the timeout now for disk allocation (set to 10 minutes, so
6373                 # that we always timeout if something goes wrong)
6374                 alarm(600);
6375                 &$print_devmap();
6376                 print $fifofh "done\n";
6377                 my $tmp = $oldtimeout || 0;
6378                 $oldtimeout = undef;
6379                 alarm($tmp);
6380                 close($fifofh);
6381             }
6382         };
6383
6384         print "restore vma archive: $dbg_cmdstring\n";
6385         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6386     };
6387     my $err = $@;
6388
6389     alarm($oldtimeout) if $oldtimeout;
6390
6391     $restore_deactivate_volumes->($cfg, $devinfo);
6392
6393     unlink $mapfifo;
6394     rmtree $tmpdir;
6395
6396     if ($err) {
6397         unlink $tmpfn;
6398         $restore_destroy_volumes->($cfg, $devinfo);
6399         die $err;
6400     }
6401
6402     rename($tmpfn, $conffile) ||
6403         die "unable to commit configuration file '$conffile'\n";
6404
6405     PVE::Cluster::cfs_update(); # make sure we read new file
6406
6407     eval { rescan($vmid, 1); };
6408     warn $@ if $@;
6409 }
6410
6411 sub restore_tar_archive {
6412     my ($archive, $vmid, $user, $opts) = @_;
6413
6414     if ($archive ne '-') {
6415         my $firstfile = tar_archive_read_firstfile($archive);
6416         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6417             if $firstfile ne 'qemu-server.conf';
6418     }
6419
6420     my $storecfg = PVE::Storage::config();
6421
6422     # avoid zombie disks when restoring over an existing VM -> cleanup first
6423     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6424     # skiplock=1 because qmrestore has set the 'create' lock itself already
6425     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6426     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6427
6428     my $tocmd = "/usr/lib/qemu-server/qmextract";
6429
6430     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6431     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6432     $tocmd .= ' --prealloc' if $opts->{prealloc};
6433     $tocmd .= ' --info' if $opts->{info};
6434
6435     # tar option "xf" does not autodetect compression when read from STDIN,
6436     # so we pipe to zcat
6437     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6438         PVE::Tools::shellquote("--to-command=$tocmd");
6439
6440     my $tmpdir = "/var/tmp/vzdumptmp$$";
6441     mkpath $tmpdir;
6442
6443     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6444     local $ENV{VZDUMP_VMID} = $vmid;
6445     local $ENV{VZDUMP_USER} = $user;
6446
6447     my $conffile = PVE::QemuConfig->config_file($vmid);
6448     my $tmpfn = "$conffile.$$.tmp";
6449
6450     # disable interrupts (always do cleanups)
6451     local $SIG{INT} =
6452         local $SIG{TERM} =
6453         local $SIG{QUIT} =
6454         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6455
6456     eval {
6457         # enable interrupts
6458         local $SIG{INT} =
6459             local $SIG{TERM} =
6460             local $SIG{QUIT} =
6461             local $SIG{HUP} =
6462             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6463
6464         if ($archive eq '-') {
6465             print "extracting archive from STDIN\n";
6466             run_command($cmd, input => "<&STDIN");
6467         } else {
6468             print "extracting archive '$archive'\n";
6469             run_command($cmd);
6470         }
6471
6472         return if $opts->{info};
6473
6474         # read new mapping
6475         my $map = {};
6476         my $statfile = "$tmpdir/qmrestore.stat";
6477         if (my $fd = IO::File->new($statfile, "r")) {
6478             while (defined (my $line = <$fd>)) {
6479                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6480                     $map->{$1} = $2 if $1;
6481                 } else {
6482                     print STDERR "unable to parse line in statfile - $line\n";
6483                 }
6484             }
6485             $fd->close();
6486         }
6487
6488         my $confsrc = "$tmpdir/qemu-server.conf";
6489
6490         my $srcfd = new IO::File($confsrc, "r") ||
6491             die "unable to open file '$confsrc'\n";
6492
6493         my $outfd = new IO::File ($tmpfn, "w") ||
6494             die "unable to write config for VM $vmid\n";
6495
6496         my $cookie = { netcount => 0 };
6497         while (defined (my $line = <$srcfd>)) {
6498             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6499         }
6500
6501         $srcfd->close();
6502         $outfd->close();
6503     };
6504     if (my $err = $@) {
6505         unlink $tmpfn;
6506         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6507         die $err;
6508     }
6509
6510     rmtree $tmpdir;
6511
6512     rename $tmpfn, $conffile ||
6513         die "unable to commit configuration file '$conffile'\n";
6514
6515     PVE::Cluster::cfs_update(); # make sure we read new file
6516
6517     eval { rescan($vmid, 1); };
6518     warn $@ if $@;
6519 };
6520
6521 sub foreach_storage_used_by_vm {
6522     my ($conf, $func) = @_;
6523
6524     my $sidhash = {};
6525
6526     PVE::QemuConfig->foreach_volume($conf, sub {
6527         my ($ds, $drive) = @_;
6528         return if drive_is_cdrom($drive);
6529
6530         my $volid = $drive->{file};
6531
6532         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6533         $sidhash->{$sid} = $sid if $sid;
6534     });
6535
6536     foreach my $sid (sort keys %$sidhash) {
6537         &$func($sid);
6538     }
6539 }
6540
6541 my $qemu_snap_storage = {
6542     rbd => 1,
6543 };
6544 sub do_snapshots_with_qemu {
6545     my ($storecfg, $volid) = @_;
6546
6547     my $storage_name = PVE::Storage::parse_volume_id($volid);
6548     my $scfg = $storecfg->{ids}->{$storage_name};
6549
6550     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6551         return 1;
6552     }
6553
6554     if ($volid =~ m/\.(qcow2|qed)$/){
6555         return 1;
6556     }
6557
6558     return undef;
6559 }
6560
6561 sub qga_check_running {
6562     my ($vmid, $nowarn) = @_;
6563
6564     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6565     if ($@) {
6566         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6567         return 0;
6568     }
6569     return 1;
6570 }
6571
6572 sub template_create {
6573     my ($vmid, $conf, $disk) = @_;
6574
6575     my $storecfg = PVE::Storage::config();
6576
6577     PVE::QemuConfig->foreach_volume($conf, sub {
6578         my ($ds, $drive) = @_;
6579
6580         return if drive_is_cdrom($drive);
6581         return if $disk && $ds ne $disk;
6582
6583         my $volid = $drive->{file};
6584         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6585
6586         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6587         $drive->{file} = $voliddst;
6588         $conf->{$ds} = print_drive($drive);
6589         PVE::QemuConfig->write_config($vmid, $conf);
6590     });
6591 }
6592
6593 sub convert_iscsi_path {
6594     my ($path) = @_;
6595
6596     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6597         my $portal = $1;
6598         my $target = $2;
6599         my $lun = $3;
6600
6601         my $initiator_name = get_initiator_name();
6602
6603         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6604                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6605     }
6606
6607     die "cannot convert iscsi path '$path', unkown format\n";
6608 }
6609
6610 sub qemu_img_convert {
6611     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6612
6613     my $storecfg = PVE::Storage::config();
6614     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6615     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6616
6617     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6618
6619     my $cachemode;
6620     my $src_path;
6621     my $src_is_iscsi = 0;
6622     my $src_format;
6623
6624     if ($src_storeid) {
6625         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6626         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6627         $src_format = qemu_img_format($src_scfg, $src_volname);
6628         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6629         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6630         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6631     } elsif (-f $src_volid) {
6632         $src_path = $src_volid;
6633         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6634             $src_format = $1;
6635         }
6636     }
6637
6638     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6639
6640     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6641     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6642     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6643     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6644
6645     my $cmd = [];
6646     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6647     push @$cmd, '-l', "snapshot.name=$snapname"
6648         if $snapname && $src_format && $src_format eq "qcow2";
6649     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6650     push @$cmd, '-T', $cachemode if defined($cachemode);
6651
6652     if ($src_is_iscsi) {
6653         push @$cmd, '--image-opts';
6654         $src_path = convert_iscsi_path($src_path);
6655     } elsif ($src_format) {
6656         push @$cmd, '-f', $src_format;
6657     }
6658
6659     if ($dst_is_iscsi) {
6660         push @$cmd, '--target-image-opts';
6661         $dst_path = convert_iscsi_path($dst_path);
6662     } else {
6663         push @$cmd, '-O', $dst_format;
6664     }
6665
6666     push @$cmd, $src_path;
6667
6668     if (!$dst_is_iscsi && $is_zero_initialized) {
6669         push @$cmd, "zeroinit:$dst_path";
6670     } else {
6671         push @$cmd, $dst_path;
6672     }
6673
6674     my $parser = sub {
6675         my $line = shift;
6676         if($line =~ m/\((\S+)\/100\%\)/){
6677             my $percent = $1;
6678             my $transferred = int($size * $percent / 100);
6679             my $remaining = $size - $transferred;
6680
6681             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6682         }
6683
6684     };
6685
6686     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6687     my $err = $@;
6688     die "copy failed: $err" if $err;
6689 }
6690
6691 sub qemu_img_format {
6692     my ($scfg, $volname) = @_;
6693
6694     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6695         return $1;
6696     } else {
6697         return "raw";
6698     }
6699 }
6700
6701 sub qemu_drive_mirror {
6702     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6703
6704     $jobs = {} if !$jobs;
6705
6706     my $qemu_target;
6707     my $format;
6708     $jobs->{"drive-$drive"} = {};
6709
6710     if ($dst_volid =~ /^nbd:/) {
6711         $qemu_target = $dst_volid;
6712         $format = "nbd";
6713     } else {
6714         my $storecfg = PVE::Storage::config();
6715         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6716
6717         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6718
6719         $format = qemu_img_format($dst_scfg, $dst_volname);
6720
6721         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6722
6723         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6724     }
6725
6726     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6727     $opts->{format} = $format if $format;
6728
6729     if (defined($src_bitmap)) {
6730         $opts->{sync} = 'incremental';
6731         $opts->{bitmap} = $src_bitmap;
6732         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6733     }
6734
6735     if (defined($bwlimit)) {
6736         $opts->{speed} = $bwlimit * 1024;
6737         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6738     } else {
6739         print "drive mirror is starting for drive-$drive\n";
6740     }
6741
6742     # if a job already runs for this device we get an error, catch it for cleanup
6743     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6744     if (my $err = $@) {
6745         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6746         warn "$@\n" if $@;
6747         die "mirroring error: $err\n";
6748     }
6749
6750     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6751 }
6752
6753 # $completion can be either
6754 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6755 # 'cancel': wait until all jobs are ready, block-job-cancel them
6756 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6757 sub qemu_drive_mirror_monitor {
6758     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6759
6760     $completion //= 'complete';
6761
6762     eval {
6763         my $err_complete = 0;
6764
6765         while (1) {
6766             die "storage migration timed out\n" if $err_complete > 300;
6767
6768             my $stats = mon_cmd($vmid, "query-block-jobs");
6769
6770             my $running_mirror_jobs = {};
6771             foreach my $stat (@$stats) {
6772                 next if $stat->{type} ne 'mirror';
6773                 $running_mirror_jobs->{$stat->{device}} = $stat;
6774             }
6775
6776             my $readycounter = 0;
6777
6778             foreach my $job (keys %$jobs) {
6779
6780                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6781                     print "$job : finished\n";
6782                     delete $jobs->{$job};
6783                     next;
6784                 }
6785
6786                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6787
6788                 my $busy = $running_mirror_jobs->{$job}->{busy};
6789                 my $ready = $running_mirror_jobs->{$job}->{ready};
6790                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6791                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6792                     my $remaining = $total - $transferred;
6793                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6794
6795                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6796                 }
6797
6798                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6799             }
6800
6801             last if scalar(keys %$jobs) == 0;
6802
6803             if ($readycounter == scalar(keys %$jobs)) {
6804                 print "all mirroring jobs are ready \n";
6805                 last if $completion eq 'skip'; #do the complete later
6806
6807                 if ($vmiddst && $vmiddst != $vmid) {
6808                     my $agent_running = $qga && qga_check_running($vmid);
6809                     if ($agent_running) {
6810                         print "freeze filesystem\n";
6811                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6812                     } else {
6813                         print "suspend vm\n";
6814                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6815                     }
6816
6817                     # if we clone a disk for a new target vm, we don't switch the disk
6818                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6819
6820                     if ($agent_running) {
6821                         print "unfreeze filesystem\n";
6822                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6823                     } else {
6824                         print "resume vm\n";
6825                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6826                     }
6827
6828                     last;
6829                 } else {
6830
6831                     foreach my $job (keys %$jobs) {
6832                         # try to switch the disk if source and destination are on the same guest
6833                         print "$job: Completing block job...\n";
6834
6835                         my $op;
6836                         if ($completion eq 'complete') {
6837                             $op = 'block-job-complete';
6838                         } elsif ($completion eq 'cancel') {
6839                             $op = 'block-job-cancel';
6840                         } else {
6841                             die "invalid completion value: $completion\n";
6842                         }
6843                         eval { mon_cmd($vmid, $op, device => $job) };
6844                         if ($@ =~ m/cannot be completed/) {
6845                             print "$job: Block job cannot be completed, try again.\n";
6846                             $err_complete++;
6847                         }else {
6848                             print "$job: Completed successfully.\n";
6849                             $jobs->{$job}->{complete} = 1;
6850                         }
6851                     }
6852                 }
6853             }
6854             sleep 1;
6855         }
6856     };
6857     my $err = $@;
6858
6859     if ($err) {
6860         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6861         die "mirroring error: $err";
6862     }
6863
6864 }
6865
6866 sub qemu_blockjobs_cancel {
6867     my ($vmid, $jobs) = @_;
6868
6869     foreach my $job (keys %$jobs) {
6870         print "$job: Cancelling block job\n";
6871         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6872         $jobs->{$job}->{cancel} = 1;
6873     }
6874
6875     while (1) {
6876         my $stats = mon_cmd($vmid, "query-block-jobs");
6877
6878         my $running_jobs = {};
6879         foreach my $stat (@$stats) {
6880             $running_jobs->{$stat->{device}} = $stat;
6881         }
6882
6883         foreach my $job (keys %$jobs) {
6884
6885             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6886                 print "$job: Done.\n";
6887                 delete $jobs->{$job};
6888             }
6889         }
6890
6891         last if scalar(keys %$jobs) == 0;
6892
6893         sleep 1;
6894     }
6895 }
6896
6897 sub clone_disk {
6898     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6899         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6900
6901     my $newvolid;
6902
6903     if (!$full) {
6904         print "create linked clone of drive $drivename ($drive->{file})\n";
6905         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6906         push @$newvollist, $newvolid;
6907     } else {
6908
6909         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6910         $storeid = $storage if $storage;
6911
6912         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6913         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6914
6915         print "create full clone of drive $drivename ($drive->{file})\n";
6916         my $name = undef;
6917         if (drive_is_cloudinit($drive)) {
6918             $name = "vm-$newvmid-cloudinit";
6919             $name .= ".$dst_format" if $dst_format ne 'raw';
6920             $snapname = undef;
6921             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6922         } elsif ($drivename eq 'efidisk0') {
6923             $size = get_efivars_size($conf);
6924         }
6925         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6926         push @$newvollist, $newvolid;
6927
6928         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6929
6930         if (drive_is_cloudinit($drive)) {
6931             goto no_data_clone;
6932         }
6933
6934         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6935         if (!$running || $snapname) {
6936             # TODO: handle bwlimits
6937             if ($drivename eq 'efidisk0') {
6938                 # the relevant data on the efidisk may be smaller than the source
6939                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6940                 # that is given by the OVMF_VARS.fd
6941                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6942                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6943                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6944             } else {
6945                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6946             }
6947         } else {
6948
6949             my $kvmver = get_running_qemu_version ($vmid);
6950             if (!min_version($kvmver, 2, 7)) {
6951                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6952                     if $drive->{iothread};
6953             }
6954
6955             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6956         }
6957     }
6958
6959 no_data_clone:
6960     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6961
6962     my $disk = $drive;
6963     $disk->{format} = undef;
6964     $disk->{file} = $newvolid;
6965     $disk->{size} = $size;
6966
6967     return $disk;
6968 }
6969
6970 sub get_running_qemu_version {
6971     my ($vmid) = @_;
6972     my $res = mon_cmd($vmid, "query-version");
6973     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6974 }
6975
6976 sub qemu_use_old_bios_files {
6977     my ($machine_type) = @_;
6978
6979     return if !$machine_type;
6980
6981     my $use_old_bios_files = undef;
6982
6983     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6984         $machine_type = $1;
6985         $use_old_bios_files = 1;
6986     } else {
6987         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6988         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6989         # load new efi bios files on migration. So this hack is required to allow
6990         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6991         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6992         $use_old_bios_files = !min_version($version, 2, 4);
6993     }
6994
6995     return ($use_old_bios_files, $machine_type);
6996 }
6997
6998 sub get_efivars_size {
6999     my ($conf) = @_;
7000     my $arch = get_vm_arch($conf);
7001     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7002     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7003     return -s $ovmf_vars;
7004 }
7005
7006 sub update_efidisk_size {
7007     my ($conf) = @_;
7008
7009     return if !defined($conf->{efidisk0});
7010
7011     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7012     $disk->{size} = get_efivars_size($conf);
7013     $conf->{efidisk0} = print_drive($disk);
7014
7015     return;
7016 }
7017
7018 sub create_efidisk($$$$$) {
7019     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7020
7021     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7022     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7023
7024     my $vars_size_b = -s $ovmf_vars;
7025     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7026     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7027     PVE::Storage::activate_volumes($storecfg, [$volid]);
7028
7029     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7030     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7031
7032     return ($volid, $size/1024);
7033 }
7034
7035 sub vm_iothreads_list {
7036     my ($vmid) = @_;
7037
7038     my $res = mon_cmd($vmid, 'query-iothreads');
7039
7040     my $iothreads = {};
7041     foreach my $iothread (@$res) {
7042         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7043     }
7044
7045     return $iothreads;
7046 }
7047
7048 sub scsihw_infos {
7049     my ($conf, $drive) = @_;
7050
7051     my $maxdev = 0;
7052
7053     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7054         $maxdev = 7;
7055     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7056         $maxdev = 1;
7057     } else {
7058         $maxdev = 256;
7059     }
7060
7061     my $controller = int($drive->{index} / $maxdev);
7062     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7063
7064     return ($maxdev, $controller, $controller_prefix);
7065 }
7066
7067 sub windows_version {
7068     my ($ostype) = @_;
7069
7070     return 0 if !$ostype;
7071
7072     my $winversion = 0;
7073
7074     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7075         $winversion = 5;
7076     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7077         $winversion = 6;
7078     } elsif ($ostype =~ m/^win(\d+)$/) {
7079         $winversion = $1;
7080     }
7081
7082     return $winversion;
7083 }
7084
7085 sub resolve_dst_disk_format {
7086         my ($storecfg, $storeid, $src_volname, $format) = @_;
7087         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7088
7089         if (!$format) {
7090             # if no target format is specified, use the source disk format as hint
7091             if ($src_volname) {
7092                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7093                 $format = qemu_img_format($scfg, $src_volname);
7094             } else {
7095                 return $defFormat;
7096             }
7097         }
7098
7099         # test if requested format is supported - else use default
7100         my $supported = grep { $_ eq $format } @$validFormats;
7101         $format = $defFormat if !$supported;
7102         return $format;
7103 }
7104
7105 # NOTE: if this logic changes, please update docs & possibly gui logic
7106 sub find_vmstate_storage {
7107     my ($conf, $storecfg) = @_;
7108
7109     # first, return storage from conf if set
7110     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7111
7112     my ($target, $shared, $local);
7113
7114     foreach_storage_used_by_vm($conf, sub {
7115         my ($sid) = @_;
7116         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7117         my $dst = $scfg->{shared} ? \$shared : \$local;
7118         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7119     });
7120
7121     # second, use shared storage where VM has at least one disk
7122     # third, use local storage where VM has at least one disk
7123     # fall back to local storage
7124     $target = $shared // $local // 'local';
7125
7126     return $target;
7127 }
7128
7129 sub generate_uuid {
7130     my ($uuid, $uuid_str);
7131     UUID::generate($uuid);
7132     UUID::unparse($uuid, $uuid_str);
7133     return $uuid_str;
7134 }
7135
7136 sub generate_smbios1_uuid {
7137     return "uuid=".generate_uuid();
7138 }
7139
7140 sub nbd_stop {
7141     my ($vmid) = @_;
7142
7143     mon_cmd($vmid, 'nbd-server-stop');
7144 }
7145
7146 sub create_reboot_request {
7147     my ($vmid) = @_;
7148     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7149         or die "failed to create reboot trigger file: $!\n";
7150     close($fh);
7151 }
7152
7153 sub clear_reboot_request {
7154     my ($vmid) = @_;
7155     my $path = "/run/qemu-server/$vmid.reboot";
7156     my $res = 0;
7157
7158     $res = unlink($path);
7159     die "could not remove reboot request for $vmid: $!"
7160         if !$res && $! != POSIX::ENOENT;
7161
7162     return $res;
7163 }
7164
7165 # bash completion helper
7166
7167 sub complete_backup_archives {
7168     my ($cmdname, $pname, $cvalue) = @_;
7169
7170     my $cfg = PVE::Storage::config();
7171
7172     my $storeid;
7173
7174     if ($cvalue =~ m/^([^:]+):/) {
7175         $storeid = $1;
7176     }
7177
7178     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7179
7180     my $res = [];
7181     foreach my $id (keys %$data) {
7182         foreach my $item (@{$data->{$id}}) {
7183             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7184             push @$res, $item->{volid} if defined($item->{volid});
7185         }
7186     }
7187
7188     return $res;
7189 }
7190
7191 my $complete_vmid_full = sub {
7192     my ($running) = @_;
7193
7194     my $idlist = vmstatus();
7195
7196     my $res = [];
7197
7198     foreach my $id (keys %$idlist) {
7199         my $d = $idlist->{$id};
7200         if (defined($running)) {
7201             next if $d->{template};
7202             next if $running && $d->{status} ne 'running';
7203             next if !$running && $d->{status} eq 'running';
7204         }
7205         push @$res, $id;
7206
7207     }
7208     return $res;
7209 };
7210
7211 sub complete_vmid {
7212     return &$complete_vmid_full();
7213 }
7214
7215 sub complete_vmid_stopped {
7216     return &$complete_vmid_full(0);
7217 }
7218
7219 sub complete_vmid_running {
7220     return &$complete_vmid_full(1);
7221 }
7222
7223 sub complete_storage {
7224
7225     my $cfg = PVE::Storage::config();
7226     my $ids = $cfg->{ids};
7227
7228     my $res = [];
7229     foreach my $sid (keys %$ids) {
7230         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7231         next if !$ids->{$sid}->{content}->{images};
7232         push @$res, $sid;
7233     }
7234
7235     return $res;
7236 }
7237
7238 sub complete_migration_storage {
7239     my ($cmd, $param, $current_value, $all_args) = @_;
7240
7241     my $targetnode = @$all_args[1];
7242
7243     my $cfg = PVE::Storage::config();
7244     my $ids = $cfg->{ids};
7245
7246     my $res = [];
7247     foreach my $sid (keys %$ids) {
7248         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7249         next if !$ids->{$sid}->{content}->{images};
7250         push @$res, $sid;
7251     }
7252
7253     return $res;
7254 }
7255
7256 1;