PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_SERIAL_PORTS = 4;
 772 my $MAX_PARALLEL_PORTS = 3;
 773 my $MAX_NUMA = 8;
 774
 775 my $numa_fmt = {
 776     cpus => {
 777         type => "string",
 778         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 779         description => "CPUs accessing this NUMA node.",
 780         format_description => "id[-id];...",
 781     },
 782     memory => {
 783         type => "number",
 784         description => "Amount of memory this NUMA node provides.",
 785         optional => 1,
 786     },
 787     hostnodes => {
 788         type => "string",
 789         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 790         description => "Host NUMA nodes to use.",
 791         format_description => "id[-id];...",
 792         optional => 1,
 793     },
 794     policy => {
 795         type => 'string',
 796         enum => [qw(preferred bind interleave)],
 797         description => "NUMA allocation policy.",
 798         optional => 1,
 799     },
 800 };
 801 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 802 my $numadesc = {
 803     optional => 1,
 804     type => 'string', format => $numa_fmt,
 805     description => "NUMA topology.",
 806 };
 807 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 808
 809 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 810     $confdesc->{"numa$i"} = $numadesc;
 811 }
 812
 813 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 814                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 815                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 816 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 817
 818 my $net_fmt_bridge_descr = <<__EOD__;
 819 Bridge to attach the network device to. The Proxmox VE standard bridge
 820 is called 'vmbr0'.
 821
 822 If you do not specify a bridge, we create a kvm user (NATed) network
 823 device, which provides DHCP and DNS services. The following addresses
 824 are used:
 825
 826  10.0.2.2   Gateway
 827  10.0.2.3   DNS Server
 828  10.0.2.4   SMB Server
 829
 830 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 831 __EOD__
 832
 833 my $net_fmt = {
 834     macaddr  => get_standard_option('mac-addr', {
 835         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 836     }),
 837     model => {
 838         type => 'string',
 839         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 840         enum => $nic_model_list,
 841         default_key => 1,
 842     },
 843     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 844     bridge => {
 845         type => 'string',
 846         description => $net_fmt_bridge_descr,
 847         format_description => 'bridge',
 848         pattern => '[-_.\w\d]+',
 849         optional => 1,
 850     },
 851     queues => {
 852         type => 'integer',
 853         minimum => 0, maximum => 16,
 854         description => 'Number of packet queues to be used on the device.',
 855         optional => 1,
 856     },
 857     rate => {
 858         type => 'number',
 859         minimum => 0,
 860         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 861         optional => 1,
 862     },
 863     tag => {
 864         type => 'integer',
 865         minimum => 1, maximum => 4094,
 866         description => 'VLAN tag to apply to packets on this interface.',
 867         optional => 1,
 868     },
 869     trunks => {
 870         type => 'string',
 871         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 872         description => 'VLAN trunks to pass through this interface.',
 873         format_description => 'vlanid[;vlanid...]',
 874         optional => 1,
 875     },
 876     firewall => {
 877         type => 'boolean',
 878         description => 'Whether this interface should be protected by the firewall.',
 879         optional => 1,
 880     },
 881     link_down => {
 882         type => 'boolean',
 883         description => 'Whether this interface should be disconnected (like pulling the plug).',
 884         optional => 1,
 885     },
 886     mtu => {
 887         type => 'integer',
 888         minimum => 1, maximum => 65520,
 889         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 890         optional => 1,
 891     },
 892 };
 893
 894 my $netdesc = {
 895     optional => 1,
 896     type => 'string', format => $net_fmt,
 897     description => "Specify network devices.",
 898 };
 899
 900 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 901
 902 my $ipconfig_fmt = {
 903     ip => {
 904         type => 'string',
 905         format => 'pve-ipv4-config',
 906         format_description => 'IPv4Format/CIDR',
 907         description => 'IPv4 address in CIDR format.',
 908         optional => 1,
 909         default => 'dhcp',
 910     },
 911     gw => {
 912         type => 'string',
 913         format => 'ipv4',
 914         format_description => 'GatewayIPv4',
 915         description => 'Default gateway for IPv4 traffic.',
 916         optional => 1,
 917         requires => 'ip',
 918     },
 919     ip6 => {
 920         type => 'string',
 921         format => 'pve-ipv6-config',
 922         format_description => 'IPv6Format/CIDR',
 923         description => 'IPv6 address in CIDR format.',
 924         optional => 1,
 925         default => 'dhcp',
 926     },
 927     gw6 => {
 928         type => 'string',
 929         format => 'ipv6',
 930         format_description => 'GatewayIPv6',
 931         description => 'Default gateway for IPv6 traffic.',
 932         optional => 1,
 933         requires => 'ip6',
 934     },
 935 };
 936 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 937 my $ipconfigdesc = {
 938     optional => 1,
 939     type => 'string', format => 'pve-qm-ipconfig',
 940     description => <<'EODESCR',
 941 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 942
 943 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 944
 945 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 946 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 947
 948 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 949 EODESCR
 950 };
 951 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 952
 953 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 954     $confdesc->{"net$i"} = $netdesc;
 955     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 956 }
 957
 958 foreach my $key (keys %$confdesc_cloudinit) {
 959     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 960 }
 961
 962 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 963 sub verify_volume_id_or_qm_path {
 964     my ($volid, $noerr) = @_;
 965
 966     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 967         return $volid;
 968     }
 969
 970     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 971     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 972     if ($@) {
 973         return undef if $noerr;
 974         die $@;
 975     }
 976     return $volid;
 977 }
 978
 979 my $usb_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string', format => 'pve-qm-usb-device',
 983         format_description => 'HOSTUSBDEVICE|spice',
 984         description => <<EODESCR,
 985 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 986
 987  'bus-port(.port)*' (decimal numbers) or
 988  'vendor_id:product_id' (hexadeciaml numbers) or
 989  'spice'
 990
 991 You can use the 'lsusb -t' command to list existing usb devices.
 992
 993 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 994
 995 The value 'spice' can be used to add a usb redirection devices for spice.
 996 EODESCR
 997     },
 998     usb3 => {
 999         optional => 1,
1000         type => 'boolean',
1001         description => "Specifies whether if given host option is a USB3 device or port.",
1002         default => 0,
1003     },
1004 };
1005
1006 my $usbdesc = {
1007     optional => 1,
1008     type => 'string', format => $usb_fmt,
1009     description => "Configure an USB device (n is 0 to 4).",
1010 };
1011 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1012
1013 my $serialdesc = {
1014         optional => 1,
1015         type => 'string',
1016         pattern => '(/dev/.+|socket)',
1017         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1018         verbose_description =>  <<EODESCR,
1019 Create a serial device inside the VM (n is 0 to 3), and pass through a
1020 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1021 host side (use 'qm terminal' to open a terminal connection).
1022
1023 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1024
1025 CAUTION: Experimental! User reported problems with this option.
1026 EODESCR
1027 };
1028
1029 my $paralleldesc= {
1030         optional => 1,
1031         type => 'string',
1032         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1033         description =>  "Map host parallel devices (n is 0 to 2).",
1034         verbose_description =>  <<EODESCR,
1035 Map host parallel devices (n is 0 to 2).
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1038
1039 CAUTION: Experimental! User reported problems with this option.
1040 EODESCR
1041 };
1042
1043 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1044     $confdesc->{"parallel$i"} = $paralleldesc;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1048     $confdesc->{"serial$i"} = $serialdesc;
1049 }
1050
1051 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1052     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1053 }
1054
1055 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1056     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1057 }
1058
1059 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1060     $confdesc->{"usb$i"} = $usbdesc;
1061 }
1062
1063 my $kvm_api_version = 0;
1064
1065 sub kvm_version {
1066     return $kvm_api_version if $kvm_api_version;
1067
1068     open my $fh, '<', '/dev/kvm'
1069         or return undef;
1070
1071     # 0xae00 => KVM_GET_API_VERSION
1072     $kvm_api_version = ioctl($fh, 0xae00, 0);
1073
1074     return $kvm_api_version;
1075 }
1076
1077 my $kvm_user_version = {};
1078 my $kvm_mtime = {};
1079
1080 sub kvm_user_version {
1081     my ($binary) = @_;
1082
1083     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1084     my $st = stat($binary);
1085
1086     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1087     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1088         $cachedmtime == $st->mtime;
1089
1090     $kvm_user_version->{$binary} = 'unknown';
1091     $kvm_mtime->{$binary} = $st->mtime;
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version->{$binary} = $2;
1097         }
1098     };
1099
1100     eval { run_command([$binary, '--version'], outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version->{$binary};
1104
1105 }
1106
1107 sub kernel_has_vhost_net {
1108     return -c '/dev/vhost-net';
1109 }
1110
1111 sub option_exists {
1112     my $key = shift;
1113     return defined($confdesc->{$key});
1114 }
1115
1116 my $cdrom_path;
1117 sub get_cdrom_path {
1118
1119     return  $cdrom_path if $cdrom_path;
1120
1121     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1122     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1123     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1124 }
1125
1126 sub get_iso_path {
1127     my ($storecfg, $vmid, $cdrom) = @_;
1128
1129     if ($cdrom eq 'cdrom') {
1130         return get_cdrom_path();
1131     } elsif ($cdrom eq 'none') {
1132         return '';
1133     } elsif ($cdrom =~ m|^/|) {
1134         return $cdrom;
1135     } else {
1136         return PVE::Storage::path($storecfg, $cdrom);
1137     }
1138 }
1139
1140 # try to convert old style file names to volume IDs
1141 sub filename_to_volume_id {
1142     my ($vmid, $file, $media) = @_;
1143
1144      if (!($file eq 'none' || $file eq 'cdrom' ||
1145           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1146
1147         return undef if $file =~ m|/|;
1148
1149         if ($media && $media eq 'cdrom') {
1150             $file = "local:iso/$file";
1151         } else {
1152             $file = "local:$vmid/$file";
1153         }
1154     }
1155
1156     return $file;
1157 }
1158
1159 sub verify_media_type {
1160     my ($opt, $vtype, $media) = @_;
1161
1162     return if !$media;
1163
1164     my $etype;
1165     if ($media eq 'disk') {
1166         $etype = 'images';
1167     } elsif ($media eq 'cdrom') {
1168         $etype = 'iso';
1169     } else {
1170         die "internal error";
1171     }
1172
1173     return if ($vtype eq $etype);
1174
1175     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1176 }
1177
1178 sub cleanup_drive_path {
1179     my ($opt, $storecfg, $drive) = @_;
1180
1181     # try to convert filesystem paths to volume IDs
1182
1183     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1184         ($drive->{file} !~ m|^/dev/.+|) &&
1185         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1186         ($drive->{file} !~ m/^\d+$/)) {
1187         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1188         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1189         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1190         verify_media_type($opt, $vtype, $drive->{media});
1191         $drive->{file} = $volid;
1192     }
1193
1194     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1195 }
1196
1197 sub parse_hotplug_features {
1198     my ($data) = @_;
1199
1200     my $res = {};
1201
1202     return $res if $data eq '0';
1203
1204     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1205
1206     foreach my $feature (PVE::Tools::split_list($data)) {
1207         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1208             $res->{$1} = 1;
1209         } else {
1210             die "invalid hotplug feature '$feature'\n";
1211         }
1212     }
1213     return $res;
1214 }
1215
1216 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1217 sub pve_verify_hotplug_features {
1218     my ($value, $noerr) = @_;
1219
1220     return $value if parse_hotplug_features($value);
1221
1222     return undef if $noerr;
1223
1224     die "unable to parse hotplug option\n";
1225 }
1226
1227 sub scsi_inquiry {
1228     my($fh, $noerr) = @_;
1229
1230     my $SG_IO = 0x2285;
1231     my $SG_GET_VERSION_NUM = 0x2282;
1232
1233     my $versionbuf = "\x00" x 8;
1234     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1235     if (!$ret) {
1236         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1237         return undef;
1238     }
1239     my $version = unpack("I", $versionbuf);
1240     if ($version < 30000) {
1241         die "scsi generic interface too old\n"  if !$noerr;
1242         return undef;
1243     }
1244
1245     my $buf = "\x00" x 36;
1246     my $sensebuf = "\x00" x 8;
1247     my $cmd = pack("C x3 C x1", 0x12, 36);
1248
1249     # see /usr/include/scsi/sg.h
1250     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1251
1252     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1253                       length($sensebuf), 0, length($buf), $buf,
1254                       $cmd, $sensebuf, 6000);
1255
1256     $ret = ioctl($fh, $SG_IO, $packet);
1257     if (!$ret) {
1258         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1259         return undef;
1260     }
1261
1262     my @res = unpack($sg_io_hdr_t, $packet);
1263     if ($res[17] || $res[18]) {
1264         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1265         return undef;
1266     }
1267
1268     my $res = {};
1269     (my $byte0, my $byte1, $res->{vendor},
1270      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1271
1272     $res->{removable} = $byte1 & 128 ? 1 : 0;
1273     $res->{type} = $byte0 & 31;
1274
1275     return $res;
1276 }
1277
1278 sub path_is_scsi {
1279     my ($path) = @_;
1280
1281     my $fh = IO::File->new("+<$path") || return undef;
1282     my $res = scsi_inquiry($fh, 1);
1283     close($fh);
1284
1285     return $res;
1286 }
1287
1288 sub print_tabletdevice_full {
1289     my ($conf, $arch) = @_;
1290
1291     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1292
1293     # we use uhci for old VMs because tablet driver was buggy in older qemu
1294     my $usbbus;
1295     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1296         $usbbus = 'ehci';
1297     } else {
1298         $usbbus = 'uhci';
1299     }
1300
1301     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1302 }
1303
1304 sub print_keyboarddevice_full {
1305     my ($conf, $arch, $machine) = @_;
1306
1307     return undef if $arch ne 'aarch64';
1308
1309     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1310 }
1311
1312 sub print_drivedevice_full {
1313     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1314
1315     my $device = '';
1316     my $maxdev = 0;
1317
1318     if ($drive->{interface} eq 'virtio') {
1319         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1320         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1321         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1322     } elsif ($drive->{interface} eq 'scsi') {
1323
1324         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1325         my $unit = $drive->{index} % $maxdev;
1326         my $devicetype = 'hd';
1327         my $path = '';
1328         if (drive_is_cdrom($drive)) {
1329             $devicetype = 'cd';
1330         } else {
1331             if ($drive->{file} =~ m|^/|) {
1332                 $path = $drive->{file};
1333                 if (my $info = path_is_scsi($path)) {
1334                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1335                         $devicetype = 'block';
1336                     } elsif ($info->{type} == 1) { # tape
1337                         $devicetype = 'generic';
1338                     }
1339                 }
1340             } else {
1341                  $path = PVE::Storage::path($storecfg, $drive->{file});
1342             }
1343
1344             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1345             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1346             if ($path =~ m/^iscsi\:\/\// &&
1347                !min_version($version, 4, 1)) {
1348                 $devicetype = 'generic';
1349             }
1350         }
1351
1352         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1353             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1354         } else {
1355             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1356         }
1357
1358         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1359             $device .= ",rotation_rate=1";
1360         }
1361         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1362
1363     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1364         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1365         my $controller = int($drive->{index} / $maxdev);
1366         my $unit = $drive->{index} % $maxdev;
1367         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1368
1369         $device = "ide-$devicetype";
1370         if ($drive->{interface} eq 'ide') {
1371             $device .= ",bus=ide.$controller,unit=$unit";
1372         } else {
1373             $device .= ",bus=ahci$controller.$unit";
1374         }
1375         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1376
1377         if ($devicetype eq 'hd') {
1378             if (my $model = $drive->{model}) {
1379                 $model = URI::Escape::uri_unescape($model);
1380                 $device .= ",model=$model";
1381             }
1382             if ($drive->{ssd}) {
1383                 $device .= ",rotation_rate=1";
1384             }
1385         }
1386         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1387     } elsif ($drive->{interface} eq 'usb') {
1388         die "implement me";
1389         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1390     } else {
1391         die "unsupported interface type";
1392     }
1393
1394     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1395
1396     if (my $serial = $drive->{serial}) {
1397         $serial = URI::Escape::uri_unescape($serial);
1398         $device .= ",serial=$serial";
1399     }
1400
1401
1402     return $device;
1403 }
1404
1405 sub get_initiator_name {
1406     my $initiator;
1407
1408     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1409     while (defined(my $line = <$fh>)) {
1410         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1411         $initiator = $1;
1412         last;
1413     }
1414     $fh->close();
1415
1416     return $initiator;
1417 }
1418
1419 sub print_drive_commandline_full {
1420     my ($storecfg, $vmid, $drive) = @_;
1421
1422     my $path;
1423     my $volid = $drive->{file};
1424     my $format;
1425
1426     if (drive_is_cdrom($drive)) {
1427         $path = get_iso_path($storecfg, $vmid, $volid);
1428     } else {
1429         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1430         if ($storeid) {
1431             $path = PVE::Storage::path($storecfg, $volid);
1432             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1433             $format = qemu_img_format($scfg, $volname);
1434         } else {
1435             $path = $volid;
1436             $format = "raw";
1437         }
1438    }
1439
1440     my $opts = '';
1441     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1442     foreach my $o (@qemu_drive_options) {
1443         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1444     }
1445
1446     # snapshot only accepts on|off
1447     if (defined($drive->{snapshot})) {
1448         my $v = $drive->{snapshot} ? 'on' : 'off';
1449         $opts .= ",snapshot=$v";
1450     }
1451
1452     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1453         my ($dir, $qmpname) = @$type;
1454         if (my $v = $drive->{"mbps$dir"}) {
1455             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1456         }
1457         if (my $v = $drive->{"mbps${dir}_max"}) {
1458             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1459         }
1460         if (my $v = $drive->{"bps${dir}_max_length"}) {
1461             $opts .= ",throttling.bps$qmpname-max-length=$v";
1462         }
1463         if (my $v = $drive->{"iops${dir}"}) {
1464             $opts .= ",throttling.iops$qmpname=$v";
1465         }
1466         if (my $v = $drive->{"iops${dir}_max"}) {
1467             $opts .= ",throttling.iops$qmpname-max=$v";
1468         }
1469         if (my $v = $drive->{"iops${dir}_max_length"}) {
1470             $opts .= ",throttling.iops$qmpname-max-length=$v";
1471         }
1472     }
1473
1474     $opts .= ",format=$format" if $format && !$drive->{format};
1475
1476     my $cache_direct = 0;
1477
1478     if (my $cache = $drive->{cache}) {
1479         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1480     } elsif (!drive_is_cdrom($drive)) {
1481         $opts .= ",cache=none";
1482         $cache_direct = 1;
1483     }
1484
1485     # aio native works only with O_DIRECT
1486     if (!$drive->{aio}) {
1487         if($cache_direct) {
1488             $opts .= ",aio=native";
1489         } else {
1490             $opts .= ",aio=threads";
1491         }
1492     }
1493
1494     if (!drive_is_cdrom($drive)) {
1495         my $detectzeroes;
1496         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1497             $detectzeroes = 'off';
1498         } elsif ($drive->{discard}) {
1499             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1500         } else {
1501             # This used to be our default with discard not being specified:
1502             $detectzeroes = 'on';
1503         }
1504         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1505     }
1506
1507     my $pathinfo = $path ? "file=$path," : '';
1508
1509     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1510 }
1511
1512 sub print_netdevice_full {
1513     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1514
1515     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1516
1517     my $device = $net->{model};
1518     if ($net->{model} eq 'virtio') {
1519          $device = 'virtio-net-pci';
1520      };
1521
1522     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1523     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1524     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1525         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1526         my $vectors = $net->{queues} * 2 + 2;
1527         $tmpstr .= ",vectors=$vectors,mq=on";
1528     }
1529     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1530
1531     if (my $mtu = $net->{mtu}) {
1532         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1533             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1534             if ($mtu == 1) {
1535                  $mtu = $bridge_mtu;
1536             } elsif ($mtu < 576) {
1537                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1538             } elsif ($mtu > $bridge_mtu) {
1539                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1540             }
1541             $tmpstr .= ",host_mtu=$mtu";
1542         } else {
1543             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1544         }
1545     }
1546
1547     if ($use_old_bios_files) {
1548         my $romfile;
1549         if ($device eq 'virtio-net-pci') {
1550             $romfile = 'pxe-virtio.rom';
1551         } elsif ($device eq 'e1000') {
1552             $romfile = 'pxe-e1000.rom';
1553         } elsif ($device eq 'ne2k') {
1554             $romfile = 'pxe-ne2k_pci.rom';
1555         } elsif ($device eq 'pcnet') {
1556             $romfile = 'pxe-pcnet.rom';
1557         } elsif ($device eq 'rtl8139') {
1558             $romfile = 'pxe-rtl8139.rom';
1559         }
1560         $tmpstr .= ",romfile=$romfile" if $romfile;
1561     }
1562
1563     return $tmpstr;
1564 }
1565
1566 sub print_netdev_full {
1567     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1568
1569     my $i = '';
1570     if ($netid =~ m/^net(\d+)$/) {
1571         $i = int($1);
1572     }
1573
1574     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1575
1576     my $ifname = "tap${vmid}i$i";
1577
1578     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1579     die "interface name '$ifname' is too long (max 15 character)\n"
1580         if length($ifname) >= 16;
1581
1582     my $vhostparam = '';
1583     if (is_native($arch)) {
1584         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1585     }
1586
1587     my $vmname = $conf->{name} || "vm$vmid";
1588
1589     my $netdev = "";
1590     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1591
1592     if ($net->{bridge}) {
1593         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1594     } else {
1595         $netdev = "type=user,id=$netid,hostname=$vmname";
1596     }
1597
1598     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1599
1600     return $netdev;
1601 }
1602
1603 my $vga_map = {
1604     'cirrus' => 'cirrus-vga',
1605     'std' => 'VGA',
1606     'vmware' => 'vmware-svga',
1607     'virtio' => 'virtio-vga',
1608 };
1609
1610 sub print_vga_device {
1611     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1612
1613     my $type = $vga_map->{$vga->{type}};
1614     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1615         $type = 'virtio-gpu';
1616     }
1617     my $vgamem_mb = $vga->{memory};
1618
1619     my $max_outputs = '';
1620     if ($qxlnum) {
1621         $type = $id ? 'qxl' : 'qxl-vga';
1622
1623         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1624             # set max outputs so linux can have up to 4 qxl displays with one device
1625             if (min_version($machine_version, 4, 1)) {
1626                 $max_outputs = ",max_outputs=4";
1627             }
1628         }
1629     }
1630
1631     die "no devicetype for $vga->{type}\n" if !$type;
1632
1633     my $memory = "";
1634     if ($vgamem_mb) {
1635         if ($vga->{type} eq 'virtio') {
1636             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1637             $memory = ",max_hostmem=$bytes";
1638         } elsif ($qxlnum) {
1639             # from https://www.spice-space.org/multiple-monitors.html
1640             $memory = ",vgamem_mb=$vga->{memory}";
1641             my $ram = $vgamem_mb * 4;
1642             my $vram = $vgamem_mb * 2;
1643             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1644         } else {
1645             $memory = ",vgamem_mb=$vga->{memory}";
1646         }
1647     } elsif ($qxlnum && $id) {
1648         $memory = ",ram_size=67108864,vram_size=33554432";
1649     }
1650
1651     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1652     my $vgaid = "vga" . ($id // '');
1653     my $pciaddr;
1654
1655     if ($q35 && $vgaid eq 'vga') {
1656         # the first display uses pcie.0 bus on q35 machines
1657         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1658     } else {
1659         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1660     }
1661
1662     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1663 }
1664
1665 sub parse_number_sets {
1666     my ($set) = @_;
1667     my $res = [];
1668     foreach my $part (split(/;/, $set)) {
1669         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1670             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1671             push @$res, [ $1, $2 ];
1672         } else {
1673             die "invalid range: $part\n";
1674         }
1675     }
1676     return $res;
1677 }
1678
1679 sub parse_numa {
1680     my ($data) = @_;
1681
1682     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1683     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1684     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1685     return $res;
1686 }
1687
1688 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1689 sub parse_net {
1690     my ($data) = @_;
1691
1692     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1693     if ($@) {
1694         warn $@;
1695         return undef;
1696     }
1697     if (!defined($res->{macaddr})) {
1698         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1699         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1700     }
1701     return $res;
1702 }
1703
1704 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1705 sub parse_ipconfig {
1706     my ($data) = @_;
1707
1708     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1709     if ($@) {
1710         warn $@;
1711         return undef;
1712     }
1713
1714     if ($res->{gw} && !$res->{ip}) {
1715         warn 'gateway specified without specifying an IP address';
1716         return undef;
1717     }
1718     if ($res->{gw6} && !$res->{ip6}) {
1719         warn 'IPv6 gateway specified without specifying an IPv6 address';
1720         return undef;
1721     }
1722     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1723         warn 'gateway specified together with DHCP';
1724         return undef;
1725     }
1726     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1727         # gw6 + auto/dhcp
1728         warn "IPv6 gateway specified together with $res->{ip6} address";
1729         return undef;
1730     }
1731
1732     if (!$res->{ip} && !$res->{ip6}) {
1733         return { ip => 'dhcp', ip6 => 'dhcp' };
1734     }
1735
1736     return $res;
1737 }
1738
1739 sub print_net {
1740     my $net = shift;
1741
1742     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1743 }
1744
1745 sub add_random_macs {
1746     my ($settings) = @_;
1747
1748     foreach my $opt (keys %$settings) {
1749         next if $opt !~ m/^net(\d+)$/;
1750         my $net = parse_net($settings->{$opt});
1751         next if !$net;
1752         $settings->{$opt} = print_net($net);
1753     }
1754 }
1755
1756 sub vm_is_volid_owner {
1757     my ($storecfg, $vmid, $volid) = @_;
1758
1759     if ($volid !~  m|^/|) {
1760         my ($path, $owner);
1761         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1762         if ($owner && ($owner == $vmid)) {
1763             return 1;
1764         }
1765     }
1766
1767     return undef;
1768 }
1769
1770 sub vmconfig_register_unused_drive {
1771     my ($storecfg, $vmid, $conf, $drive) = @_;
1772
1773     if (drive_is_cloudinit($drive)) {
1774         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1775         warn $@ if $@;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         my $volid = $drive->{file};
1778         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1779             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1780         }
1781     }
1782 }
1783
1784 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1785 my $smbios1_fmt = {
1786     uuid => {
1787         type => 'string',
1788         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1789         format_description => 'UUID',
1790         description => "Set SMBIOS1 UUID.",
1791         optional => 1,
1792     },
1793     version => {
1794         type => 'string',
1795         pattern => '[A-Za-z0-9+\/]+={0,2}',
1796         format_description => 'Base64 encoded string',
1797         description => "Set SMBIOS1 version.",
1798         optional => 1,
1799     },
1800     serial => {
1801         type => 'string',
1802         pattern => '[A-Za-z0-9+\/]+={0,2}',
1803         format_description => 'Base64 encoded string',
1804         description => "Set SMBIOS1 serial number.",
1805         optional => 1,
1806     },
1807     manufacturer => {
1808         type => 'string',
1809         pattern => '[A-Za-z0-9+\/]+={0,2}',
1810         format_description => 'Base64 encoded string',
1811         description => "Set SMBIOS1 manufacturer.",
1812         optional => 1,
1813     },
1814     product => {
1815         type => 'string',
1816         pattern => '[A-Za-z0-9+\/]+={0,2}',
1817         format_description => 'Base64 encoded string',
1818         description => "Set SMBIOS1 product ID.",
1819         optional => 1,
1820     },
1821     sku => {
1822         type => 'string',
1823         pattern => '[A-Za-z0-9+\/]+={0,2}',
1824         format_description => 'Base64 encoded string',
1825         description => "Set SMBIOS1 SKU string.",
1826         optional => 1,
1827     },
1828     family => {
1829         type => 'string',
1830         pattern => '[A-Za-z0-9+\/]+={0,2}',
1831         format_description => 'Base64 encoded string',
1832         description => "Set SMBIOS1 family string.",
1833         optional => 1,
1834     },
1835     base64 => {
1836         type => 'boolean',
1837         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1838         optional => 1,
1839     },
1840 };
1841
1842 sub parse_smbios1 {
1843     my ($data) = @_;
1844
1845     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1846     warn $@ if $@;
1847     return $res;
1848 }
1849
1850 sub print_smbios1 {
1851     my ($smbios1) = @_;
1852     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1853 }
1854
1855 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1856
1857 sub parse_watchdog {
1858     my ($value) = @_;
1859
1860     return undef if !$value;
1861
1862     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1863     warn $@ if $@;
1864     return $res;
1865 }
1866
1867 sub parse_guest_agent {
1868     my ($value) = @_;
1869
1870     return {} if !defined($value->{agent});
1871
1872     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1873     warn $@ if $@;
1874
1875     # if the agent is disabled ignore the other potentially set properties
1876     return {} if !$res->{enabled};
1877     return $res;
1878 }
1879
1880 sub parse_vga {
1881     my ($value) = @_;
1882
1883     return {} if !$value;
1884     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1885     warn $@ if $@;
1886     return $res;
1887 }
1888
1889 sub parse_rng {
1890     my ($value) = @_;
1891
1892     return undef if !$value;
1893
1894     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1895     warn $@ if $@;
1896     return $res;
1897 }
1898
1899 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1900 sub verify_usb_device {
1901     my ($value, $noerr) = @_;
1902
1903     return $value if parse_usb_device($value);
1904
1905     return undef if $noerr;
1906
1907     die "unable to parse usb device\n";
1908 }
1909
1910 # add JSON properties for create and set function
1911 sub json_config_properties {
1912     my $prop = shift;
1913
1914     foreach my $opt (keys %$confdesc) {
1915         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1916             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1917         $prop->{$opt} = $confdesc->{$opt};
1918     }
1919
1920     return $prop;
1921 }
1922
1923 # return copy of $confdesc_cloudinit to generate documentation
1924 sub cloudinit_config_properties {
1925
1926     return dclone($confdesc_cloudinit);
1927 }
1928
1929 sub check_type {
1930     my ($key, $value) = @_;
1931
1932     die "unknown setting '$key'\n" if !$confdesc->{$key};
1933
1934     my $type = $confdesc->{$key}->{type};
1935
1936     if (!defined($value)) {
1937         die "got undefined value\n";
1938     }
1939
1940     if ($value =~ m/[\n\r]/) {
1941         die "property contains a line feed\n";
1942     }
1943
1944     if ($type eq 'boolean') {
1945         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1946         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1947         die "type check ('boolean') failed - got '$value'\n";
1948     } elsif ($type eq 'integer') {
1949         return int($1) if $value =~ m/^(\d+)$/;
1950         die "type check ('integer') failed - got '$value'\n";
1951     } elsif ($type eq 'number') {
1952         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1953         die "type check ('number') failed - got '$value'\n";
1954     } elsif ($type eq 'string') {
1955         if (my $fmt = $confdesc->{$key}->{format}) {
1956             PVE::JSONSchema::check_format($fmt, $value);
1957             return $value;
1958         }
1959         $value =~ s/^\"(.*)\"$/$1/;
1960         return $value;
1961     } else {
1962         die "internal error"
1963     }
1964 }
1965
1966 sub destroy_vm {
1967     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1968
1969     my $conf = PVE::QemuConfig->load_config($vmid);
1970
1971     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1972
1973     if ($conf->{template}) {
1974         # check if any base image is still used by a linked clone
1975         PVE::QemuConfig->foreach_volume($conf, sub {
1976                 my ($ds, $drive) = @_;
1977                 return if drive_is_cdrom($drive);
1978
1979                 my $volid = $drive->{file};
1980                 return if !$volid || $volid =~ m|^/|;
1981
1982                 die "base volume '$volid' is still in use by linked cloned\n"
1983                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
1984
1985         });
1986     }
1987
1988     # only remove disks owned by this VM
1989     PVE::QemuConfig->foreach_volume($conf, sub {
1990         my ($ds, $drive) = @_;
1991         return if drive_is_cdrom($drive, 1);
1992
1993         my $volid = $drive->{file};
1994         return if !$volid || $volid =~ m|^/|;
1995
1996         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1997         return if !$path || !$owner || ($owner != $vmid);
1998
1999         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2000         warn "Could not remove disk '$volid', check manually: $@" if $@;
2001     });
2002
2003     # also remove unused disk
2004     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2005     PVE::Storage::foreach_volid($vmdisks, sub {
2006         my ($volid, $sid, $volname, $d) = @_;
2007         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2008         warn $@ if $@;
2009     });
2010
2011     if (defined $replacement_conf) {
2012         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2013     } else {
2014         PVE::QemuConfig->destroy_config($vmid);
2015     }
2016 }
2017
2018 sub parse_vm_config {
2019     my ($filename, $raw) = @_;
2020
2021     return undef if !defined($raw);
2022
2023     my $res = {
2024         digest => Digest::SHA::sha1_hex($raw),
2025         snapshots => {},
2026         pending => {},
2027     };
2028
2029     $filename =~ m|/qemu-server/(\d+)\.conf$|
2030         || die "got strange filename '$filename'";
2031
2032     my $vmid = $1;
2033
2034     my $conf = $res;
2035     my $descr;
2036     my $section = '';
2037
2038     my @lines = split(/\n/, $raw);
2039     foreach my $line (@lines) {
2040         next if $line =~ m/^\s*$/;
2041
2042         if ($line =~ m/^\[PENDING\]\s*$/i) {
2043             $section = 'pending';
2044             if (defined($descr)) {
2045                 $descr =~ s/\s+$//;
2046                 $conf->{description} = $descr;
2047             }
2048             $descr = undef;
2049             $conf = $res->{$section} = {};
2050             next;
2051
2052         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2053             $section = $1;
2054             if (defined($descr)) {
2055                 $descr =~ s/\s+$//;
2056                 $conf->{description} = $descr;
2057             }
2058             $descr = undef;
2059             $conf = $res->{snapshots}->{$section} = {};
2060             next;
2061         }
2062
2063         if ($line =~ m/^\#(.*)\s*$/) {
2064             $descr = '' if !defined($descr);
2065             $descr .= PVE::Tools::decode_text($1) . "\n";
2066             next;
2067         }
2068
2069         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2070             $descr = '' if !defined($descr);
2071             $descr .= PVE::Tools::decode_text($2);
2072         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2073             $conf->{snapstate} = $1;
2074         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2075             my $key = $1;
2076             my $value = $2;
2077             $conf->{$key} = $value;
2078         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2079             my $value = $1;
2080             if ($section eq 'pending') {
2081                 $conf->{delete} = $value; # we parse this later
2082             } else {
2083                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2084             }
2085         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2086             my $key = $1;
2087             my $value = $2;
2088             eval { $value = check_type($key, $value); };
2089             if ($@) {
2090                 warn "vm $vmid - unable to parse value of '$key' - $@";
2091             } else {
2092                 $key = 'ide2' if $key eq 'cdrom';
2093                 my $fmt = $confdesc->{$key}->{format};
2094                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2095                     my $v = parse_drive($key, $value);
2096                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2097                         $v->{file} = $volid;
2098                         $value = print_drive($v);
2099                     } else {
2100                         warn "vm $vmid - unable to parse value of '$key'\n";
2101                         next;
2102                     }
2103                 }
2104
2105                 $conf->{$key} = $value;
2106             }
2107         }
2108     }
2109
2110     if (defined($descr)) {
2111         $descr =~ s/\s+$//;
2112         $conf->{description} = $descr;
2113     }
2114     delete $res->{snapstate}; # just to be sure
2115
2116     return $res;
2117 }
2118
2119 sub write_vm_config {
2120     my ($filename, $conf) = @_;
2121
2122     delete $conf->{snapstate}; # just to be sure
2123
2124     if ($conf->{cdrom}) {
2125         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2126         $conf->{ide2} = $conf->{cdrom};
2127         delete $conf->{cdrom};
2128     }
2129
2130     # we do not use 'smp' any longer
2131     if ($conf->{sockets}) {
2132         delete $conf->{smp};
2133     } elsif ($conf->{smp}) {
2134         $conf->{sockets} = $conf->{smp};
2135         delete $conf->{cores};
2136         delete $conf->{smp};
2137     }
2138
2139     my $used_volids = {};
2140
2141     my $cleanup_config = sub {
2142         my ($cref, $pending, $snapname) = @_;
2143
2144         foreach my $key (keys %$cref) {
2145             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2146                 $key eq 'snapstate' || $key eq 'pending';
2147             my $value = $cref->{$key};
2148             if ($key eq 'delete') {
2149                 die "propertry 'delete' is only allowed in [PENDING]\n"
2150                     if !$pending;
2151                 # fixme: check syntax?
2152                 next;
2153             }
2154             eval { $value = check_type($key, $value); };
2155             die "unable to parse value of '$key' - $@" if $@;
2156
2157             $cref->{$key} = $value;
2158
2159             if (!$snapname && is_valid_drivename($key)) {
2160                 my $drive = parse_drive($key, $value);
2161                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2162             }
2163         }
2164     };
2165
2166     &$cleanup_config($conf);
2167
2168     &$cleanup_config($conf->{pending}, 1);
2169
2170     foreach my $snapname (keys %{$conf->{snapshots}}) {
2171         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2172         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2173     }
2174
2175     # remove 'unusedX' settings if we re-add a volume
2176     foreach my $key (keys %$conf) {
2177         my $value = $conf->{$key};
2178         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2179             delete $conf->{$key};
2180         }
2181     }
2182
2183     my $generate_raw_config = sub {
2184         my ($conf, $pending) = @_;
2185
2186         my $raw = '';
2187
2188         # add description as comment to top of file
2189         if (defined(my $descr = $conf->{description})) {
2190             if ($descr) {
2191                 foreach my $cl (split(/\n/, $descr)) {
2192                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2193                 }
2194             } else {
2195                 $raw .= "#\n" if $pending;
2196             }
2197         }
2198
2199         foreach my $key (sort keys %$conf) {
2200             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2201             $raw .= "$key: $conf->{$key}\n";
2202         }
2203         return $raw;
2204     };
2205
2206     my $raw = &$generate_raw_config($conf);
2207
2208     if (scalar(keys %{$conf->{pending}})){
2209         $raw .= "\n[PENDING]\n";
2210         $raw .= &$generate_raw_config($conf->{pending}, 1);
2211     }
2212
2213     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2214         $raw .= "\n[$snapname]\n";
2215         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2216     }
2217
2218     return $raw;
2219 }
2220
2221 sub load_defaults {
2222
2223     my $res = {};
2224
2225     # we use static defaults from our JSON schema configuration
2226     foreach my $key (keys %$confdesc) {
2227         if (defined(my $default = $confdesc->{$key}->{default})) {
2228             $res->{$key} = $default;
2229         }
2230     }
2231
2232     return $res;
2233 }
2234
2235 sub config_list {
2236     my $vmlist = PVE::Cluster::get_vmlist();
2237     my $res = {};
2238     return $res if !$vmlist || !$vmlist->{ids};
2239     my $ids = $vmlist->{ids};
2240     my $nodename = nodename();
2241
2242     foreach my $vmid (keys %$ids) {
2243         my $d = $ids->{$vmid};
2244         next if !$d->{node} || $d->{node} ne $nodename;
2245         next if !$d->{type} || $d->{type} ne 'qemu';
2246         $res->{$vmid}->{exists} = 1;
2247     }
2248     return $res;
2249 }
2250
2251 # test if VM uses local resources (to prevent migration)
2252 sub check_local_resources {
2253     my ($conf, $noerr) = @_;
2254
2255     my @loc_res = ();
2256
2257     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2258     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2259
2260     push @loc_res, "ivshmem" if $conf->{ivshmem};
2261
2262     foreach my $k (keys %$conf) {
2263         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2264         # sockets are safe: they will recreated be on the target side post-migrate
2265         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2266         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2267     }
2268
2269     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2270
2271     return \@loc_res;
2272 }
2273
2274 # check if used storages are available on all nodes (use by migrate)
2275 sub check_storage_availability {
2276     my ($storecfg, $conf, $node) = @_;
2277
2278     PVE::QemuConfig->foreach_volume($conf, sub {
2279         my ($ds, $drive) = @_;
2280
2281         my $volid = $drive->{file};
2282         return if !$volid;
2283
2284         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2285         return if !$sid;
2286
2287         # check if storage is available on both nodes
2288         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2289         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2290    });
2291 }
2292
2293 # list nodes where all VM images are available (used by has_feature API)
2294 sub shared_nodes {
2295     my ($conf, $storecfg) = @_;
2296
2297     my $nodelist = PVE::Cluster::get_nodelist();
2298     my $nodehash = { map { $_ => 1 } @$nodelist };
2299     my $nodename = nodename();
2300
2301     PVE::QemuConfig->foreach_volume($conf, sub {
2302         my ($ds, $drive) = @_;
2303
2304         my $volid = $drive->{file};
2305         return if !$volid;
2306
2307         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2308         if ($storeid) {
2309             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2310             if ($scfg->{disable}) {
2311                 $nodehash = {};
2312             } elsif (my $avail = $scfg->{nodes}) {
2313                 foreach my $node (keys %$nodehash) {
2314                     delete $nodehash->{$node} if !$avail->{$node};
2315                 }
2316             } elsif (!$scfg->{shared}) {
2317                 foreach my $node (keys %$nodehash) {
2318                     delete $nodehash->{$node} if $node ne $nodename
2319                 }
2320             }
2321         }
2322     });
2323
2324     return $nodehash
2325 }
2326
2327 sub check_local_storage_availability {
2328     my ($conf, $storecfg) = @_;
2329
2330     my $nodelist = PVE::Cluster::get_nodelist();
2331     my $nodehash = { map { $_ => {} } @$nodelist };
2332
2333     PVE::QemuConfig->foreach_volume($conf, sub {
2334         my ($ds, $drive) = @_;
2335
2336         my $volid = $drive->{file};
2337         return if !$volid;
2338
2339         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2340         if ($storeid) {
2341             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2342
2343             if ($scfg->{disable}) {
2344                 foreach my $node (keys %$nodehash) {
2345                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2346                 }
2347             } elsif (my $avail = $scfg->{nodes}) {
2348                 foreach my $node (keys %$nodehash) {
2349                     if (!$avail->{$node}) {
2350                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2351                     }
2352                 }
2353             }
2354         }
2355     });
2356
2357     foreach my $node (values %$nodehash) {
2358         if (my $unavail = $node->{unavailable_storages}) {
2359             $node->{unavailable_storages} = [ sort keys %$unavail ];
2360         }
2361     }
2362
2363     return $nodehash
2364 }
2365
2366 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2367 sub check_running {
2368     my ($vmid, $nocheck, $node) = @_;
2369
2370     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2371     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2372 }
2373
2374 sub vzlist {
2375
2376     my $vzlist = config_list();
2377
2378     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2379
2380     while (defined(my $de = $fd->read)) {
2381         next if $de !~ m/^(\d+)\.pid$/;
2382         my $vmid = $1;
2383         next if !defined($vzlist->{$vmid});
2384         if (my $pid = check_running($vmid)) {
2385             $vzlist->{$vmid}->{pid} = $pid;
2386         }
2387     }
2388
2389     return $vzlist;
2390 }
2391
2392 our $vmstatus_return_properties = {
2393     vmid => get_standard_option('pve-vmid'),
2394     status => {
2395         description => "Qemu process status.",
2396         type => 'string',
2397         enum => ['stopped', 'running'],
2398     },
2399     maxmem => {
2400         description => "Maximum memory in bytes.",
2401         type => 'integer',
2402         optional => 1,
2403         renderer => 'bytes',
2404     },
2405     maxdisk => {
2406         description => "Root disk size in bytes.",
2407         type => 'integer',
2408         optional => 1,
2409         renderer => 'bytes',
2410     },
2411     name => {
2412         description => "VM name.",
2413         type => 'string',
2414         optional => 1,
2415     },
2416     qmpstatus => {
2417         description => "Qemu QMP agent status.",
2418         type => 'string',
2419         optional => 1,
2420     },
2421     pid => {
2422         description => "PID of running qemu process.",
2423         type => 'integer',
2424         optional => 1,
2425     },
2426     uptime => {
2427         description => "Uptime.",
2428         type => 'integer',
2429         optional => 1,
2430         renderer => 'duration',
2431     },
2432     cpus => {
2433         description => "Maximum usable CPUs.",
2434         type => 'number',
2435         optional => 1,
2436     },
2437     lock => {
2438         description => "The current config lock, if any.",
2439         type => 'string',
2440         optional => 1,
2441     },
2442     tags => {
2443         description  => "The current configured tags, if any",
2444         type => 'string',
2445         optional => 1,
2446     },
2447 };
2448
2449 my $last_proc_pid_stat;
2450
2451 # get VM status information
2452 # This must be fast and should not block ($full == false)
2453 # We only query KVM using QMP if $full == true (this can be slow)
2454 sub vmstatus {
2455     my ($opt_vmid, $full) = @_;
2456
2457     my $res = {};
2458
2459     my $storecfg = PVE::Storage::config();
2460
2461     my $list = vzlist();
2462     my $defaults = load_defaults();
2463
2464     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2465
2466     my $cpucount = $cpuinfo->{cpus} || 1;
2467
2468     foreach my $vmid (keys %$list) {
2469         next if $opt_vmid && ($vmid ne $opt_vmid);
2470
2471         my $conf = PVE::QemuConfig->load_config($vmid);
2472
2473         my $d = { vmid => $vmid };
2474         $d->{pid} = $list->{$vmid}->{pid};
2475
2476         # fixme: better status?
2477         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2478
2479         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2480         if (defined($size)) {
2481             $d->{disk} = 0; # no info available
2482             $d->{maxdisk} = $size;
2483         } else {
2484             $d->{disk} = 0;
2485             $d->{maxdisk} = 0;
2486         }
2487
2488         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2489             * ($conf->{cores} || $defaults->{cores});
2490         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2491         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2492
2493         $d->{name} = $conf->{name} || "VM $vmid";
2494         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2495             : $defaults->{memory}*(1024*1024);
2496
2497         if ($conf->{balloon}) {
2498             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2499             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2500                 : $defaults->{shares};
2501         }
2502
2503         $d->{uptime} = 0;
2504         $d->{cpu} = 0;
2505         $d->{mem} = 0;
2506
2507         $d->{netout} = 0;
2508         $d->{netin} = 0;
2509
2510         $d->{diskread} = 0;
2511         $d->{diskwrite} = 0;
2512
2513         $d->{template} = PVE::QemuConfig->is_template($conf);
2514
2515         $d->{serial} = 1 if conf_has_serial($conf);
2516         $d->{lock} = $conf->{lock} if $conf->{lock};
2517         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2518
2519         $res->{$vmid} = $d;
2520     }
2521
2522     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2523     foreach my $dev (keys %$netdev) {
2524         next if $dev !~ m/^tap([1-9]\d*)i/;
2525         my $vmid = $1;
2526         my $d = $res->{$vmid};
2527         next if !$d;
2528
2529         $d->{netout} += $netdev->{$dev}->{receive};
2530         $d->{netin} += $netdev->{$dev}->{transmit};
2531
2532         if ($full) {
2533             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2534             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2535         }
2536
2537     }
2538
2539     my $ctime = gettimeofday;
2540
2541     foreach my $vmid (keys %$list) {
2542
2543         my $d = $res->{$vmid};
2544         my $pid = $d->{pid};
2545         next if !$pid;
2546
2547         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2548         next if !$pstat; # not running
2549
2550         my $used = $pstat->{utime} + $pstat->{stime};
2551
2552         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2553
2554         if ($pstat->{vsize}) {
2555             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2556         }
2557
2558         my $old = $last_proc_pid_stat->{$pid};
2559         if (!$old) {
2560             $last_proc_pid_stat->{$pid} = {
2561                 time => $ctime,
2562                 used => $used,
2563                 cpu => 0,
2564             };
2565             next;
2566         }
2567
2568         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2569
2570         if ($dtime > 1000) {
2571             my $dutime = $used -  $old->{used};
2572
2573             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2574             $last_proc_pid_stat->{$pid} = {
2575                 time => $ctime,
2576                 used => $used,
2577                 cpu => $d->{cpu},
2578             };
2579         } else {
2580             $d->{cpu} = $old->{cpu};
2581         }
2582     }
2583
2584     return $res if !$full;
2585
2586     my $qmpclient = PVE::QMPClient->new();
2587
2588     my $ballooncb = sub {
2589         my ($vmid, $resp) = @_;
2590
2591         my $info = $resp->{'return'};
2592         return if !$info->{max_mem};
2593
2594         my $d = $res->{$vmid};
2595
2596         # use memory assigned to VM
2597         $d->{maxmem} = $info->{max_mem};
2598         $d->{balloon} = $info->{actual};
2599
2600         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2601             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2602             $d->{freemem} = $info->{free_mem};
2603         }
2604
2605         $d->{ballooninfo} = $info;
2606     };
2607
2608     my $blockstatscb = sub {
2609         my ($vmid, $resp) = @_;
2610         my $data = $resp->{'return'} || [];
2611         my $totalrdbytes = 0;
2612         my $totalwrbytes = 0;
2613
2614         for my $blockstat (@$data) {
2615             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2616             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2617
2618             $blockstat->{device} =~ s/drive-//;
2619             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2620         }
2621         $res->{$vmid}->{diskread} = $totalrdbytes;
2622         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2623     };
2624
2625     my $statuscb = sub {
2626         my ($vmid, $resp) = @_;
2627
2628         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2629         # this fails if ballon driver is not loaded, so this must be
2630         # the last commnand (following command are aborted if this fails).
2631         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2632
2633         my $status = 'unknown';
2634         if (!defined($status = $resp->{'return'}->{status})) {
2635             warn "unable to get VM status\n";
2636             return;
2637         }
2638
2639         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2640     };
2641
2642     foreach my $vmid (keys %$list) {
2643         next if $opt_vmid && ($vmid ne $opt_vmid);
2644         next if !$res->{$vmid}->{pid}; # not running
2645         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2646     }
2647
2648     $qmpclient->queue_execute(undef, 2);
2649
2650     foreach my $vmid (keys %$list) {
2651         next if $opt_vmid && ($vmid ne $opt_vmid);
2652         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2653     }
2654
2655     return $res;
2656 }
2657
2658 sub conf_has_serial {
2659     my ($conf) = @_;
2660
2661     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2662         if ($conf->{"serial$i"}) {
2663             return 1;
2664         }
2665     }
2666
2667     return 0;
2668 }
2669
2670 sub conf_has_audio {
2671     my ($conf, $id) = @_;
2672
2673     $id //= 0;
2674     my $audio = $conf->{"audio$id"};
2675     return undef if !defined($audio);
2676
2677     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2678     my $audiodriver = $audioproperties->{driver} // 'spice';
2679
2680     return {
2681         dev => $audioproperties->{device},
2682         dev_id => "audiodev$id",
2683         backend => $audiodriver,
2684         backend_id => "$audiodriver-backend${id}",
2685     };
2686 }
2687
2688 sub audio_devs {
2689     my ($audio, $audiopciaddr, $machine_version) = @_;
2690
2691     my $devs = [];
2692
2693     my $id = $audio->{dev_id};
2694     my $audiodev = "";
2695     if (min_version($machine_version, 4, 2)) {
2696         $audiodev = ",audiodev=$audio->{backend_id}";
2697     }
2698
2699     if ($audio->{dev} eq 'AC97') {
2700         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2701     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2702         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2703         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2704         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2705     } else {
2706         die "unkown audio device '$audio->{dev}', implement me!";
2707     }
2708
2709     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2710
2711     return $devs;
2712 }
2713
2714 sub vga_conf_has_spice {
2715     my ($vga) = @_;
2716
2717     my $vgaconf = parse_vga($vga);
2718     my $vgatype = $vgaconf->{type};
2719     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2720
2721     return $1 || 1;
2722 }
2723
2724 sub is_native($) {
2725     my ($arch) = @_;
2726     return get_host_arch() eq $arch;
2727 }
2728
2729 sub get_vm_arch {
2730     my ($conf) = @_;
2731     return $conf->{arch} // get_host_arch();
2732 }
2733
2734 my $default_machines = {
2735     x86_64 => 'pc',
2736     aarch64 => 'virt',
2737 };
2738
2739 sub get_vm_machine {
2740     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2741
2742     my $machine = $forcemachine || $conf->{machine};
2743
2744     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2745         $arch //= 'x86_64';
2746         $machine ||= $default_machines->{$arch};
2747         if ($add_pve_version) {
2748             $kvmversion //= kvm_user_version();
2749             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2750             $machine .= "+pve$pvever";
2751         }
2752     }
2753
2754     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2755         # for version-pinned machines that do not include a pve-version (e.g.
2756         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2757         $machine .= '+pve0';
2758     }
2759
2760     return $machine;
2761 }
2762
2763 sub get_ovmf_files($) {
2764     my ($arch) = @_;
2765
2766     my $ovmf = $OVMF->{$arch}
2767         or die "no OVMF images known for architecture '$arch'\n";
2768
2769     return @$ovmf;
2770 }
2771
2772 my $Arch2Qemu = {
2773     aarch64 => '/usr/bin/qemu-system-aarch64',
2774     x86_64 => '/usr/bin/qemu-system-x86_64',
2775 };
2776 sub get_command_for_arch($) {
2777     my ($arch) = @_;
2778     return '/usr/bin/kvm' if is_native($arch);
2779
2780     my $cmd = $Arch2Qemu->{$arch}
2781         or die "don't know how to emulate architecture '$arch'\n";
2782     return $cmd;
2783 }
2784
2785 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2786 # to use in a QEMU command line (-cpu element), first array_intersect the result
2787 # of query_supported_ with query_understood_. This is necessary because:
2788 #
2789 # a) query_understood_ returns flags the host cannot use and
2790 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2791 #    flags, but CPU settings - with most of them being flags. Those settings
2792 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2793 #
2794 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2795 # expensive. If you need the value returned from this, you can get it much
2796 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2797 # $accel being 'kvm' or 'tcg'.
2798 #
2799 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2800 # changes, automatically populating pmxcfs.
2801 #
2802 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2803 # since kvm and tcg machines support different flags
2804 #
2805 sub query_supported_cpu_flags {
2806     my ($arch) = @_;
2807
2808     $arch //= get_host_arch();
2809     my $default_machine = $default_machines->{$arch};
2810
2811     my $flags = {};
2812
2813     # FIXME: Once this is merged, the code below should work for ARM as well:
2814     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2815     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2816         $arch eq "aarch64";
2817
2818     my $kvm_supported = defined(kvm_version());
2819     my $qemu_cmd = get_command_for_arch($arch);
2820     my $fakevmid = -1;
2821     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2822
2823     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2824     my $query_supported_run_qemu = sub {
2825         my ($kvm) = @_;
2826
2827         my $flags = {};
2828         my $cmd = [
2829             $qemu_cmd,
2830             '-machine', $default_machine,
2831             '-display', 'none',
2832             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2833             '-mon', 'chardev=qmp,mode=control',
2834             '-pidfile', $pidfile,
2835             '-S', '-daemonize'
2836         ];
2837
2838         if (!$kvm) {
2839             push @$cmd, '-accel', 'tcg';
2840         }
2841
2842         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2843         die "QEMU flag querying VM exited with code " . $rc if $rc;
2844
2845         eval {
2846             my $cmd_result = mon_cmd(
2847                 $fakevmid,
2848                 'query-cpu-model-expansion',
2849                 type => 'full',
2850                 model => { name => 'host' }
2851             );
2852
2853             my $props = $cmd_result->{model}->{props};
2854             foreach my $prop (keys %$props) {
2855                 next if $props->{$prop} ne '1';
2856                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2857                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2858                 # We only keep those with underscores, to match /proc/cpuinfo
2859                 $prop =~ s/\.|-/_/g;
2860                 $flags->{$prop} = 1;
2861             }
2862         };
2863         my $err = $@;
2864
2865         # force stop with 10 sec timeout and 'nocheck'
2866         # always stop, even if QMP failed
2867         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2868
2869         die $err if $err;
2870
2871         return [ sort keys %$flags ];
2872     };
2873
2874     # We need to query QEMU twice, since KVM and TCG have different supported flags
2875     PVE::QemuConfig->lock_config($fakevmid, sub {
2876         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2877         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2878
2879         if ($kvm_supported) {
2880             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2881             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2882         }
2883     });
2884
2885     return $flags;
2886 }
2887
2888 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2889 my $understood_cpu_flag_dir = "/usr/share/kvm";
2890 sub query_understood_cpu_flags {
2891     my $arch = get_host_arch();
2892     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2893
2894     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2895         if ! -e $filepath;
2896
2897     my $raw = file_get_contents($filepath);
2898     $raw =~ s/^\s+|\s+$//g;
2899     my @flags = split(/\s+/, $raw);
2900
2901     return \@flags;
2902 }
2903
2904 sub config_to_command {
2905     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2906
2907     my $cmd = [];
2908     my $globalFlags = [];
2909     my $machineFlags = [];
2910     my $rtcFlags = [];
2911     my $devices = [];
2912     my $pciaddr = '';
2913     my $bridges = {};
2914     my $ostype = $conf->{ostype};
2915     my $winversion = windows_version($ostype);
2916     my $kvm = $conf->{kvm};
2917     my $nodename = nodename();
2918
2919     my $arch = get_vm_arch($conf);
2920     my $kvm_binary = get_command_for_arch($arch);
2921     my $kvmver = kvm_user_version($kvm_binary);
2922
2923     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2924         $kvmver //= "undefined";
2925         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2926     }
2927
2928     my $add_pve_version = min_version($kvmver, 4, 1);
2929
2930     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2931     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2932     $kvm //= 1 if is_native($arch);
2933
2934     $machine_version =~ m/(\d+)\.(\d+)/;
2935     my ($machine_major, $machine_minor) = ($1, $2);
2936
2937     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2938         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2939     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2940         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2941     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2942         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2943         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2944             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2945     }
2946
2947     # if a specific +pve version is required for a feature, use $version_guard
2948     # instead of min_version to allow machines to be run with the minimum
2949     # required version
2950     my $required_pve_version = 0;
2951     my $version_guard = sub {
2952         my ($major, $minor, $pve) = @_;
2953         return 0 if !min_version($machine_version, $major, $minor, $pve);
2954         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2955         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2956         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2957         return 1;
2958     };
2959
2960     if ($kvm) {
2961         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2962             if !defined kvm_version();
2963     }
2964
2965     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2966     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2967     my $use_old_bios_files = undef;
2968     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2969
2970     my $cpuunits = defined($conf->{cpuunits}) ?
2971             $conf->{cpuunits} : $defaults->{cpuunits};
2972
2973     push @$cmd, $kvm_binary;
2974
2975     push @$cmd, '-id', $vmid;
2976
2977     my $vmname = $conf->{name} || "vm$vmid";
2978
2979     push @$cmd, '-name', $vmname;
2980
2981     my $use_virtio = 0;
2982
2983     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2984     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2985     push @$cmd, '-mon', "chardev=qmp,mode=control";
2986
2987     if (min_version($machine_version, 2, 12)) {
2988         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2989         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2990     }
2991
2992     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
2993
2994     push @$cmd, '-daemonize';
2995
2996     if ($conf->{smbios1}) {
2997         my $smbios_conf = parse_smbios1($conf->{smbios1});
2998         if ($smbios_conf->{base64}) {
2999             # Do not pass base64 flag to qemu
3000             delete $smbios_conf->{base64};
3001             my $smbios_string = "";
3002             foreach my $key (keys %$smbios_conf) {
3003                 my $value;
3004                 if ($key eq "uuid") {
3005                     $value = $smbios_conf->{uuid}
3006                 } else {
3007                     $value = decode_base64($smbios_conf->{$key});
3008                 }
3009                 # qemu accepts any binary data, only commas need escaping by double comma
3010                 $value =~ s/,/,,/g;
3011                 $smbios_string .= "," . $key . "=" . $value if $value;
3012             }
3013             push @$cmd, '-smbios', "type=1" . $smbios_string;
3014         } else {
3015             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3016         }
3017     }
3018
3019     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3020         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3021         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3022
3023         my ($path, $format);
3024         if (my $efidisk = $conf->{efidisk0}) {
3025             my $d = parse_drive('efidisk0', $efidisk);
3026             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3027             $format = $d->{format};
3028             if ($storeid) {
3029                 $path = PVE::Storage::path($storecfg, $d->{file});
3030                 if (!defined($format)) {
3031                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3032                     $format = qemu_img_format($scfg, $volname);
3033                 }
3034             } else {
3035                 $path = $d->{file};
3036                 die "efidisk format must be specified\n"
3037                     if !defined($format);
3038             }
3039         } else {
3040             warn "no efidisk configured! Using temporary efivars disk.\n";
3041             $path = "/tmp/$vmid-ovmf.fd";
3042             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3043             $format = 'raw';
3044         }
3045
3046         my $size_str = "";
3047
3048         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3049             $size_str = ",size=" . (-s $ovmf_vars);
3050         }
3051
3052         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3053         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3054     }
3055
3056     # load q35 config
3057     if ($q35) {
3058         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3059         if (min_version($machine_version, 4, 0)) {
3060             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3061         } else {
3062             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3063         }
3064     }
3065
3066     if ($conf->{vmgenid}) {
3067         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3068     }
3069
3070     # add usb controllers
3071     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3072     push @$devices, @usbcontrollers if @usbcontrollers;
3073     my $vga = parse_vga($conf->{vga});
3074
3075     my $qxlnum = vga_conf_has_spice($conf->{vga});
3076     $vga->{type} = 'qxl' if $qxlnum;
3077
3078     if (!$vga->{type}) {
3079         if ($arch eq 'aarch64') {
3080             $vga->{type} = 'virtio';
3081         } elsif (min_version($machine_version, 2, 9)) {
3082             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3083         } else {
3084             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3085         }
3086     }
3087
3088     # enable absolute mouse coordinates (needed by vnc)
3089     my $tablet;
3090     if (defined($conf->{tablet})) {
3091         $tablet = $conf->{tablet};
3092     } else {
3093         $tablet = $defaults->{tablet};
3094         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3095         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3096     }
3097
3098     if ($tablet) {
3099         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3100         my $kbd = print_keyboarddevice_full($conf, $arch);
3101         push @$devices, '-device', $kbd if defined($kbd);
3102     }
3103
3104     # host pci device passthrough
3105     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3106         $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3107
3108     # usb devices
3109     my $usb_dev_features = {};
3110     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3111
3112     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3113     push @$devices, @usbdevices if @usbdevices;
3114     # serial devices
3115     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3116         if (my $path = $conf->{"serial$i"}) {
3117             if ($path eq 'socket') {
3118                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3119                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3120                 # On aarch64, serial0 is the UART device. Qemu only allows
3121                 # connecting UART devices via the '-serial' command line, as
3122                 # the device has a fixed slot on the hardware...
3123                 if ($arch eq 'aarch64' && $i == 0) {
3124                     push @$devices, '-serial', "chardev:serial$i";
3125                 } else {
3126                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3127                 }
3128             } else {
3129                 die "no such serial device\n" if ! -c $path;
3130                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3131                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3132             }
3133         }
3134     }
3135
3136     # parallel devices
3137     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3138         if (my $path = $conf->{"parallel$i"}) {
3139             die "no such parallel device\n" if ! -c $path;
3140             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3141             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3142             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3143         }
3144     }
3145
3146     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3147         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3148         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3149         push @$devices, @$audio_devs;
3150     }
3151
3152     my $sockets = 1;
3153     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3154     $sockets = $conf->{sockets} if  $conf->{sockets};
3155
3156     my $cores = $conf->{cores} || 1;
3157
3158     my $maxcpus = $sockets * $cores;
3159
3160     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3161
3162     my $allowed_vcpus = $cpuinfo->{cpus};
3163
3164     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3165         if ($allowed_vcpus < $maxcpus);
3166
3167     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3168
3169         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3170         for (my $i = 2; $i <= $vcpus; $i++)  {
3171             my $cpustr = print_cpu_device($conf,$i);
3172             push @$cmd, '-device', $cpustr;
3173         }
3174
3175     } else {
3176
3177         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3178     }
3179     push @$cmd, '-nodefaults';
3180
3181     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3182
3183     my $bootindex_hash = {};
3184     my $i = 1;
3185     foreach my $o (split(//, $bootorder)) {
3186         $bootindex_hash->{$o} = $i*100;
3187         $i++;
3188     }
3189
3190     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3191
3192     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3193
3194     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3195
3196     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3197         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3198         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3199         push @$cmd,  '-vnc', "unix:$socket,password";
3200     } else {
3201         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3202         push @$cmd, '-nographic';
3203     }
3204
3205     # time drift fix
3206     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3207     my $useLocaltime = $conf->{localtime};
3208
3209     if ($winversion >= 5) { # windows
3210         $useLocaltime = 1 if !defined($conf->{localtime});
3211
3212         # use time drift fix when acpi is enabled
3213         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3214             $tdf = 1 if !defined($conf->{tdf});
3215         }
3216     }
3217
3218     if ($winversion >= 6) {
3219         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3220         push @$cmd, '-no-hpet';
3221     }
3222
3223     push @$rtcFlags, 'driftfix=slew' if $tdf;
3224
3225     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3226         push @$rtcFlags, "base=$conf->{startdate}";
3227     } elsif ($useLocaltime) {
3228         push @$rtcFlags, 'base=localtime';
3229     }
3230
3231     if ($forcecpu) {
3232         push @$cmd, '-cpu', $forcecpu;
3233     } else {
3234         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3235     }
3236
3237     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3238
3239     push @$cmd, '-S' if $conf->{freeze};
3240
3241     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3242
3243     my $guest_agent = parse_guest_agent($conf);
3244
3245     if ($guest_agent->{enabled}) {
3246         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3247         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3248
3249         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3250             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3251             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3252             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3253         } elsif ($guest_agent->{type} eq 'isa') {
3254             push @$devices, '-device', "isa-serial,chardev=qga0";
3255         }
3256     }
3257
3258     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3259     if ($rng && &$version_guard(4, 1, 2)) {
3260         check_rng_source($rng->{source});
3261
3262         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3263         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3264         my $limiter_str = "";
3265         if ($max_bytes) {
3266             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3267         }
3268
3269         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3270         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3271         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3272     }
3273
3274     my $spice_port;
3275
3276     if ($qxlnum) {
3277         if ($qxlnum > 1) {
3278             if ($winversion){
3279                 for (my $i = 1; $i < $qxlnum; $i++){
3280                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3281                 }
3282             } else {
3283                 # assume other OS works like Linux
3284                 my ($ram, $vram) = ("134217728", "67108864");
3285                 if ($vga->{memory}) {
3286                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3287                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3288                 }
3289                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3290                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3291             }
3292         }
3293
3294         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3295
3296         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3297         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3298         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3299
3300         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3301         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3302         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3303
3304         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3305         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3306
3307         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3308         if ($spice_enhancement->{foldersharing}) {
3309             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3310             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3311         }
3312
3313         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3314         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3315         push @$devices, '-spice', "$spice_opts";
3316     }
3317
3318     # enable balloon by default, unless explicitly disabled
3319     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3320         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3321         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3322     }
3323
3324     if ($conf->{watchdog}) {
3325         my $wdopts = parse_watchdog($conf->{watchdog});
3326         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3327         my $watchdog = $wdopts->{model} || 'i6300esb';
3328         push @$devices, '-device', "$watchdog$pciaddr";
3329         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3330     }
3331
3332     my $vollist = [];
3333     my $scsicontroller = {};
3334     my $ahcicontroller = {};
3335     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3336
3337     # Add iscsi initiator name if available
3338     if (my $initiator = get_initiator_name()) {
3339         push @$devices, '-iscsi', "initiator-name=$initiator";
3340     }
3341
3342     PVE::QemuConfig->foreach_volume($conf, sub {
3343         my ($ds, $drive) = @_;
3344
3345         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3346             push @$vollist, $drive->{file};
3347         }
3348
3349         # ignore efidisk here, already added in bios/fw handling code above
3350         return if $drive->{interface} eq 'efidisk';
3351
3352         $use_virtio = 1 if $ds =~ m/^virtio/;
3353
3354         if (drive_is_cdrom ($drive)) {
3355             if ($bootindex_hash->{d}) {
3356                 $drive->{bootindex} = $bootindex_hash->{d};
3357                 $bootindex_hash->{d} += 1;
3358             }
3359         } else {
3360             if ($bootindex_hash->{c}) {
3361                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3362                 $bootindex_hash->{c} += 1;
3363             }
3364         }
3365
3366         if ($drive->{interface} eq 'virtio'){
3367            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3368         }
3369
3370         if ($drive->{interface} eq 'scsi') {
3371
3372             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3373
3374             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3375                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3376
3377             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3378             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3379
3380             my $iothread = '';
3381             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3382                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3383                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3384             } elsif ($drive->{iothread}) {
3385                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3386             }
3387
3388             my $queues = '';
3389             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3390                 $queues = ",num_queues=$drive->{queues}";
3391             }
3392
3393             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3394             $scsicontroller->{$controller}=1;
3395         }
3396
3397         if ($drive->{interface} eq 'sata') {
3398             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3399             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3400             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3401             $ahcicontroller->{$controller}=1;
3402         }
3403
3404         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3405         push @$devices, '-drive',$drive_cmd;
3406         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3407     });
3408
3409     for (my $i = 0; $i < $MAX_NETS; $i++) {
3410          next if !$conf->{"net$i"};
3411          my $d = parse_net($conf->{"net$i"});
3412          next if !$d;
3413
3414          $use_virtio = 1 if $d->{model} eq 'virtio';
3415
3416          if ($bootindex_hash->{n}) {
3417             $d->{bootindex} = $bootindex_hash->{n};
3418             $bootindex_hash->{n} += 1;
3419          }
3420
3421          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3422          push @$devices, '-netdev', $netdevfull;
3423
3424          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3425          push @$devices, '-device', $netdevicefull;
3426     }
3427
3428     if ($conf->{ivshmem}) {
3429         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3430
3431         my $bus;
3432         if ($q35) {
3433             $bus = print_pcie_addr("ivshmem");
3434         } else {
3435             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3436         }
3437
3438         my $ivshmem_name = $ivshmem->{name} // $vmid;
3439         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3440
3441         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3442         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3443     }
3444
3445     # pci.4 is nested in pci.1
3446     $bridges->{1} = 1 if $bridges->{4};
3447
3448     if (!$q35) {
3449         # add pci bridges
3450         if (min_version($machine_version, 2, 3)) {
3451            $bridges->{1} = 1;
3452            $bridges->{2} = 1;
3453         }
3454
3455         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3456
3457     }
3458
3459     for my $k (sort {$b cmp $a} keys %$bridges) {
3460         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3461
3462         my $k_name = $k;
3463         if ($k == 2 && $legacy_igd) {
3464             $k_name = "$k-igd";
3465         }
3466         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3467
3468         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3469         if ($q35) {
3470             # add after -readconfig pve-q35.cfg
3471             splice @$devices, 2, 0, '-device', $devstr;
3472         } else {
3473             unshift @$devices, '-device', $devstr if $k > 0;
3474         }
3475     }
3476
3477     if (!$kvm) {
3478         push @$machineFlags, 'accel=tcg';
3479     }
3480
3481     my $machine_type_min = $machine_type;
3482     if ($add_pve_version) {
3483         $machine_type_min =~ s/\+pve\d+$//;
3484         $machine_type_min .= "+pve$required_pve_version";
3485     }
3486     push @$machineFlags, "type=${machine_type_min}";
3487
3488     push @$cmd, @$devices;
3489     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3490     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3491     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3492
3493     if (my $vmstate = $conf->{vmstate}) {
3494         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3495         push @$vollist, $vmstate;
3496         push @$cmd, '-loadstate', $statepath;
3497         print "activating and using '$vmstate' as vmstate\n";
3498     }
3499
3500     # add custom args
3501     if ($conf->{args}) {
3502         my $aa = PVE::Tools::split_args($conf->{args});
3503         push @$cmd, @$aa;
3504     }
3505
3506     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3507 }
3508
3509 sub check_rng_source {
3510     my ($source) = @_;
3511
3512     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3513     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3514         if ! -e $source;
3515
3516     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3517     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3518         # Needs to abort, otherwise QEMU crashes on first rng access.
3519         # Note that rng_current cannot be changed to 'none' manually, so
3520         # once the VM is past this point, it is no longer an issue.
3521         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3522             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3523             . " a compatible hardware-RNG is attached to the host.\n";
3524     }
3525 }
3526
3527 sub spice_port {
3528     my ($vmid) = @_;
3529
3530     my $res = mon_cmd($vmid, 'query-spice');
3531
3532     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3533 }
3534
3535 sub vm_devices_list {
3536     my ($vmid) = @_;
3537
3538     my $res = mon_cmd($vmid, 'query-pci');
3539     my $devices_to_check = [];
3540     my $devices = {};
3541     foreach my $pcibus (@$res) {
3542         push @$devices_to_check, @{$pcibus->{devices}},
3543     }
3544
3545     while (@$devices_to_check) {
3546         my $to_check = [];
3547         for my $d (@$devices_to_check) {
3548             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3549             next if !$d->{'pci_bridge'};
3550
3551             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3552             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3553         }
3554         $devices_to_check = $to_check;
3555     }
3556
3557     my $resblock = mon_cmd($vmid, 'query-block');
3558     foreach my $block (@$resblock) {
3559         if($block->{device} =~ m/^drive-(\S+)/){
3560                 $devices->{$1} = 1;
3561         }
3562     }
3563
3564     my $resmice = mon_cmd($vmid, 'query-mice');
3565     foreach my $mice (@$resmice) {
3566         if ($mice->{name} eq 'QEMU HID Tablet') {
3567             $devices->{tablet} = 1;
3568             last;
3569         }
3570     }
3571
3572     # for usb devices there is no query-usb
3573     # but we can iterate over the entries in
3574     # qom-list path=/machine/peripheral
3575     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3576     foreach my $per (@$resperipheral) {
3577         if ($per->{name} =~ m/^usb\d+$/) {
3578             $devices->{$per->{name}} = 1;
3579         }
3580     }
3581
3582     return $devices;
3583 }
3584
3585 sub vm_deviceplug {
3586     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3587
3588     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3589
3590     my $devices_list = vm_devices_list($vmid);
3591     return 1 if defined($devices_list->{$deviceid});
3592
3593     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3594
3595     if ($deviceid eq 'tablet') {
3596
3597         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3598
3599     } elsif ($deviceid eq 'keyboard') {
3600
3601         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3602
3603     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3604
3605         die "usb hotplug currently not reliable\n";
3606         # since we can't reliably hot unplug all added usb devices
3607         # and usb passthrough disables live migration
3608         # we disable usb hotplugging for now
3609         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3610
3611     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3612
3613         qemu_iothread_add($vmid, $deviceid, $device);
3614
3615         qemu_driveadd($storecfg, $vmid, $device);
3616         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3617
3618         qemu_deviceadd($vmid, $devicefull);
3619         eval { qemu_deviceaddverify($vmid, $deviceid); };
3620         if (my $err = $@) {
3621             eval { qemu_drivedel($vmid, $deviceid); };
3622             warn $@ if $@;
3623             die $err;
3624         }
3625
3626     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3627
3628
3629         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3630         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3631         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3632
3633         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3634
3635         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3636             qemu_iothread_add($vmid, $deviceid, $device);
3637             $devicefull .= ",iothread=iothread-$deviceid";
3638         }
3639
3640         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3641             $devicefull .= ",num_queues=$device->{queues}";
3642         }
3643
3644         qemu_deviceadd($vmid, $devicefull);
3645         qemu_deviceaddverify($vmid, $deviceid);
3646
3647     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3648
3649         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3650         qemu_driveadd($storecfg, $vmid, $device);
3651
3652         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3653         eval { qemu_deviceadd($vmid, $devicefull); };
3654         if (my $err = $@) {
3655             eval { qemu_drivedel($vmid, $deviceid); };
3656             warn $@ if $@;
3657             die $err;
3658         }
3659
3660     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3661
3662         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3663
3664         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3665         my $use_old_bios_files = undef;
3666         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3667
3668         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3669         qemu_deviceadd($vmid, $netdevicefull);
3670         eval {
3671             qemu_deviceaddverify($vmid, $deviceid);
3672             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3673         };
3674         if (my $err = $@) {
3675             eval { qemu_netdevdel($vmid, $deviceid); };
3676             warn $@ if $@;
3677             die $err;
3678         }
3679
3680     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3681
3682         my $bridgeid = $2;
3683         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3684         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3685
3686         qemu_deviceadd($vmid, $devicefull);
3687         qemu_deviceaddverify($vmid, $deviceid);
3688
3689     } else {
3690         die "can't hotplug device '$deviceid'\n";
3691     }
3692
3693     return 1;
3694 }
3695
3696 # fixme: this should raise exceptions on error!
3697 sub vm_deviceunplug {
3698     my ($vmid, $conf, $deviceid) = @_;
3699
3700     my $devices_list = vm_devices_list($vmid);
3701     return 1 if !defined($devices_list->{$deviceid});
3702
3703     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3704
3705     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3706
3707         qemu_devicedel($vmid, $deviceid);
3708
3709     } elsif ($deviceid =~ m/^usb\d+$/) {
3710
3711         die "usb hotplug currently not reliable\n";
3712         # when unplugging usb devices this way,
3713         # there may be remaining usb controllers/hubs
3714         # so we disable it for now
3715         qemu_devicedel($vmid, $deviceid);
3716         qemu_devicedelverify($vmid, $deviceid);
3717
3718     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3719
3720         qemu_devicedel($vmid, $deviceid);
3721         qemu_devicedelverify($vmid, $deviceid);
3722         qemu_drivedel($vmid, $deviceid);
3723         qemu_iothread_del($conf, $vmid, $deviceid);
3724
3725     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3726
3727         qemu_devicedel($vmid, $deviceid);
3728         qemu_devicedelverify($vmid, $deviceid);
3729         qemu_iothread_del($conf, $vmid, $deviceid);
3730
3731     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3732
3733         qemu_devicedel($vmid, $deviceid);
3734         qemu_drivedel($vmid, $deviceid);
3735         qemu_deletescsihw($conf, $vmid, $deviceid);
3736
3737     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3738
3739         qemu_devicedel($vmid, $deviceid);
3740         qemu_devicedelverify($vmid, $deviceid);
3741         qemu_netdevdel($vmid, $deviceid);
3742
3743     } else {
3744         die "can't unplug device '$deviceid'\n";
3745     }
3746
3747     return 1;
3748 }
3749
3750 sub qemu_deviceadd {
3751     my ($vmid, $devicefull) = @_;
3752
3753     $devicefull = "driver=".$devicefull;
3754     my %options =  split(/[=,]/, $devicefull);
3755
3756     mon_cmd($vmid, "device_add" , %options);
3757 }
3758
3759 sub qemu_devicedel {
3760     my ($vmid, $deviceid) = @_;
3761
3762     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3763 }
3764
3765 sub qemu_iothread_add {
3766     my($vmid, $deviceid, $device) = @_;
3767
3768     if ($device->{iothread}) {
3769         my $iothreads = vm_iothreads_list($vmid);
3770         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3771     }
3772 }
3773
3774 sub qemu_iothread_del {
3775     my($conf, $vmid, $deviceid) = @_;
3776
3777     my $confid = $deviceid;
3778     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3779         $confid = 'scsi' . $1;
3780     }
3781     my $device = parse_drive($confid, $conf->{$confid});
3782     if ($device->{iothread}) {
3783         my $iothreads = vm_iothreads_list($vmid);
3784         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3785     }
3786 }
3787
3788 sub qemu_objectadd {
3789     my($vmid, $objectid, $qomtype) = @_;
3790
3791     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3792
3793     return 1;
3794 }
3795
3796 sub qemu_objectdel {
3797     my($vmid, $objectid) = @_;
3798
3799     mon_cmd($vmid, "object-del", id => $objectid);
3800
3801     return 1;
3802 }
3803
3804 sub qemu_driveadd {
3805     my ($storecfg, $vmid, $device) = @_;
3806
3807     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3808     $drive =~ s/\\/\\\\/g;
3809     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3810
3811     # If the command succeeds qemu prints: "OK"
3812     return 1 if $ret =~ m/OK/s;
3813
3814     die "adding drive failed: $ret\n";
3815 }
3816
3817 sub qemu_drivedel {
3818     my($vmid, $deviceid) = @_;
3819
3820     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3821     $ret =~ s/^\s+//;
3822
3823     return 1 if $ret eq "";
3824
3825     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3826     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3827
3828     die "deleting drive $deviceid failed : $ret\n";
3829 }
3830
3831 sub qemu_deviceaddverify {
3832     my ($vmid, $deviceid) = @_;
3833
3834     for (my $i = 0; $i <= 5; $i++) {
3835          my $devices_list = vm_devices_list($vmid);
3836          return 1 if defined($devices_list->{$deviceid});
3837          sleep 1;
3838     }
3839
3840     die "error on hotplug device '$deviceid'\n";
3841 }
3842
3843
3844 sub qemu_devicedelverify {
3845     my ($vmid, $deviceid) = @_;
3846
3847     # need to verify that the device is correctly removed as device_del
3848     # is async and empty return is not reliable
3849
3850     for (my $i = 0; $i <= 5; $i++) {
3851          my $devices_list = vm_devices_list($vmid);
3852          return 1 if !defined($devices_list->{$deviceid});
3853          sleep 1;
3854     }
3855
3856     die "error on hot-unplugging device '$deviceid'\n";
3857 }
3858
3859 sub qemu_findorcreatescsihw {
3860     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3861
3862     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3863
3864     my $scsihwid="$controller_prefix$controller";
3865     my $devices_list = vm_devices_list($vmid);
3866
3867     if(!defined($devices_list->{$scsihwid})) {
3868         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3869     }
3870
3871     return 1;
3872 }
3873
3874 sub qemu_deletescsihw {
3875     my ($conf, $vmid, $opt) = @_;
3876
3877     my $device = parse_drive($opt, $conf->{$opt});
3878
3879     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3880         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3881         return 1;
3882     }
3883
3884     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3885
3886     my $devices_list = vm_devices_list($vmid);
3887     foreach my $opt (keys %{$devices_list}) {
3888         if (is_valid_drivename($opt)) {
3889             my $drive = parse_drive($opt, $conf->{$opt});
3890             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3891                 return 1;
3892             }
3893         }
3894     }
3895
3896     my $scsihwid="scsihw$controller";
3897
3898     vm_deviceunplug($vmid, $conf, $scsihwid);
3899
3900     return 1;
3901 }
3902
3903 sub qemu_add_pci_bridge {
3904     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3905
3906     my $bridges = {};
3907
3908     my $bridgeid;
3909
3910     print_pci_addr($device, $bridges, $arch, $machine_type);
3911
3912     while (my ($k, $v) = each %$bridges) {
3913         $bridgeid = $k;
3914     }
3915     return 1 if !defined($bridgeid) || $bridgeid < 1;
3916
3917     my $bridge = "pci.$bridgeid";
3918     my $devices_list = vm_devices_list($vmid);
3919
3920     if (!defined($devices_list->{$bridge})) {
3921         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3922     }
3923
3924     return 1;
3925 }
3926
3927 sub qemu_set_link_status {
3928     my ($vmid, $device, $up) = @_;
3929
3930     mon_cmd($vmid, "set_link", name => $device,
3931                up => $up ? JSON::true : JSON::false);
3932 }
3933
3934 sub qemu_netdevadd {
3935     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3936
3937     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3938     my %options =  split(/[=,]/, $netdev);
3939
3940     if (defined(my $vhost = $options{vhost})) {
3941         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3942     }
3943
3944     if (defined(my $queues = $options{queues})) {
3945         $options{queues} = $queues + 0;
3946     }
3947
3948     mon_cmd($vmid, "netdev_add",  %options);
3949     return 1;
3950 }
3951
3952 sub qemu_netdevdel {
3953     my ($vmid, $deviceid) = @_;
3954
3955     mon_cmd($vmid, "netdev_del", id => $deviceid);
3956 }
3957
3958 sub qemu_usb_hotplug {
3959     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3960
3961     return if !$device;
3962
3963     # remove the old one first
3964     vm_deviceunplug($vmid, $conf, $deviceid);
3965
3966     # check if xhci controller is necessary and available
3967     if ($device->{usb3}) {
3968
3969         my $devicelist = vm_devices_list($vmid);
3970
3971         if (!$devicelist->{xhci}) {
3972             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
3973             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3974         }
3975     }
3976     my $d = parse_usb_device($device->{host});
3977     $d->{usb3} = $device->{usb3};
3978
3979     # add the new one
3980     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3981 }
3982
3983 sub qemu_cpu_hotplug {
3984     my ($vmid, $conf, $vcpus) = @_;
3985
3986     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
3987
3988     my $sockets = 1;
3989     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3990     $sockets = $conf->{sockets} if  $conf->{sockets};
3991     my $cores = $conf->{cores} || 1;
3992     my $maxcpus = $sockets * $cores;
3993
3994     $vcpus = $maxcpus if !$vcpus;
3995
3996     die "you can't add more vcpus than maxcpus\n"
3997         if $vcpus > $maxcpus;
3998
3999     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4000
4001     if ($vcpus < $currentvcpus) {
4002
4003         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4004
4005             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4006                 qemu_devicedel($vmid, "cpu$i");
4007                 my $retry = 0;
4008                 my $currentrunningvcpus = undef;
4009                 while (1) {
4010                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4011                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4012                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4013                     $retry++;
4014                     sleep 1;
4015                 }
4016                 #update conf after each succesfull cpu unplug
4017                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4018                 PVE::QemuConfig->write_config($vmid, $conf);
4019             }
4020         } else {
4021             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4022         }
4023
4024         return;
4025     }
4026
4027     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4028     die "vcpus in running vm does not match its configuration\n"
4029         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4030
4031     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4032
4033         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4034             my $cpustr = print_cpu_device($conf, $i);
4035             qemu_deviceadd($vmid, $cpustr);
4036
4037             my $retry = 0;
4038             my $currentrunningvcpus = undef;
4039             while (1) {
4040                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4041                 last if scalar(@{$currentrunningvcpus}) == $i;
4042                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4043                 sleep 1;
4044                 $retry++;
4045             }
4046             #update conf after each succesfull cpu hotplug
4047             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4048             PVE::QemuConfig->write_config($vmid, $conf);
4049         }
4050     } else {
4051
4052         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4053             mon_cmd($vmid, "cpu-add", id => int($i));
4054         }
4055     }
4056 }
4057
4058 sub qemu_block_set_io_throttle {
4059     my ($vmid, $deviceid,
4060         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4061         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4062         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4063         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4064
4065     return if !check_running($vmid) ;
4066
4067     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4068         bps => int($bps),
4069         bps_rd => int($bps_rd),
4070         bps_wr => int($bps_wr),
4071         iops => int($iops),
4072         iops_rd => int($iops_rd),
4073         iops_wr => int($iops_wr),
4074         bps_max => int($bps_max),
4075         bps_rd_max => int($bps_rd_max),
4076         bps_wr_max => int($bps_wr_max),
4077         iops_max => int($iops_max),
4078         iops_rd_max => int($iops_rd_max),
4079         iops_wr_max => int($iops_wr_max),
4080         bps_max_length => int($bps_max_length),
4081         bps_rd_max_length => int($bps_rd_max_length),
4082         bps_wr_max_length => int($bps_wr_max_length),
4083         iops_max_length => int($iops_max_length),
4084         iops_rd_max_length => int($iops_rd_max_length),
4085         iops_wr_max_length => int($iops_wr_max_length),
4086     );
4087
4088 }
4089
4090 # old code, only used to shutdown old VM after update
4091 sub __read_avail {
4092     my ($fh, $timeout) = @_;
4093
4094     my $sel = new IO::Select;
4095     $sel->add($fh);
4096
4097     my $res = '';
4098     my $buf;
4099
4100     my @ready;
4101     while (scalar (@ready = $sel->can_read($timeout))) {
4102         my $count;
4103         if ($count = $fh->sysread($buf, 8192)) {
4104             if ($buf =~ /^(.*)\(qemu\) $/s) {
4105                 $res .= $1;
4106                 last;
4107             } else {
4108                 $res .= $buf;
4109             }
4110         } else {
4111             if (!defined($count)) {
4112                 die "$!\n";
4113             }
4114             last;
4115         }
4116     }
4117
4118     die "monitor read timeout\n" if !scalar(@ready);
4119
4120     return $res;
4121 }
4122
4123 sub qemu_block_resize {
4124     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4125
4126     my $running = check_running($vmid);
4127
4128     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4129
4130     return if !$running;
4131
4132     my $padding = (1024 - $size % 1024) % 1024;
4133     $size = $size + $padding;
4134
4135     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4136
4137 }
4138
4139 sub qemu_volume_snapshot {
4140     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4141
4142     my $running = check_running($vmid);
4143
4144     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4145         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4146     } else {
4147         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4148     }
4149 }
4150
4151 sub qemu_volume_snapshot_delete {
4152     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4153
4154     my $running = check_running($vmid);
4155
4156     if($running) {
4157
4158         $running = undef;
4159         my $conf = PVE::QemuConfig->load_config($vmid);
4160         PVE::QemuConfig->foreach_volume($conf, sub {
4161             my ($ds, $drive) = @_;
4162             $running = 1 if $drive->{file} eq $volid;
4163         });
4164     }
4165
4166     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4167         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4168     } else {
4169         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4170     }
4171 }
4172
4173 sub set_migration_caps {
4174     my ($vmid) = @_;
4175
4176     my $cap_ref = [];
4177
4178     my $enabled_cap = {
4179         "auto-converge" => 1,
4180         "xbzrle" => 1,
4181         "x-rdma-pin-all" => 0,
4182         "zero-blocks" => 0,
4183         "compress" => 0
4184     };
4185
4186     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4187
4188     for my $supported_capability (@$supported_capabilities) {
4189         push @$cap_ref, {
4190             capability => $supported_capability->{capability},
4191             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4192         };
4193     }
4194
4195     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4196 }
4197
4198 sub foreach_volid {
4199     my ($conf, $func, @param) = @_;
4200
4201     my $volhash = {};
4202
4203     my $test_volid = sub {
4204         my ($key, $drive, $snapname) = @_;
4205
4206         my $volid = $drive->{file};
4207         return if !$volid;
4208
4209         $volhash->{$volid}->{cdrom} //= 1;
4210         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4211
4212         my $replicate = $drive->{replicate} // 1;
4213         $volhash->{$volid}->{replicate} //= 0;
4214         $volhash->{$volid}->{replicate} = 1 if $replicate;
4215
4216         $volhash->{$volid}->{shared} //= 0;
4217         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4218
4219         $volhash->{$volid}->{referenced_in_config} //= 0;
4220         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4221
4222         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4223             if defined($snapname);
4224
4225         my $size = $drive->{size};
4226         $volhash->{$volid}->{size} //= $size if $size;
4227
4228         $volhash->{$volid}->{is_vmstate} //= 0;
4229         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4230
4231         $volhash->{$volid}->{is_unused} //= 0;
4232         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4233     };
4234
4235     my $include_opts = {
4236         extra_keys => ['vmstate'],
4237         include_unused => 1,
4238     };
4239
4240     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4241     foreach my $snapname (keys %{$conf->{snapshots}}) {
4242         my $snap = $conf->{snapshots}->{$snapname};
4243         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4244     }
4245
4246     foreach my $volid (keys %$volhash) {
4247         &$func($volid, $volhash->{$volid}, @param);
4248     }
4249 }
4250
4251 my $fast_plug_option = {
4252     'lock' => 1,
4253     'name' => 1,
4254     'onboot' => 1,
4255     'shares' => 1,
4256     'startup' => 1,
4257     'description' => 1,
4258     'protection' => 1,
4259     'vmstatestorage' => 1,
4260     'hookscript' => 1,
4261     'tags' => 1,
4262 };
4263
4264 # hotplug changes in [PENDING]
4265 # $selection hash can be used to only apply specified options, for
4266 # example: { cores => 1 } (only apply changed 'cores')
4267 # $errors ref is used to return error messages
4268 sub vmconfig_hotplug_pending {
4269     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4270
4271     my $defaults = load_defaults();
4272     my $arch = get_vm_arch($conf);
4273     my $machine_type = get_vm_machine($conf, undef, $arch);
4274
4275     # commit values which do not have any impact on running VM first
4276     # Note: those option cannot raise errors, we we do not care about
4277     # $selection and always apply them.
4278
4279     my $add_error = sub {
4280         my ($opt, $msg) = @_;
4281         $errors->{$opt} = "hotplug problem - $msg";
4282     };
4283
4284     my $changes = 0;
4285     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4286         if ($fast_plug_option->{$opt}) {
4287             $conf->{$opt} = $conf->{pending}->{$opt};
4288             delete $conf->{pending}->{$opt};
4289             $changes = 1;
4290         }
4291     }
4292
4293     if ($changes) {
4294         PVE::QemuConfig->write_config($vmid, $conf);
4295     }
4296
4297     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4298
4299     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4300     foreach my $opt (sort keys %$pending_delete_hash) {
4301         next if $selection && !$selection->{$opt};
4302         my $force = $pending_delete_hash->{$opt}->{force};
4303         eval {
4304             if ($opt eq 'hotplug') {
4305                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4306             } elsif ($opt eq 'tablet') {
4307                 die "skip\n" if !$hotplug_features->{usb};
4308                 if ($defaults->{tablet}) {
4309                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4310                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4311                         if $arch eq 'aarch64';
4312                 } else {
4313                     vm_deviceunplug($vmid, $conf, 'tablet');
4314                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4315                 }
4316             } elsif ($opt =~ m/^usb\d+/) {
4317                 die "skip\n";
4318                 # since we cannot reliably hot unplug usb devices
4319                 # we are disabling it
4320                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4321                 vm_deviceunplug($vmid, $conf, $opt);
4322             } elsif ($opt eq 'vcpus') {
4323                 die "skip\n" if !$hotplug_features->{cpu};
4324                 qemu_cpu_hotplug($vmid, $conf, undef);
4325             } elsif ($opt eq 'balloon') {
4326                 # enable balloon device is not hotpluggable
4327                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4328                 # here we reset the ballooning value to memory
4329                 my $balloon = $conf->{memory} || $defaults->{memory};
4330                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4331             } elsif ($fast_plug_option->{$opt}) {
4332                 # do nothing
4333             } elsif ($opt =~ m/^net(\d+)$/) {
4334                 die "skip\n" if !$hotplug_features->{network};
4335                 vm_deviceunplug($vmid, $conf, $opt);
4336             } elsif (is_valid_drivename($opt)) {
4337                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4338                 vm_deviceunplug($vmid, $conf, $opt);
4339                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4340             } elsif ($opt =~ m/^memory$/) {
4341                 die "skip\n" if !$hotplug_features->{memory};
4342                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4343             } elsif ($opt eq 'cpuunits') {
4344                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4345             } elsif ($opt eq 'cpulimit') {
4346                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4347             } else {
4348                 die "skip\n";
4349             }
4350         };
4351         if (my $err = $@) {
4352             &$add_error($opt, $err) if $err ne "skip\n";
4353         } else {
4354             delete $conf->{$opt};
4355             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4356         }
4357     }
4358
4359     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4360     $apply_pending_cloudinit = sub {
4361         return if $apply_pending_cloudinit_done; # once is enough
4362         $apply_pending_cloudinit_done = 1; # once is enough
4363
4364         my ($key, $value) = @_;
4365
4366         my @cloudinit_opts = keys %$confdesc_cloudinit;
4367         foreach my $opt (keys %{$conf->{pending}}) {
4368             next if !grep { $_ eq $opt } @cloudinit_opts;
4369             $conf->{$opt} = delete $conf->{pending}->{$opt};
4370         }
4371
4372         my $new_conf = { %$conf };
4373         $new_conf->{$key} = $value;
4374         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4375     };
4376
4377     foreach my $opt (keys %{$conf->{pending}}) {
4378         next if $selection && !$selection->{$opt};
4379         my $value = $conf->{pending}->{$opt};
4380         eval {
4381             if ($opt eq 'hotplug') {
4382                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4383             } elsif ($opt eq 'tablet') {
4384                 die "skip\n" if !$hotplug_features->{usb};
4385                 if ($value == 1) {
4386                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4387                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4388                         if $arch eq 'aarch64';
4389                 } elsif ($value == 0) {
4390                     vm_deviceunplug($vmid, $conf, 'tablet');
4391                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4392                 }
4393             } elsif ($opt =~ m/^usb\d+$/) {
4394                 die "skip\n";
4395                 # since we cannot reliably hot unplug usb devices
4396                 # we are disabling it
4397                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4398                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4399                 die "skip\n" if !$d;
4400                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4401             } elsif ($opt eq 'vcpus') {
4402                 die "skip\n" if !$hotplug_features->{cpu};
4403                 qemu_cpu_hotplug($vmid, $conf, $value);
4404             } elsif ($opt eq 'balloon') {
4405                 # enable/disable balloning device is not hotpluggable
4406                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4407                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4408                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4409
4410                 # allow manual ballooning if shares is set to zero
4411                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4412                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4413                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4414                 }
4415             } elsif ($opt =~ m/^net(\d+)$/) {
4416                 # some changes can be done without hotplug
4417                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4418                                     $vmid, $opt, $value, $arch, $machine_type);
4419             } elsif (is_valid_drivename($opt)) {
4420                 die "skip\n" if $opt eq 'efidisk0';
4421                 # some changes can be done without hotplug
4422                 my $drive = parse_drive($opt, $value);
4423                 if (drive_is_cloudinit($drive)) {
4424                     &$apply_pending_cloudinit($opt, $value);
4425                 }
4426                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4427                                      $vmid, $opt, $value, $arch, $machine_type);
4428             } elsif ($opt =~ m/^memory$/) { #dimms
4429                 die "skip\n" if !$hotplug_features->{memory};
4430                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4431             } elsif ($opt eq 'cpuunits') {
4432                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4433             } elsif ($opt eq 'cpulimit') {
4434                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4435                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4436             } else {
4437                 die "skip\n";  # skip non-hot-pluggable options
4438             }
4439         };
4440         if (my $err = $@) {
4441             &$add_error($opt, $err) if $err ne "skip\n";
4442         } else {
4443             $conf->{$opt} = $value;
4444             delete $conf->{pending}->{$opt};
4445         }
4446     }
4447
4448     PVE::QemuConfig->write_config($vmid, $conf);
4449 }
4450
4451 sub try_deallocate_drive {
4452     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4453
4454     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4455         my $volid = $drive->{file};
4456         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4457             my $sid = PVE::Storage::parse_volume_id($volid);
4458             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4459
4460             # check if the disk is really unused
4461             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4462                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4463             PVE::Storage::vdisk_free($storecfg, $volid);
4464             return 1;
4465         } else {
4466             # If vm is not owner of this disk remove from config
4467             return 1;
4468         }
4469     }
4470
4471     return undef;
4472 }
4473
4474 sub vmconfig_delete_or_detach_drive {
4475     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4476
4477     my $drive = parse_drive($opt, $conf->{$opt});
4478
4479     my $rpcenv = PVE::RPCEnvironment::get();
4480     my $authuser = $rpcenv->get_user();
4481
4482     if ($force) {
4483         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4484         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4485     } else {
4486         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4487     }
4488 }
4489
4490
4491
4492 sub vmconfig_apply_pending {
4493     my ($vmid, $conf, $storecfg, $errors) = @_;
4494
4495     my $add_apply_error = sub {
4496         my ($opt, $msg) = @_;
4497         my $err_msg = "unable to apply pending change $opt : $msg";
4498         $errors->{$opt} = $err_msg;
4499         warn $err_msg;
4500     };
4501
4502     # cold plug
4503
4504     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4505     foreach my $opt (sort keys %$pending_delete_hash) {
4506         my $force = $pending_delete_hash->{$opt}->{force};
4507         eval {
4508             if ($opt =~ m/^unused/) {
4509                 die "internal error";
4510             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4511                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4512             }
4513         };
4514         if (my $err = $@) {
4515             $add_apply_error->($opt, $err);
4516         } else {
4517             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4518             delete $conf->{$opt};
4519         }
4520     }
4521
4522     PVE::QemuConfig->cleanup_pending($conf);
4523
4524     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4525         next if $opt eq 'delete'; # just to be sure
4526         eval {
4527             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4528                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4529             }
4530         };
4531         if (my $err = $@) {
4532             $add_apply_error->($opt, $err);
4533         } else {
4534             $conf->{$opt} = delete $conf->{pending}->{$opt};
4535         }
4536     }
4537
4538     # write all changes at once to avoid unnecessary i/o
4539     PVE::QemuConfig->write_config($vmid, $conf);
4540 }
4541
4542 sub vmconfig_update_net {
4543     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4544
4545     my $newnet = parse_net($value);
4546
4547     if ($conf->{$opt}) {
4548         my $oldnet = parse_net($conf->{$opt});
4549
4550         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4551             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4552             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4553             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4554
4555             # for non online change, we try to hot-unplug
4556             die "skip\n" if !$hotplug;
4557             vm_deviceunplug($vmid, $conf, $opt);
4558         } else {
4559
4560             die "internal error" if $opt !~ m/net(\d+)/;
4561             my $iface = "tap${vmid}i$1";
4562
4563             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4564                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4565                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4566                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4567                 PVE::Network::tap_unplug($iface);
4568
4569                 if ($have_sdn) {
4570                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4571                 } else {
4572                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4573                 }
4574             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4575                 # Rate can be applied on its own but any change above needs to
4576                 # include the rate in tap_plug since OVS resets everything.
4577                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4578             }
4579
4580             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4581                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4582             }
4583
4584             return 1;
4585         }
4586     }
4587
4588     if ($hotplug) {
4589         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4590     } else {
4591         die "skip\n";
4592     }
4593 }
4594
4595 sub vmconfig_update_disk {
4596     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4597
4598     my $drive = parse_drive($opt, $value);
4599
4600     if ($conf->{$opt}) {
4601
4602         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4603
4604             my $media = $drive->{media} || 'disk';
4605             my $oldmedia = $old_drive->{media} || 'disk';
4606             die "unable to change media type\n" if $media ne $oldmedia;
4607
4608             if (!drive_is_cdrom($old_drive)) {
4609
4610                 if ($drive->{file} ne $old_drive->{file}) {
4611
4612                     die "skip\n" if !$hotplug;
4613
4614                     # unplug and register as unused
4615                     vm_deviceunplug($vmid, $conf, $opt);
4616                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4617
4618                 } else {
4619                     # update existing disk
4620
4621                     # skip non hotpluggable value
4622                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4623                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4624                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4625                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4626                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4627                         die "skip\n";
4628                     }
4629
4630                     # apply throttle
4631                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4632                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4633                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4634                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4635                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4636                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4637                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4638                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4639                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4640                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4641                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4642                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4643                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4644                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4645                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4646                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4647                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4648                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4649
4650                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4651                                                    ($drive->{mbps} || 0)*1024*1024,
4652                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4653                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4654                                                    $drive->{iops} || 0,
4655                                                    $drive->{iops_rd} || 0,
4656                                                    $drive->{iops_wr} || 0,
4657                                                    ($drive->{mbps_max} || 0)*1024*1024,
4658                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4659                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4660                                                    $drive->{iops_max} || 0,
4661                                                    $drive->{iops_rd_max} || 0,
4662                                                    $drive->{iops_wr_max} || 0,
4663                                                    $drive->{bps_max_length} || 1,
4664                                                    $drive->{bps_rd_max_length} || 1,
4665                                                    $drive->{bps_wr_max_length} || 1,
4666                                                    $drive->{iops_max_length} || 1,
4667                                                    $drive->{iops_rd_max_length} || 1,
4668                                                    $drive->{iops_wr_max_length} || 1);
4669
4670                     }
4671
4672                     return 1;
4673                 }
4674
4675             } else { # cdrom
4676
4677                 if ($drive->{file} eq 'none') {
4678                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4679                     if (drive_is_cloudinit($old_drive)) {
4680                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4681                     }
4682                 } else {
4683                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4684
4685                     # force eject if locked
4686                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4687
4688                     if ($path) {
4689                         mon_cmd($vmid, "blockdev-change-medium",
4690                             id => "$opt", filename => "$path");
4691                     }
4692                 }
4693
4694                 return 1;
4695             }
4696         }
4697     }
4698
4699     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4700     # hotplug new disks
4701     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4702     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4703 }
4704
4705 # called in locked context by incoming migration
4706 sub vm_migrate_get_nbd_disks {
4707     my ($storecfg, $conf, $replicated_volumes) = @_;
4708
4709     my $local_volumes = {};
4710     PVE::QemuConfig->foreach_volume($conf, sub {
4711         my ($ds, $drive) = @_;
4712
4713         return if drive_is_cdrom($drive);
4714
4715         my $volid = $drive->{file};
4716
4717         return if !$volid;
4718
4719         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4720
4721         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4722         return if $scfg->{shared};
4723
4724         # replicated disks re-use existing state via bitmap
4725         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4726         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4727     });
4728     return $local_volumes;
4729 }
4730
4731 # called in locked context by incoming migration
4732 sub vm_migrate_alloc_nbd_disks {
4733     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4734
4735     my $format = undef;
4736
4737     my $nbd = {};
4738     foreach my $opt (sort keys %$source_volumes) {
4739         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4740
4741         if ($use_existing) {
4742             $nbd->{$opt}->{drivestr} = print_drive($drive);
4743             $nbd->{$opt}->{volid} = $volid;
4744             $nbd->{$opt}->{replicated} = 1;
4745             next;
4746         }
4747
4748         # If a remote storage is specified and the format of the original
4749         # volume is not available there, fall back to the default format.
4750         # Otherwise use the same format as the original.
4751         if (!$storagemap->{identity}) {
4752             $storeid = map_storage($storagemap, $storeid);
4753             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4754             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4755             my $fileFormat = qemu_img_format($scfg, $volname);
4756             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4757         } else {
4758             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4759             $format = qemu_img_format($scfg, $volname);
4760         }
4761
4762         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4763         my $newdrive = $drive;
4764         $newdrive->{format} = $format;
4765         $newdrive->{file} = $newvolid;
4766         my $drivestr = print_drive($newdrive);
4767         $nbd->{$opt}->{drivestr} = $drivestr;
4768         $nbd->{$opt}->{volid} = $newvolid;
4769     }
4770
4771     return $nbd;
4772 }
4773
4774 # see vm_start_nolock for parameters, additionally:
4775 # migrate_opts:
4776 #   storagemap = parsed storage map for allocating NBD disks
4777 sub vm_start {
4778     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4779
4780     return PVE::QemuConfig->lock_config($vmid, sub {
4781         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4782
4783         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4784
4785         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4786
4787         PVE::QemuConfig->check_lock($conf)
4788             if !($params->{skiplock} || $params->{resume});
4789
4790         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4791
4792         if (my $storagemap = $migrate_opts->{storagemap}) {
4793             my $replicated = $migrate_opts->{replicated_volumes};
4794             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4795             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4796
4797             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4798                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4799             }
4800         }
4801
4802         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4803     });
4804 }
4805
4806
4807 # params:
4808 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4809 #   skiplock => 0/1, skip checking for config lock
4810 #   forcemachine => to force Qemu machine (rollback/migration)
4811 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4812 #   timeout => in seconds
4813 #   paused => start VM in paused state (backup)
4814 #   resume => resume from hibernation
4815 # migrate_opts:
4816 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4817 #   migratedfrom => source node
4818 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4819 #   network => CIDR of migration network
4820 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4821 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4822 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4823 sub vm_start_nolock {
4824     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4825
4826     my $statefile = $params->{statefile};
4827     my $resume = $params->{resume};
4828
4829     my $migratedfrom = $migrate_opts->{migratedfrom};
4830     my $migration_type = $migrate_opts->{type};
4831
4832     my $res = {};
4833
4834     # clean up leftover reboot request files
4835     eval { clear_reboot_request($vmid); };
4836     warn $@ if $@;
4837
4838     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4839         vmconfig_apply_pending($vmid, $conf, $storecfg);
4840         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4841     }
4842
4843     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4844
4845     my $defaults = load_defaults();
4846
4847     # set environment variable useful inside network script
4848     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4849
4850     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4851
4852     my $forcemachine = $params->{forcemachine};
4853     my $forcecpu = $params->{forcecpu};
4854     if ($resume) {
4855         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4856         $forcemachine = $conf->{runningmachine};
4857         $forcecpu = $conf->{runningcpu};
4858         print "Resuming suspended VM\n";
4859     }
4860
4861     my ($cmd, $vollist, $spice_port) =
4862         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4863
4864     my $migration_ip;
4865     my $get_migration_ip = sub {
4866         my ($nodename) = @_;
4867
4868         return $migration_ip if defined($migration_ip);
4869
4870         my $cidr = $migrate_opts->{network};
4871
4872         if (!defined($cidr)) {
4873             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4874             $cidr = $dc_conf->{migration}->{network};
4875         }
4876
4877         if (defined($cidr)) {
4878             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4879
4880             die "could not get IP: no address configured on local " .
4881                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4882
4883             die "could not get IP: multiple addresses configured on local " .
4884                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4885
4886             $migration_ip = @$ips[0];
4887         }
4888
4889         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4890             if !defined($migration_ip);
4891
4892         return $migration_ip;
4893     };
4894
4895     my $migrate_uri;
4896     if ($statefile) {
4897         if ($statefile eq 'tcp') {
4898             my $localip = "localhost";
4899             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4900             my $nodename = nodename();
4901
4902             if (!defined($migration_type)) {
4903                 if (defined($datacenterconf->{migration}->{type})) {
4904                     $migration_type = $datacenterconf->{migration}->{type};
4905                 } else {
4906                     $migration_type = 'secure';
4907                 }
4908             }
4909
4910             if ($migration_type eq 'insecure') {
4911                 $localip = $get_migration_ip->($nodename);
4912                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4913             }
4914
4915             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4916             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4917             $migrate_uri = "tcp:${localip}:${migrate_port}";
4918             push @$cmd, '-incoming', $migrate_uri;
4919             push @$cmd, '-S';
4920
4921         } elsif ($statefile eq 'unix') {
4922             # should be default for secure migrations as a ssh TCP forward
4923             # tunnel is not deterministic reliable ready and fails regurarly
4924             # to set up in time, so use UNIX socket forwards
4925             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4926             unlink $socket_addr;
4927
4928             $migrate_uri = "unix:$socket_addr";
4929
4930             push @$cmd, '-incoming', $migrate_uri;
4931             push @$cmd, '-S';
4932
4933         } elsif (-e $statefile) {
4934             push @$cmd, '-loadstate', $statefile;
4935         } else {
4936             my $statepath = PVE::Storage::path($storecfg, $statefile);
4937             push @$vollist, $statefile;
4938             push @$cmd, '-loadstate', $statepath;
4939         }
4940     } elsif ($params->{paused}) {
4941         push @$cmd, '-S';
4942     }
4943
4944     # host pci devices
4945     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
4946       my $d = parse_hostpci($conf->{"hostpci$i"});
4947       next if !$d;
4948       my $pcidevices = $d->{pciid};
4949       foreach my $pcidevice (@$pcidevices) {
4950             my $pciid = $pcidevice->{id};
4951
4952             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4953             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4954             die "no pci device info for device '$pciid'\n" if !$info;
4955
4956             if ($d->{mdev}) {
4957                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4958                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4959             } else {
4960                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4961                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4962                 die "can't reset pci device '$pciid'\n"
4963                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4964             }
4965       }
4966     }
4967
4968     PVE::Storage::activate_volumes($storecfg, $vollist);
4969
4970     eval {
4971         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4972             outfunc => sub {}, errfunc => sub {});
4973     };
4974     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4975     # timeout should be more than enough here...
4976     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4977
4978     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4979                                               : $defaults->{cpuunits};
4980
4981     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4982     my %run_params = (
4983         timeout => $statefile ? undef : $start_timeout,
4984         umask => 0077,
4985         noerr => 1,
4986     );
4987
4988     # when migrating, prefix QEMU output so other side can pick up any
4989     # errors that might occur and show the user
4990     if ($migratedfrom) {
4991         $run_params{quiet} = 1;
4992         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4993     }
4994
4995     my %properties = (
4996         Slice => 'qemu.slice',
4997         KillMode => 'none',
4998         CPUShares => $cpuunits
4999     );
5000
5001     if (my $cpulimit = $conf->{cpulimit}) {
5002         $properties{CPUQuota} = int($cpulimit * 100);
5003     }
5004     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5005
5006     my $run_qemu = sub {
5007         PVE::Tools::run_fork sub {
5008             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5009
5010             my $exitcode = run_command($cmd, %run_params);
5011             die "QEMU exited with code $exitcode\n" if $exitcode;
5012         };
5013     };
5014
5015     if ($conf->{hugepages}) {
5016
5017         my $code = sub {
5018             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5019             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5020
5021             PVE::QemuServer::Memory::hugepages_mount();
5022             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5023
5024             eval { $run_qemu->() };
5025             if (my $err = $@) {
5026                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5027                 die $err;
5028             }
5029
5030             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5031         };
5032         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5033
5034     } else {
5035         eval { $run_qemu->() };
5036     }
5037
5038     if (my $err = $@) {
5039         # deactivate volumes if start fails
5040         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5041         die "start failed: $err";
5042     }
5043
5044     print "migration listens on $migrate_uri\n" if $migrate_uri;
5045     $res->{migrate_uri} = $migrate_uri;
5046
5047     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5048         eval { mon_cmd($vmid, "cont"); };
5049         warn $@ if $@;
5050     }
5051
5052     #start nbd server for storage migration
5053     if (my $nbd = $migrate_opts->{nbd}) {
5054         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5055
5056         my $migrate_storage_uri;
5057         # nbd_protocol_version > 0 for unix socket support
5058         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5059             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5060             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5061             $migrate_storage_uri = "nbd:unix:$socket_path";
5062         } else {
5063             my $nodename = nodename();
5064             my $localip = $get_migration_ip->($nodename);
5065             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5066             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5067
5068             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5069             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5070             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5071         }
5072
5073         $res->{migrate_storage_uri} = $migrate_storage_uri;
5074
5075         foreach my $opt (sort keys %$nbd) {
5076             my $drivestr = $nbd->{$opt}->{drivestr};
5077             my $volid = $nbd->{$opt}->{volid};
5078             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5079             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5080             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5081             print "re-using replicated volume: $opt - $volid\n"
5082                 if $nbd->{$opt}->{replicated};
5083
5084             $res->{drives}->{$opt} = $nbd->{$opt};
5085             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5086         }
5087     }
5088
5089     if ($migratedfrom) {
5090         eval {
5091             set_migration_caps($vmid);
5092         };
5093         warn $@ if $@;
5094
5095         if ($spice_port) {
5096             print "spice listens on port $spice_port\n";
5097             $res->{spice_port} = $spice_port;
5098             if ($migrate_opts->{spice_ticket}) {
5099                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5100                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5101             }
5102         }
5103
5104     } else {
5105         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5106             if !$statefile && $conf->{balloon};
5107
5108         foreach my $opt (keys %$conf) {
5109             next if $opt !~  m/^net\d+$/;
5110             my $nicconf = parse_net($conf->{$opt});
5111             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5112         }
5113     }
5114
5115     mon_cmd($vmid, 'qom-set',
5116                 path => "machine/peripheral/balloon0",
5117                 property => "guest-stats-polling-interval",
5118                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5119
5120     if ($resume) {
5121         print "Resumed VM, removing state\n";
5122         if (my $vmstate = $conf->{vmstate}) {
5123             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5124             PVE::Storage::vdisk_free($storecfg, $vmstate);
5125         }
5126         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5127         PVE::QemuConfig->write_config($vmid, $conf);
5128     }
5129
5130     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5131
5132     return $res;
5133 }
5134
5135 sub vm_commandline {
5136     my ($storecfg, $vmid, $snapname) = @_;
5137
5138     my $conf = PVE::QemuConfig->load_config($vmid);
5139     my $forcemachine;
5140     my $forcecpu;
5141
5142     if ($snapname) {
5143         my $snapshot = $conf->{snapshots}->{$snapname};
5144         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5145
5146         # check for machine or CPU overrides in snapshot
5147         $forcemachine = $snapshot->{runningmachine};
5148         $forcecpu = $snapshot->{runningcpu};
5149
5150         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5151
5152         $conf = $snapshot;
5153     }
5154
5155     my $defaults = load_defaults();
5156
5157     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5158         $forcemachine, $forcecpu);
5159
5160     return PVE::Tools::cmd2string($cmd);
5161 }
5162
5163 sub vm_reset {
5164     my ($vmid, $skiplock) = @_;
5165
5166     PVE::QemuConfig->lock_config($vmid, sub {
5167
5168         my $conf = PVE::QemuConfig->load_config($vmid);
5169
5170         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5171
5172         mon_cmd($vmid, "system_reset");
5173     });
5174 }
5175
5176 sub get_vm_volumes {
5177     my ($conf) = @_;
5178
5179     my $vollist = [];
5180     foreach_volid($conf, sub {
5181         my ($volid, $attr) = @_;
5182
5183         return if $volid =~ m|^/|;
5184
5185         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5186         return if !$sid;
5187
5188         push @$vollist, $volid;
5189     });
5190
5191     return $vollist;
5192 }
5193
5194 sub vm_stop_cleanup {
5195     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5196
5197     eval {
5198
5199         if (!$keepActive) {
5200             my $vollist = get_vm_volumes($conf);
5201             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5202         }
5203
5204         foreach my $ext (qw(mon qmp pid vnc qga)) {
5205             unlink "/var/run/qemu-server/${vmid}.$ext";
5206         }
5207
5208         if ($conf->{ivshmem}) {
5209             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5210             # just delete it for now, VMs which have this already open do not
5211             # are affected, but new VMs will get a separated one. If this
5212             # becomes an issue we either add some sort of ref-counting or just
5213             # add a "don't delete on stop" flag to the ivshmem format.
5214             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5215         }
5216
5217         foreach my $key (keys %$conf) {
5218             next if $key !~ m/^hostpci(\d+)$/;
5219             my $hostpciindex = $1;
5220             my $d = parse_hostpci($conf->{$key});
5221             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5222
5223             foreach my $pci (@{$d->{pciid}}) {
5224                 my $pciid = $pci->{id};
5225                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5226             }
5227         }
5228
5229         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5230     };
5231     warn $@ if $@; # avoid errors - just warn
5232 }
5233
5234 # call only in locked context
5235 sub _do_vm_stop {
5236     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5237
5238     my $pid = check_running($vmid, $nocheck);
5239     return if !$pid;
5240
5241     my $conf;
5242     if (!$nocheck) {
5243         $conf = PVE::QemuConfig->load_config($vmid);
5244         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5245         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5246             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5247             $timeout = $opts->{down} if $opts->{down};
5248         }
5249         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5250     }
5251
5252     eval {
5253         if ($shutdown) {
5254             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5255                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5256             } else {
5257                 mon_cmd($vmid, "system_powerdown");
5258             }
5259         } else {
5260             mon_cmd($vmid, "quit");
5261         }
5262     };
5263     my $err = $@;
5264
5265     if (!$err) {
5266         $timeout = 60 if !defined($timeout);
5267
5268         my $count = 0;
5269         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5270             $count++;
5271             sleep 1;
5272         }
5273
5274         if ($count >= $timeout) {
5275             if ($force) {
5276                 warn "VM still running - terminating now with SIGTERM\n";
5277                 kill 15, $pid;
5278             } else {
5279                 die "VM quit/powerdown failed - got timeout\n";
5280             }
5281         } else {
5282             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5283             return;
5284         }
5285     } else {
5286        if (!check_running($vmid, $nocheck)) {
5287             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5288             return;
5289        }
5290        if ($force) {
5291             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5292             kill 15, $pid;
5293         } else {
5294             die "VM quit/powerdown failed\n";
5295         }
5296     }
5297
5298     # wait again
5299     $timeout = 10;
5300
5301     my $count = 0;
5302     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5303         $count++;
5304         sleep 1;
5305     }
5306
5307     if ($count >= $timeout) {
5308         warn "VM still running - terminating now with SIGKILL\n";
5309         kill 9, $pid;
5310         sleep 1;
5311     }
5312
5313     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5314 }
5315
5316 # Note: use $nocheck to skip tests if VM configuration file exists.
5317 # We need that when migration VMs to other nodes (files already moved)
5318 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5319 sub vm_stop {
5320     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5321
5322     $force = 1 if !defined($force) && !$shutdown;
5323
5324     if ($migratedfrom){
5325         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5326         kill 15, $pid if $pid;
5327         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5328         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5329         return;
5330     }
5331
5332     PVE::QemuConfig->lock_config($vmid, sub {
5333         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5334    });
5335 }
5336
5337 sub vm_reboot {
5338     my ($vmid, $timeout) = @_;
5339
5340     PVE::QemuConfig->lock_config($vmid, sub {
5341         eval {
5342
5343             # only reboot if running, as qmeventd starts it again on a stop event
5344             return if !check_running($vmid);
5345
5346             create_reboot_request($vmid);
5347
5348             my $storecfg = PVE::Storage::config();
5349             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5350
5351         };
5352         if (my $err = $@) {
5353             # avoid that the next normal shutdown will be confused for a reboot
5354             clear_reboot_request($vmid);
5355             die $err;
5356         }
5357    });
5358 }
5359
5360 # note: if using the statestorage parameter, the caller has to check privileges
5361 sub vm_suspend {
5362     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5363
5364     my $conf;
5365     my $path;
5366     my $storecfg;
5367     my $vmstate;
5368
5369     PVE::QemuConfig->lock_config($vmid, sub {
5370
5371         $conf = PVE::QemuConfig->load_config($vmid);
5372
5373         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5374         PVE::QemuConfig->check_lock($conf)
5375             if !($skiplock || $is_backing_up);
5376
5377         die "cannot suspend to disk during backup\n"
5378             if $is_backing_up && $includestate;
5379
5380         if ($includestate) {
5381             $conf->{lock} = 'suspending';
5382             my $date = strftime("%Y-%m-%d", localtime(time()));
5383             $storecfg = PVE::Storage::config();
5384             if (!$statestorage) {
5385                 $statestorage = find_vmstate_storage($conf, $storecfg);
5386                 # check permissions for the storage
5387                 my $rpcenv = PVE::RPCEnvironment::get();
5388                 if ($rpcenv->{type} ne 'cli') {
5389                     my $authuser = $rpcenv->get_user();
5390                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5391                 }
5392             }
5393
5394
5395             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5396             $path = PVE::Storage::path($storecfg, $vmstate);
5397             PVE::QemuConfig->write_config($vmid, $conf);
5398         } else {
5399             mon_cmd($vmid, "stop");
5400         }
5401     });
5402
5403     if ($includestate) {
5404         # save vm state
5405         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5406
5407         eval {
5408             mon_cmd($vmid, "savevm-start", statefile => $path);
5409             for(;;) {
5410                 my $state = mon_cmd($vmid, "query-savevm");
5411                 if (!$state->{status}) {
5412                     die "savevm not active\n";
5413                 } elsif ($state->{status} eq 'active') {
5414                     sleep(1);
5415                     next;
5416                 } elsif ($state->{status} eq 'completed') {
5417                     print "State saved, quitting\n";
5418                     last;
5419                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5420                     die "query-savevm failed with error '$state->{error}'\n"
5421                 } else {
5422                     die "query-savevm returned status '$state->{status}'\n";
5423                 }
5424             }
5425         };
5426         my $err = $@;
5427
5428         PVE::QemuConfig->lock_config($vmid, sub {
5429             $conf = PVE::QemuConfig->load_config($vmid);
5430             if ($err) {
5431                 # cleanup, but leave suspending lock, to indicate something went wrong
5432                 eval {
5433                     mon_cmd($vmid, "savevm-end");
5434                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5435                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5436                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5437                     PVE::QemuConfig->write_config($vmid, $conf);
5438                 };
5439                 warn $@ if $@;
5440                 die $err;
5441             }
5442
5443             die "lock changed unexpectedly\n"
5444                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5445
5446             mon_cmd($vmid, "quit");
5447             $conf->{lock} = 'suspended';
5448             PVE::QemuConfig->write_config($vmid, $conf);
5449         });
5450     }
5451 }
5452
5453 sub vm_resume {
5454     my ($vmid, $skiplock, $nocheck) = @_;
5455
5456     PVE::QemuConfig->lock_config($vmid, sub {
5457         my $res = mon_cmd($vmid, 'query-status');
5458         my $resume_cmd = 'cont';
5459
5460         if ($res->{status} && $res->{status} eq 'suspended') {
5461             $resume_cmd = 'system_wakeup';
5462         }
5463
5464         if (!$nocheck) {
5465
5466             my $conf = PVE::QemuConfig->load_config($vmid);
5467
5468             PVE::QemuConfig->check_lock($conf)
5469                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5470         }
5471
5472         mon_cmd($vmid, $resume_cmd);
5473     });
5474 }
5475
5476 sub vm_sendkey {
5477     my ($vmid, $skiplock, $key) = @_;
5478
5479     PVE::QemuConfig->lock_config($vmid, sub {
5480
5481         my $conf = PVE::QemuConfig->load_config($vmid);
5482
5483         # there is no qmp command, so we use the human monitor command
5484         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5485         die $res if $res ne '';
5486     });
5487 }
5488
5489 # vzdump restore implementaion
5490
5491 sub tar_archive_read_firstfile {
5492     my $archive = shift;
5493
5494     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5495
5496     # try to detect archive type first
5497     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5498         die "unable to open file '$archive'\n";
5499     my $firstfile = <$fh>;
5500     kill 15, $pid;
5501     close $fh;
5502
5503     die "ERROR: archive contaions no data\n" if !$firstfile;
5504     chomp $firstfile;
5505
5506     return $firstfile;
5507 }
5508
5509 sub tar_restore_cleanup {
5510     my ($storecfg, $statfile) = @_;
5511
5512     print STDERR "starting cleanup\n";
5513
5514     if (my $fd = IO::File->new($statfile, "r")) {
5515         while (defined(my $line = <$fd>)) {
5516             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5517                 my $volid = $2;
5518                 eval {
5519                     if ($volid =~ m|^/|) {
5520                         unlink $volid || die 'unlink failed\n';
5521                     } else {
5522                         PVE::Storage::vdisk_free($storecfg, $volid);
5523                     }
5524                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5525                 };
5526                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5527             } else {
5528                 print STDERR "unable to parse line in statfile - $line";
5529             }
5530         }
5531         $fd->close();
5532     }
5533 }
5534
5535 sub restore_file_archive {
5536     my ($archive, $vmid, $user, $opts) = @_;
5537
5538     return restore_vma_archive($archive, $vmid, $user, $opts)
5539         if $archive eq '-';
5540
5541     my $info = PVE::Storage::archive_info($archive);
5542     my $format = $opts->{format} // $info->{format};
5543     my $comp = $info->{compression};
5544
5545     # try to detect archive format
5546     if ($format eq 'tar') {
5547         return restore_tar_archive($archive, $vmid, $user, $opts);
5548     } else {
5549         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5550     }
5551 }
5552
5553 # hepler to remove disks that will not be used after restore
5554 my $restore_cleanup_oldconf = sub {
5555     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5556
5557     PVE::QemuConfig->foreach_volume($oldconf, sub {
5558         my ($ds, $drive) = @_;
5559
5560         return if drive_is_cdrom($drive, 1);
5561
5562         my $volid = $drive->{file};
5563         return if !$volid || $volid =~ m|^/|;
5564
5565         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5566         return if !$path || !$owner || ($owner != $vmid);
5567
5568         # Note: only delete disk we want to restore
5569         # other volumes will become unused
5570         if ($virtdev_hash->{$ds}) {
5571             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5572             if (my $err = $@) {
5573                 warn $err;
5574             }
5575         }
5576     });
5577
5578     # delete vmstate files, after the restore we have no snapshots anymore
5579     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5580         my $snap = $oldconf->{snapshots}->{$snapname};
5581         if ($snap->{vmstate}) {
5582             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5583             if (my $err = $@) {
5584                 warn $err;
5585             }
5586         }
5587     }
5588 };
5589
5590 # Helper to parse vzdump backup device hints
5591 #
5592 # $rpcenv: Environment, used to ckeck storage permissions
5593 # $user: User ID, to check storage permissions
5594 # $storecfg: Storage configuration
5595 # $fh: the file handle for reading the configuration
5596 # $devinfo: should contain device sizes for all backu-up'ed devices
5597 # $options: backup options (pool, default storage)
5598 #
5599 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5600 my $parse_backup_hints = sub {
5601     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5602
5603     my $virtdev_hash = {};
5604
5605     while (defined(my $line = <$fh>)) {
5606         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5607             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5608             die "archive does not contain data for drive '$virtdev'\n"
5609                 if !$devinfo->{$devname};
5610
5611             if (defined($options->{storage})) {
5612                 $storeid = $options->{storage} || 'local';
5613             } elsif (!$storeid) {
5614                 $storeid = 'local';
5615             }
5616             $format = 'raw' if !$format;
5617             $devinfo->{$devname}->{devname} = $devname;
5618             $devinfo->{$devname}->{virtdev} = $virtdev;
5619             $devinfo->{$devname}->{format} = $format;
5620             $devinfo->{$devname}->{storeid} = $storeid;
5621
5622             # check permission on storage
5623             my $pool = $options->{pool}; # todo: do we need that?
5624             if ($user ne 'root@pam') {
5625                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5626             }
5627
5628             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5629         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5630             my $virtdev = $1;
5631             my $drive = parse_drive($virtdev, $2);
5632             if (drive_is_cloudinit($drive)) {
5633                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5634                 $storeid = $options->{storage} if defined ($options->{storage});
5635                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5636                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5637
5638                 $virtdev_hash->{$virtdev} = {
5639                     format => $format,
5640                     storeid => $storeid,
5641                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5642                     is_cloudinit => 1,
5643                 };
5644             }
5645         }
5646     }
5647
5648     return $virtdev_hash;
5649 };
5650
5651 # Helper to allocate and activate all volumes required for a restore
5652 #
5653 # $storecfg: Storage configuration
5654 # $virtdev_hash: as returned by parse_backup_hints()
5655 #
5656 # Returns: { $virtdev => $volid }
5657 my $restore_allocate_devices = sub {
5658     my ($storecfg, $virtdev_hash, $vmid) = @_;
5659
5660     my $map = {};
5661     foreach my $virtdev (sort keys %$virtdev_hash) {
5662         my $d = $virtdev_hash->{$virtdev};
5663         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5664         my $storeid = $d->{storeid};
5665         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5666
5667         # test if requested format is supported
5668         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5669         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5670         $d->{format} = $defFormat if !$supported;
5671
5672         my $name;
5673         if ($d->{is_cloudinit}) {
5674             $name = "vm-$vmid-cloudinit";
5675             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5676         }
5677
5678         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5679
5680         print STDERR "new volume ID is '$volid'\n";
5681         $d->{volid} = $volid;
5682
5683         PVE::Storage::activate_volumes($storecfg, [$volid]);
5684
5685         $map->{$virtdev} = $volid;
5686     }
5687
5688     return $map;
5689 };
5690
5691 my $restore_update_config_line = sub {
5692     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5693
5694     return if $line =~ m/^\#qmdump\#/;
5695     return if $line =~ m/^\#vzdump\#/;
5696     return if $line =~ m/^lock:/;
5697     return if $line =~ m/^unused\d+:/;
5698     return if $line =~ m/^parent:/;
5699
5700     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5701     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5702         # try to convert old 1.X settings
5703         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5704         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5705             my ($model, $macaddr) = split(/\=/, $devconfig);
5706             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5707             my $net = {
5708                 model => $model,
5709                 bridge => "vmbr$ind",
5710                 macaddr => $macaddr,
5711             };
5712             my $netstr = print_net($net);
5713
5714             print $outfd "net$cookie->{netcount}: $netstr\n";
5715             $cookie->{netcount}++;
5716         }
5717     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5718         my ($id, $netstr) = ($1, $2);
5719         my $net = parse_net($netstr);
5720         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5721         $netstr = print_net($net);
5722         print $outfd "$id: $netstr\n";
5723     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5724         my $virtdev = $1;
5725         my $value = $3;
5726         my $di = parse_drive($virtdev, $value);
5727         if (defined($di->{backup}) && !$di->{backup}) {
5728             print $outfd "#$line";
5729         } elsif ($map->{$virtdev}) {
5730             delete $di->{format}; # format can change on restore
5731             $di->{file} = $map->{$virtdev};
5732             $value = print_drive($di);
5733             print $outfd "$virtdev: $value\n";
5734         } else {
5735             print $outfd $line;
5736         }
5737     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5738         my $vmgenid = $1;
5739         if ($vmgenid ne '0') {
5740             # always generate a new vmgenid if there was a valid one setup
5741             $vmgenid = generate_uuid();
5742         }
5743         print $outfd "vmgenid: $vmgenid\n";
5744     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5745         my ($uuid, $uuid_str);
5746         UUID::generate($uuid);
5747         UUID::unparse($uuid, $uuid_str);
5748         my $smbios1 = parse_smbios1($2);
5749         $smbios1->{uuid} = $uuid_str;
5750         print $outfd $1.print_smbios1($smbios1)."\n";
5751     } else {
5752         print $outfd $line;
5753     }
5754 };
5755
5756 my $restore_deactivate_volumes = sub {
5757     my ($storecfg, $devinfo) = @_;
5758
5759     my $vollist = [];
5760     foreach my $devname (keys %$devinfo) {
5761         my $volid = $devinfo->{$devname}->{volid};
5762         push @$vollist, $volid if $volid;
5763     }
5764
5765     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5766 };
5767
5768 my $restore_destroy_volumes = sub {
5769     my ($storecfg, $devinfo) = @_;
5770
5771     foreach my $devname (keys %$devinfo) {
5772         my $volid = $devinfo->{$devname}->{volid};
5773         next if !$volid;
5774         eval {
5775             if ($volid =~ m|^/|) {
5776                 unlink $volid || die 'unlink failed\n';
5777             } else {
5778                 PVE::Storage::vdisk_free($storecfg, $volid);
5779             }
5780             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5781         };
5782         print STDERR "unable to cleanup '$volid' - $@" if $@;
5783     }
5784 };
5785
5786 sub scan_volids {
5787     my ($cfg, $vmid) = @_;
5788
5789     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5790
5791     my $volid_hash = {};
5792     foreach my $storeid (keys %$info) {
5793         foreach my $item (@{$info->{$storeid}}) {
5794             next if !($item->{volid} && $item->{size});
5795             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5796             $volid_hash->{$item->{volid}} = $item;
5797         }
5798     }
5799
5800     return $volid_hash;
5801 }
5802
5803 sub update_disk_config {
5804     my ($vmid, $conf, $volid_hash) = @_;
5805
5806     my $changes;
5807     my $prefix = "VM $vmid:";
5808
5809     # used and unused disks
5810     my $referenced = {};
5811
5812     # Note: it is allowed to define multiple storages with same path (alias), so
5813     # we need to check both 'volid' and real 'path' (two different volid can point
5814     # to the same path).
5815
5816     my $referencedpath = {};
5817
5818     # update size info
5819     PVE::QemuConfig->foreach_volume($conf, sub {
5820         my ($opt, $drive) = @_;
5821
5822         my $volid = $drive->{file};
5823         return if !$volid;
5824
5825         # mark volid as "in-use" for next step
5826         $referenced->{$volid} = 1;
5827         if ($volid_hash->{$volid} &&
5828             (my $path = $volid_hash->{$volid}->{path})) {
5829             $referencedpath->{$path} = 1;
5830         }
5831
5832         return if drive_is_cdrom($drive);
5833         return if !$volid_hash->{$volid};
5834
5835         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5836         if (defined($updated)) {
5837             $changes = 1;
5838             $conf->{$opt} = print_drive($updated);
5839             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5840         }
5841     });
5842
5843     # remove 'unusedX' entry if volume is used
5844     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5845         my ($opt, $drive) = @_;
5846
5847         my $volid = $drive->{file};
5848         return if !$volid;
5849
5850         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5851         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5852             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5853             $changes = 1;
5854             delete $conf->{$opt};
5855         }
5856
5857         $referenced->{$volid} = 1;
5858         $referencedpath->{$path} = 1 if $path;
5859     });
5860
5861     foreach my $volid (sort keys %$volid_hash) {
5862         next if $volid =~ m/vm-$vmid-state-/;
5863         next if $referenced->{$volid};
5864         my $path = $volid_hash->{$volid}->{path};
5865         next if !$path; # just to be sure
5866         next if $referencedpath->{$path};
5867         $changes = 1;
5868         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5869         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5870         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5871     }
5872
5873     return $changes;
5874 }
5875
5876 sub rescan {
5877     my ($vmid, $nolock, $dryrun) = @_;
5878
5879     my $cfg = PVE::Storage::config();
5880
5881     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5882     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5883     foreach my $stor (keys %{$cfg->{ids}}) {
5884         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5885     }
5886
5887     print "rescan volumes...\n";
5888     my $volid_hash = scan_volids($cfg, $vmid);
5889
5890     my $updatefn =  sub {
5891         my ($vmid) = @_;
5892
5893         my $conf = PVE::QemuConfig->load_config($vmid);
5894
5895         PVE::QemuConfig->check_lock($conf);
5896
5897         my $vm_volids = {};
5898         foreach my $volid (keys %$volid_hash) {
5899             my $info = $volid_hash->{$volid};
5900             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5901         }
5902
5903         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5904
5905         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5906     };
5907
5908     if (defined($vmid)) {
5909         if ($nolock) {
5910             &$updatefn($vmid);
5911         } else {
5912             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5913         }
5914     } else {
5915         my $vmlist = config_list();
5916         foreach my $vmid (keys %$vmlist) {
5917             if ($nolock) {
5918                 &$updatefn($vmid);
5919             } else {
5920                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5921             }
5922         }
5923     }
5924 }
5925
5926 sub restore_proxmox_backup_archive {
5927     my ($archive, $vmid, $user, $options) = @_;
5928
5929     my $storecfg = PVE::Storage::config();
5930
5931     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5932     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5933
5934     my $server = $scfg->{server};
5935     my $datastore = $scfg->{datastore};
5936     my $username = $scfg->{username} // 'root@pam';
5937     my $fingerprint = $scfg->{fingerprint};
5938
5939     my $repo = "$username\@$server:$datastore";
5940     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5941     local $ENV{PBS_PASSWORD} = $password;
5942     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5943
5944     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5945         PVE::Storage::parse_volname($storecfg, $archive);
5946
5947     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5948
5949     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5950
5951     my $tmpdir = "/var/tmp/vzdumptmp$$";
5952     rmtree $tmpdir;
5953     mkpath $tmpdir;
5954
5955     my $conffile = PVE::QemuConfig->config_file($vmid);
5956     my $tmpfn = "$conffile.$$.tmp";
5957      # disable interrupts (always do cleanups)
5958     local $SIG{INT} =
5959         local $SIG{TERM} =
5960         local $SIG{QUIT} =
5961         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5962
5963     # Note: $oldconf is undef if VM does not exists
5964     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5965     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5966
5967     my $rpcenv = PVE::RPCEnvironment::get();
5968     my $devinfo = {};
5969
5970     eval {
5971         # enable interrupts
5972         local $SIG{INT} =
5973             local $SIG{TERM} =
5974             local $SIG{QUIT} =
5975             local $SIG{HUP} =
5976             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5977
5978         my $cfgfn = "$tmpdir/qemu-server.conf";
5979         my $firewall_config_fn = "$tmpdir/fw.conf";
5980         my $index_fn = "$tmpdir/index.json";
5981
5982         my $cmd = "restore";
5983
5984         my $param = [$pbs_backup_name, "index.json", $index_fn];
5985         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5986         my $index = PVE::Tools::file_get_contents($index_fn);
5987         $index = decode_json($index);
5988
5989         # print Dumper($index);
5990         foreach my $info (@{$index->{files}}) {
5991             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
5992                 my $devname = $1;
5993                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
5994                     $devinfo->{$devname}->{size} = $1;
5995                 } else {
5996                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
5997                 }
5998             }
5999         }
6000
6001         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6002         if (!$is_qemu_server_backup) {
6003             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6004         }
6005         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6006
6007         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6008         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6009
6010         if ($has_firewall_config) {
6011             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6012             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6013
6014             my $pve_firewall_dir = '/etc/pve/firewall';
6015             mkdir $pve_firewall_dir; # make sure the dir exists
6016             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6017         }
6018
6019         my $fh = IO::File->new($cfgfn, "r") ||
6020             "unable to read qemu-server.conf - $!\n";
6021
6022         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6023
6024         # fixme: rate limit?
6025
6026         # create empty/temp config
6027         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6028
6029         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6030
6031         # allocate volumes
6032         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6033
6034         foreach my $virtdev (sort keys %$virtdev_hash) {
6035             my $d = $virtdev_hash->{$virtdev};
6036             next if $d->{is_cloudinit}; # no need to restore cloudinit
6037
6038             my $volid = $d->{volid};
6039
6040             my $path = PVE::Storage::path($storecfg, $volid);
6041
6042             my $pbs_restore_cmd = [
6043                 '/usr/bin/pbs-restore',
6044                 '--repository', $repo,
6045                 $pbs_backup_name,
6046                 "$d->{devname}.img.fidx",
6047                 $path,
6048                 '--verbose',
6049                 ];
6050
6051             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6052
6053             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6054                 push @$pbs_restore_cmd, '--skip-zero';
6055             }
6056
6057             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6058             print "restore proxmox backup image: $dbg_cmdstring\n";
6059             run_command($pbs_restore_cmd);
6060         }
6061
6062         $fh->seek(0, 0) || die "seek failed - $!\n";
6063
6064         my $outfd = new IO::File ($tmpfn, "w") ||
6065             die "unable to write config for VM $vmid\n";
6066
6067         my $cookie = { netcount => 0 };
6068         while (defined(my $line = <$fh>)) {
6069             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6070         }
6071
6072         $fh->close();
6073         $outfd->close();
6074     };
6075     my $err = $@;
6076
6077     $restore_deactivate_volumes->($storecfg, $devinfo);
6078
6079     rmtree $tmpdir;
6080
6081     if ($err) {
6082         unlink $tmpfn;
6083         $restore_destroy_volumes->($storecfg, $devinfo);
6084         die $err;
6085     }
6086
6087     rename($tmpfn, $conffile) ||
6088         die "unable to commit configuration file '$conffile'\n";
6089
6090     PVE::Cluster::cfs_update(); # make sure we read new file
6091
6092     eval { rescan($vmid, 1); };
6093     warn $@ if $@;
6094 }
6095
6096 sub restore_vma_archive {
6097     my ($archive, $vmid, $user, $opts, $comp) = @_;
6098
6099     my $readfrom = $archive;
6100
6101     my $cfg = PVE::Storage::config();
6102     my $commands = [];
6103     my $bwlimit = $opts->{bwlimit};
6104
6105     my $dbg_cmdstring = '';
6106     my $add_pipe = sub {
6107         my ($cmd) = @_;
6108         push @$commands, $cmd;
6109         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6110         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6111         $readfrom = '-';
6112     };
6113
6114     my $input = undef;
6115     if ($archive eq '-') {
6116         $input = '<&STDIN';
6117     } else {
6118         # If we use a backup from a PVE defined storage we also consider that
6119         # storage's rate limit:
6120         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6121         if (defined($volid)) {
6122             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6123             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6124             if ($readlimit) {
6125                 print STDERR "applying read rate limit: $readlimit\n";
6126                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6127                 $add_pipe->($cstream);
6128             }
6129         }
6130     }
6131
6132     if ($comp) {
6133         my $info = PVE::Storage::decompressor_info('vma', $comp);
6134         my $cmd = $info->{decompressor};
6135         push @$cmd, $readfrom;
6136         $add_pipe->($cmd);
6137     }
6138
6139     my $tmpdir = "/var/tmp/vzdumptmp$$";
6140     rmtree $tmpdir;
6141
6142     # disable interrupts (always do cleanups)
6143     local $SIG{INT} =
6144         local $SIG{TERM} =
6145         local $SIG{QUIT} =
6146         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6147
6148     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6149     POSIX::mkfifo($mapfifo, 0600);
6150     my $fifofh;
6151
6152     my $openfifo = sub {
6153         open($fifofh, '>', $mapfifo) || die $!;
6154     };
6155
6156     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6157
6158     my $oldtimeout;
6159     my $timeout = 5;
6160
6161     my $devinfo = {};
6162
6163     my $rpcenv = PVE::RPCEnvironment::get();
6164
6165     my $conffile = PVE::QemuConfig->config_file($vmid);
6166     my $tmpfn = "$conffile.$$.tmp";
6167
6168     # Note: $oldconf is undef if VM does not exist
6169     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6170     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6171
6172     my %storage_limits;
6173
6174     my $print_devmap = sub {
6175         my $cfgfn = "$tmpdir/qemu-server.conf";
6176
6177         # we can read the config - that is already extracted
6178         my $fh = IO::File->new($cfgfn, "r") ||
6179             "unable to read qemu-server.conf - $!\n";
6180
6181         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6182         if (-f $fwcfgfn) {
6183             my $pve_firewall_dir = '/etc/pve/firewall';
6184             mkdir $pve_firewall_dir; # make sure the dir exists
6185             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6186         }
6187
6188         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6189
6190         foreach my $key (keys %storage_limits) {
6191             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6192             next if !$limit;
6193             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6194             $storage_limits{$key} = $limit * 1024;
6195         }
6196
6197         foreach my $devname (keys %$devinfo) {
6198             die "found no device mapping information for device '$devname'\n"
6199                 if !$devinfo->{$devname}->{virtdev};
6200         }
6201
6202         # create empty/temp config
6203         if ($oldconf) {
6204             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6205             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6206         }
6207
6208         # allocate volumes
6209         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6210
6211         # print restore information to $fifofh
6212         foreach my $virtdev (sort keys %$virtdev_hash) {
6213             my $d = $virtdev_hash->{$virtdev};
6214             next if $d->{is_cloudinit}; # no need to restore cloudinit
6215
6216             my $storeid = $d->{storeid};
6217             my $volid = $d->{volid};
6218
6219             my $map_opts = '';
6220             if (my $limit = $storage_limits{$storeid}) {
6221                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6222             }
6223
6224             my $write_zeros = 1;
6225             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6226                 $write_zeros = 0;
6227             }
6228
6229             my $path = PVE::Storage::path($cfg, $volid);
6230
6231             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6232
6233             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6234         }
6235
6236         $fh->seek(0, 0) || die "seek failed - $!\n";
6237
6238         my $outfd = new IO::File ($tmpfn, "w") ||
6239             die "unable to write config for VM $vmid\n";
6240
6241         my $cookie = { netcount => 0 };
6242         while (defined(my $line = <$fh>)) {
6243             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6244         }
6245
6246         $fh->close();
6247         $outfd->close();
6248     };
6249
6250     eval {
6251         # enable interrupts
6252         local $SIG{INT} =
6253             local $SIG{TERM} =
6254             local $SIG{QUIT} =
6255             local $SIG{HUP} =
6256             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6257         local $SIG{ALRM} = sub { die "got timeout\n"; };
6258
6259         $oldtimeout = alarm($timeout);
6260
6261         my $parser = sub {
6262             my $line = shift;
6263
6264             print "$line\n";
6265
6266             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6267                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6268                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6269             } elsif ($line =~ m/^CTIME: /) {
6270                 # we correctly received the vma config, so we can disable
6271                 # the timeout now for disk allocation (set to 10 minutes, so
6272                 # that we always timeout if something goes wrong)
6273                 alarm(600);
6274                 &$print_devmap();
6275                 print $fifofh "done\n";
6276                 my $tmp = $oldtimeout || 0;
6277                 $oldtimeout = undef;
6278                 alarm($tmp);
6279                 close($fifofh);
6280             }
6281         };
6282
6283         print "restore vma archive: $dbg_cmdstring\n";
6284         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6285     };
6286     my $err = $@;
6287
6288     alarm($oldtimeout) if $oldtimeout;
6289
6290     $restore_deactivate_volumes->($cfg, $devinfo);
6291
6292     unlink $mapfifo;
6293     rmtree $tmpdir;
6294
6295     if ($err) {
6296         unlink $tmpfn;
6297         $restore_destroy_volumes->($cfg, $devinfo);
6298         die $err;
6299     }
6300
6301     rename($tmpfn, $conffile) ||
6302         die "unable to commit configuration file '$conffile'\n";
6303
6304     PVE::Cluster::cfs_update(); # make sure we read new file
6305
6306     eval { rescan($vmid, 1); };
6307     warn $@ if $@;
6308 }
6309
6310 sub restore_tar_archive {
6311     my ($archive, $vmid, $user, $opts) = @_;
6312
6313     if ($archive ne '-') {
6314         my $firstfile = tar_archive_read_firstfile($archive);
6315         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6316             if $firstfile ne 'qemu-server.conf';
6317     }
6318
6319     my $storecfg = PVE::Storage::config();
6320
6321     # avoid zombie disks when restoring over an existing VM -> cleanup first
6322     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6323     # skiplock=1 because qmrestore has set the 'create' lock itself already
6324     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6325     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6326
6327     my $tocmd = "/usr/lib/qemu-server/qmextract";
6328
6329     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6330     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6331     $tocmd .= ' --prealloc' if $opts->{prealloc};
6332     $tocmd .= ' --info' if $opts->{info};
6333
6334     # tar option "xf" does not autodetect compression when read from STDIN,
6335     # so we pipe to zcat
6336     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6337         PVE::Tools::shellquote("--to-command=$tocmd");
6338
6339     my $tmpdir = "/var/tmp/vzdumptmp$$";
6340     mkpath $tmpdir;
6341
6342     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6343     local $ENV{VZDUMP_VMID} = $vmid;
6344     local $ENV{VZDUMP_USER} = $user;
6345
6346     my $conffile = PVE::QemuConfig->config_file($vmid);
6347     my $tmpfn = "$conffile.$$.tmp";
6348
6349     # disable interrupts (always do cleanups)
6350     local $SIG{INT} =
6351         local $SIG{TERM} =
6352         local $SIG{QUIT} =
6353         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6354
6355     eval {
6356         # enable interrupts
6357         local $SIG{INT} =
6358             local $SIG{TERM} =
6359             local $SIG{QUIT} =
6360             local $SIG{HUP} =
6361             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6362
6363         if ($archive eq '-') {
6364             print "extracting archive from STDIN\n";
6365             run_command($cmd, input => "<&STDIN");
6366         } else {
6367             print "extracting archive '$archive'\n";
6368             run_command($cmd);
6369         }
6370
6371         return if $opts->{info};
6372
6373         # read new mapping
6374         my $map = {};
6375         my $statfile = "$tmpdir/qmrestore.stat";
6376         if (my $fd = IO::File->new($statfile, "r")) {
6377             while (defined (my $line = <$fd>)) {
6378                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6379                     $map->{$1} = $2 if $1;
6380                 } else {
6381                     print STDERR "unable to parse line in statfile - $line\n";
6382                 }
6383             }
6384             $fd->close();
6385         }
6386
6387         my $confsrc = "$tmpdir/qemu-server.conf";
6388
6389         my $srcfd = new IO::File($confsrc, "r") ||
6390             die "unable to open file '$confsrc'\n";
6391
6392         my $outfd = new IO::File ($tmpfn, "w") ||
6393             die "unable to write config for VM $vmid\n";
6394
6395         my $cookie = { netcount => 0 };
6396         while (defined (my $line = <$srcfd>)) {
6397             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6398         }
6399
6400         $srcfd->close();
6401         $outfd->close();
6402     };
6403     if (my $err = $@) {
6404         unlink $tmpfn;
6405         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6406         die $err;
6407     }
6408
6409     rmtree $tmpdir;
6410
6411     rename $tmpfn, $conffile ||
6412         die "unable to commit configuration file '$conffile'\n";
6413
6414     PVE::Cluster::cfs_update(); # make sure we read new file
6415
6416     eval { rescan($vmid, 1); };
6417     warn $@ if $@;
6418 };
6419
6420 sub foreach_storage_used_by_vm {
6421     my ($conf, $func) = @_;
6422
6423     my $sidhash = {};
6424
6425     PVE::QemuConfig->foreach_volume($conf, sub {
6426         my ($ds, $drive) = @_;
6427         return if drive_is_cdrom($drive);
6428
6429         my $volid = $drive->{file};
6430
6431         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6432         $sidhash->{$sid} = $sid if $sid;
6433     });
6434
6435     foreach my $sid (sort keys %$sidhash) {
6436         &$func($sid);
6437     }
6438 }
6439
6440 my $qemu_snap_storage = {
6441     rbd => 1,
6442 };
6443 sub do_snapshots_with_qemu {
6444     my ($storecfg, $volid) = @_;
6445
6446     my $storage_name = PVE::Storage::parse_volume_id($volid);
6447     my $scfg = $storecfg->{ids}->{$storage_name};
6448
6449     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6450         return 1;
6451     }
6452
6453     if ($volid =~ m/\.(qcow2|qed)$/){
6454         return 1;
6455     }
6456
6457     return undef;
6458 }
6459
6460 sub qga_check_running {
6461     my ($vmid, $nowarn) = @_;
6462
6463     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6464     if ($@) {
6465         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6466         return 0;
6467     }
6468     return 1;
6469 }
6470
6471 sub template_create {
6472     my ($vmid, $conf, $disk) = @_;
6473
6474     my $storecfg = PVE::Storage::config();
6475
6476     PVE::QemuConfig->foreach_volume($conf, sub {
6477         my ($ds, $drive) = @_;
6478
6479         return if drive_is_cdrom($drive);
6480         return if $disk && $ds ne $disk;
6481
6482         my $volid = $drive->{file};
6483         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6484
6485         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6486         $drive->{file} = $voliddst;
6487         $conf->{$ds} = print_drive($drive);
6488         PVE::QemuConfig->write_config($vmid, $conf);
6489     });
6490 }
6491
6492 sub convert_iscsi_path {
6493     my ($path) = @_;
6494
6495     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6496         my $portal = $1;
6497         my $target = $2;
6498         my $lun = $3;
6499
6500         my $initiator_name = get_initiator_name();
6501
6502         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6503                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6504     }
6505
6506     die "cannot convert iscsi path '$path', unkown format\n";
6507 }
6508
6509 sub qemu_img_convert {
6510     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6511
6512     my $storecfg = PVE::Storage::config();
6513     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6514     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6515
6516     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6517
6518     my $cachemode;
6519     my $src_path;
6520     my $src_is_iscsi = 0;
6521     my $src_format;
6522
6523     if ($src_storeid) {
6524         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6525         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6526         $src_format = qemu_img_format($src_scfg, $src_volname);
6527         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6528         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6529         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6530     } elsif (-f $src_volid) {
6531         $src_path = $src_volid;
6532         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6533             $src_format = $1;
6534         }
6535     }
6536
6537     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6538
6539     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6540     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6541     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6542     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6543
6544     my $cmd = [];
6545     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6546     push @$cmd, '-l', "snapshot.name=$snapname"
6547         if $snapname && $src_format && $src_format eq "qcow2";
6548     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6549     push @$cmd, '-T', $cachemode if defined($cachemode);
6550
6551     if ($src_is_iscsi) {
6552         push @$cmd, '--image-opts';
6553         $src_path = convert_iscsi_path($src_path);
6554     } elsif ($src_format) {
6555         push @$cmd, '-f', $src_format;
6556     }
6557
6558     if ($dst_is_iscsi) {
6559         push @$cmd, '--target-image-opts';
6560         $dst_path = convert_iscsi_path($dst_path);
6561     } else {
6562         push @$cmd, '-O', $dst_format;
6563     }
6564
6565     push @$cmd, $src_path;
6566
6567     if (!$dst_is_iscsi && $is_zero_initialized) {
6568         push @$cmd, "zeroinit:$dst_path";
6569     } else {
6570         push @$cmd, $dst_path;
6571     }
6572
6573     my $parser = sub {
6574         my $line = shift;
6575         if($line =~ m/\((\S+)\/100\%\)/){
6576             my $percent = $1;
6577             my $transferred = int($size * $percent / 100);
6578             my $remaining = $size - $transferred;
6579
6580             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6581         }
6582
6583     };
6584
6585     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6586     my $err = $@;
6587     die "copy failed: $err" if $err;
6588 }
6589
6590 sub qemu_img_format {
6591     my ($scfg, $volname) = @_;
6592
6593     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6594         return $1;
6595     } else {
6596         return "raw";
6597     }
6598 }
6599
6600 sub qemu_drive_mirror {
6601     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6602
6603     $jobs = {} if !$jobs;
6604
6605     my $qemu_target;
6606     my $format;
6607     $jobs->{"drive-$drive"} = {};
6608
6609     if ($dst_volid =~ /^nbd:/) {
6610         $qemu_target = $dst_volid;
6611         $format = "nbd";
6612     } else {
6613         my $storecfg = PVE::Storage::config();
6614         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6615
6616         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6617
6618         $format = qemu_img_format($dst_scfg, $dst_volname);
6619
6620         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6621
6622         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6623     }
6624
6625     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6626     $opts->{format} = $format if $format;
6627
6628     if (defined($src_bitmap)) {
6629         $opts->{sync} = 'incremental';
6630         $opts->{bitmap} = $src_bitmap;
6631         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6632     }
6633
6634     if (defined($bwlimit)) {
6635         $opts->{speed} = $bwlimit * 1024;
6636         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6637     } else {
6638         print "drive mirror is starting for drive-$drive\n";
6639     }
6640
6641     # if a job already runs for this device we get an error, catch it for cleanup
6642     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6643     if (my $err = $@) {
6644         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6645         warn "$@\n" if $@;
6646         die "mirroring error: $err\n";
6647     }
6648
6649     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6650 }
6651
6652 # $completion can be either
6653 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6654 # 'cancel': wait until all jobs are ready, block-job-cancel them
6655 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6656 sub qemu_drive_mirror_monitor {
6657     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6658
6659     $completion //= 'complete';
6660
6661     eval {
6662         my $err_complete = 0;
6663
6664         while (1) {
6665             die "storage migration timed out\n" if $err_complete > 300;
6666
6667             my $stats = mon_cmd($vmid, "query-block-jobs");
6668
6669             my $running_mirror_jobs = {};
6670             foreach my $stat (@$stats) {
6671                 next if $stat->{type} ne 'mirror';
6672                 $running_mirror_jobs->{$stat->{device}} = $stat;
6673             }
6674
6675             my $readycounter = 0;
6676
6677             foreach my $job (keys %$jobs) {
6678
6679                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6680                     print "$job : finished\n";
6681                     delete $jobs->{$job};
6682                     next;
6683                 }
6684
6685                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6686
6687                 my $busy = $running_mirror_jobs->{$job}->{busy};
6688                 my $ready = $running_mirror_jobs->{$job}->{ready};
6689                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6690                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6691                     my $remaining = $total - $transferred;
6692                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6693
6694                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6695                 }
6696
6697                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6698             }
6699
6700             last if scalar(keys %$jobs) == 0;
6701
6702             if ($readycounter == scalar(keys %$jobs)) {
6703                 print "all mirroring jobs are ready \n";
6704                 last if $completion eq 'skip'; #do the complete later
6705
6706                 if ($vmiddst && $vmiddst != $vmid) {
6707                     my $agent_running = $qga && qga_check_running($vmid);
6708                     if ($agent_running) {
6709                         print "freeze filesystem\n";
6710                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6711                     } else {
6712                         print "suspend vm\n";
6713                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6714                     }
6715
6716                     # if we clone a disk for a new target vm, we don't switch the disk
6717                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6718
6719                     if ($agent_running) {
6720                         print "unfreeze filesystem\n";
6721                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6722                     } else {
6723                         print "resume vm\n";
6724                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6725                     }
6726
6727                     last;
6728                 } else {
6729
6730                     foreach my $job (keys %$jobs) {
6731                         # try to switch the disk if source and destination are on the same guest
6732                         print "$job: Completing block job...\n";
6733
6734                         my $op;
6735                         if ($completion eq 'complete') {
6736                             $op = 'block-job-complete';
6737                         } elsif ($completion eq 'cancel') {
6738                             $op = 'block-job-cancel';
6739                         } else {
6740                             die "invalid completion value: $completion\n";
6741                         }
6742                         eval { mon_cmd($vmid, $op, device => $job) };
6743                         if ($@ =~ m/cannot be completed/) {
6744                             print "$job: Block job cannot be completed, try again.\n";
6745                             $err_complete++;
6746                         }else {
6747                             print "$job: Completed successfully.\n";
6748                             $jobs->{$job}->{complete} = 1;
6749                         }
6750                     }
6751                 }
6752             }
6753             sleep 1;
6754         }
6755     };
6756     my $err = $@;
6757
6758     if ($err) {
6759         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6760         die "mirroring error: $err";
6761     }
6762
6763 }
6764
6765 sub qemu_blockjobs_cancel {
6766     my ($vmid, $jobs) = @_;
6767
6768     foreach my $job (keys %$jobs) {
6769         print "$job: Cancelling block job\n";
6770         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6771         $jobs->{$job}->{cancel} = 1;
6772     }
6773
6774     while (1) {
6775         my $stats = mon_cmd($vmid, "query-block-jobs");
6776
6777         my $running_jobs = {};
6778         foreach my $stat (@$stats) {
6779             $running_jobs->{$stat->{device}} = $stat;
6780         }
6781
6782         foreach my $job (keys %$jobs) {
6783
6784             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6785                 print "$job: Done.\n";
6786                 delete $jobs->{$job};
6787             }
6788         }
6789
6790         last if scalar(keys %$jobs) == 0;
6791
6792         sleep 1;
6793     }
6794 }
6795
6796 sub clone_disk {
6797     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6798         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6799
6800     my $newvolid;
6801
6802     if (!$full) {
6803         print "create linked clone of drive $drivename ($drive->{file})\n";
6804         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6805         push @$newvollist, $newvolid;
6806     } else {
6807
6808         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6809         $storeid = $storage if $storage;
6810
6811         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6812         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6813
6814         print "create full clone of drive $drivename ($drive->{file})\n";
6815         my $name = undef;
6816         if (drive_is_cloudinit($drive)) {
6817             $name = "vm-$newvmid-cloudinit";
6818             $name .= ".$dst_format" if $dst_format ne 'raw';
6819             $snapname = undef;
6820             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6821         } elsif ($drivename eq 'efidisk0') {
6822             $size = get_efivars_size($conf);
6823         }
6824         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6825         push @$newvollist, $newvolid;
6826
6827         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6828
6829         if (drive_is_cloudinit($drive)) {
6830             goto no_data_clone;
6831         }
6832
6833         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6834         if (!$running || $snapname) {
6835             # TODO: handle bwlimits
6836             if ($drivename eq 'efidisk0') {
6837                 # the relevant data on the efidisk may be smaller than the source
6838                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6839                 # that is given by the OVMF_VARS.fd
6840                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6841                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6842                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6843             } else {
6844                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6845             }
6846         } else {
6847
6848             my $kvmver = get_running_qemu_version ($vmid);
6849             if (!min_version($kvmver, 2, 7)) {
6850                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6851                     if $drive->{iothread};
6852             }
6853
6854             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6855         }
6856     }
6857
6858 no_data_clone:
6859     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6860
6861     my $disk = $drive;
6862     $disk->{format} = undef;
6863     $disk->{file} = $newvolid;
6864     $disk->{size} = $size;
6865
6866     return $disk;
6867 }
6868
6869 sub get_running_qemu_version {
6870     my ($vmid) = @_;
6871     my $res = mon_cmd($vmid, "query-version");
6872     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6873 }
6874
6875 sub qemu_use_old_bios_files {
6876     my ($machine_type) = @_;
6877
6878     return if !$machine_type;
6879
6880     my $use_old_bios_files = undef;
6881
6882     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6883         $machine_type = $1;
6884         $use_old_bios_files = 1;
6885     } else {
6886         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6887         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6888         # load new efi bios files on migration. So this hack is required to allow
6889         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6890         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6891         $use_old_bios_files = !min_version($version, 2, 4);
6892     }
6893
6894     return ($use_old_bios_files, $machine_type);
6895 }
6896
6897 sub get_efivars_size {
6898     my ($conf) = @_;
6899     my $arch = get_vm_arch($conf);
6900     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6901     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6902     return -s $ovmf_vars;
6903 }
6904
6905 sub update_efidisk_size {
6906     my ($conf) = @_;
6907
6908     return if !defined($conf->{efidisk0});
6909
6910     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6911     $disk->{size} = get_efivars_size($conf);
6912     $conf->{efidisk0} = print_drive($disk);
6913
6914     return;
6915 }
6916
6917 sub create_efidisk($$$$$) {
6918     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6919
6920     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6921     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6922
6923     my $vars_size_b = -s $ovmf_vars;
6924     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6925     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6926     PVE::Storage::activate_volumes($storecfg, [$volid]);
6927
6928     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6929     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6930
6931     return ($volid, $size/1024);
6932 }
6933
6934 sub vm_iothreads_list {
6935     my ($vmid) = @_;
6936
6937     my $res = mon_cmd($vmid, 'query-iothreads');
6938
6939     my $iothreads = {};
6940     foreach my $iothread (@$res) {
6941         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6942     }
6943
6944     return $iothreads;
6945 }
6946
6947 sub scsihw_infos {
6948     my ($conf, $drive) = @_;
6949
6950     my $maxdev = 0;
6951
6952     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6953         $maxdev = 7;
6954     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6955         $maxdev = 1;
6956     } else {
6957         $maxdev = 256;
6958     }
6959
6960     my $controller = int($drive->{index} / $maxdev);
6961     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6962
6963     return ($maxdev, $controller, $controller_prefix);
6964 }
6965
6966 sub windows_version {
6967     my ($ostype) = @_;
6968
6969     return 0 if !$ostype;
6970
6971     my $winversion = 0;
6972
6973     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6974         $winversion = 5;
6975     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6976         $winversion = 6;
6977     } elsif ($ostype =~ m/^win(\d+)$/) {
6978         $winversion = $1;
6979     }
6980
6981     return $winversion;
6982 }
6983
6984 sub resolve_dst_disk_format {
6985         my ($storecfg, $storeid, $src_volname, $format) = @_;
6986         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6987
6988         if (!$format) {
6989             # if no target format is specified, use the source disk format as hint
6990             if ($src_volname) {
6991                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6992                 $format = qemu_img_format($scfg, $src_volname);
6993             } else {
6994                 return $defFormat;
6995             }
6996         }
6997
6998         # test if requested format is supported - else use default
6999         my $supported = grep { $_ eq $format } @$validFormats;
7000         $format = $defFormat if !$supported;
7001         return $format;
7002 }
7003
7004 # NOTE: if this logic changes, please update docs & possibly gui logic
7005 sub find_vmstate_storage {
7006     my ($conf, $storecfg) = @_;
7007
7008     # first, return storage from conf if set
7009     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7010
7011     my ($target, $shared, $local);
7012
7013     foreach_storage_used_by_vm($conf, sub {
7014         my ($sid) = @_;
7015         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7016         my $dst = $scfg->{shared} ? \$shared : \$local;
7017         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7018     });
7019
7020     # second, use shared storage where VM has at least one disk
7021     # third, use local storage where VM has at least one disk
7022     # fall back to local storage
7023     $target = $shared // $local // 'local';
7024
7025     return $target;
7026 }
7027
7028 sub generate_uuid {
7029     my ($uuid, $uuid_str);
7030     UUID::generate($uuid);
7031     UUID::unparse($uuid, $uuid_str);
7032     return $uuid_str;
7033 }
7034
7035 sub generate_smbios1_uuid {
7036     return "uuid=".generate_uuid();
7037 }
7038
7039 sub nbd_stop {
7040     my ($vmid) = @_;
7041
7042     mon_cmd($vmid, 'nbd-server-stop');
7043 }
7044
7045 sub create_reboot_request {
7046     my ($vmid) = @_;
7047     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7048         or die "failed to create reboot trigger file: $!\n";
7049     close($fh);
7050 }
7051
7052 sub clear_reboot_request {
7053     my ($vmid) = @_;
7054     my $path = "/run/qemu-server/$vmid.reboot";
7055     my $res = 0;
7056
7057     $res = unlink($path);
7058     die "could not remove reboot request for $vmid: $!"
7059         if !$res && $! != POSIX::ENOENT;
7060
7061     return $res;
7062 }
7063
7064 # bash completion helper
7065
7066 sub complete_backup_archives {
7067     my ($cmdname, $pname, $cvalue) = @_;
7068
7069     my $cfg = PVE::Storage::config();
7070
7071     my $storeid;
7072
7073     if ($cvalue =~ m/^([^:]+):/) {
7074         $storeid = $1;
7075     }
7076
7077     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7078
7079     my $res = [];
7080     foreach my $id (keys %$data) {
7081         foreach my $item (@{$data->{$id}}) {
7082             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7083             push @$res, $item->{volid} if defined($item->{volid});
7084         }
7085     }
7086
7087     return $res;
7088 }
7089
7090 my $complete_vmid_full = sub {
7091     my ($running) = @_;
7092
7093     my $idlist = vmstatus();
7094
7095     my $res = [];
7096
7097     foreach my $id (keys %$idlist) {
7098         my $d = $idlist->{$id};
7099         if (defined($running)) {
7100             next if $d->{template};
7101             next if $running && $d->{status} ne 'running';
7102             next if !$running && $d->{status} eq 'running';
7103         }
7104         push @$res, $id;
7105
7106     }
7107     return $res;
7108 };
7109
7110 sub complete_vmid {
7111     return &$complete_vmid_full();
7112 }
7113
7114 sub complete_vmid_stopped {
7115     return &$complete_vmid_full(0);
7116 }
7117
7118 sub complete_vmid_running {
7119     return &$complete_vmid_full(1);
7120 }
7121
7122 sub complete_storage {
7123
7124     my $cfg = PVE::Storage::config();
7125     my $ids = $cfg->{ids};
7126
7127     my $res = [];
7128     foreach my $sid (keys %$ids) {
7129         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7130         next if !$ids->{$sid}->{content}->{images};
7131         push @$res, $sid;
7132     }
7133
7134     return $res;
7135 }
7136
7137 sub complete_migration_storage {
7138     my ($cmd, $param, $current_value, $all_args) = @_;
7139
7140     my $targetnode = @$all_args[1];
7141
7142     my $cfg = PVE::Storage::config();
7143     my $ids = $cfg->{ids};
7144
7145     my $res = [];
7146     foreach my $sid (keys %$ids) {
7147         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7148         next if !$ids->{$sid}->{content}->{images};
7149         push @$res, $sid;
7150     }
7151
7152     return $res;
7153 }
7154
7155 1;