PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_SERIAL_PORTS = 4;
 772 my $MAX_PARALLEL_PORTS = 3;
 773 my $MAX_NUMA = 8;
 774
 775 my $numa_fmt = {
 776     cpus => {
 777         type => "string",
 778         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 779         description => "CPUs accessing this NUMA node.",
 780         format_description => "id[-id];...",
 781     },
 782     memory => {
 783         type => "number",
 784         description => "Amount of memory this NUMA node provides.",
 785         optional => 1,
 786     },
 787     hostnodes => {
 788         type => "string",
 789         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 790         description => "Host NUMA nodes to use.",
 791         format_description => "id[-id];...",
 792         optional => 1,
 793     },
 794     policy => {
 795         type => 'string',
 796         enum => [qw(preferred bind interleave)],
 797         description => "NUMA allocation policy.",
 798         optional => 1,
 799     },
 800 };
 801 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 802 my $numadesc = {
 803     optional => 1,
 804     type => 'string', format => $numa_fmt,
 805     description => "NUMA topology.",
 806 };
 807 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 808
 809 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 810     $confdesc->{"numa$i"} = $numadesc;
 811 }
 812
 813 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 814                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 815                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 816 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 817
 818 my $net_fmt_bridge_descr = <<__EOD__;
 819 Bridge to attach the network device to. The Proxmox VE standard bridge
 820 is called 'vmbr0'.
 821
 822 If you do not specify a bridge, we create a kvm user (NATed) network
 823 device, which provides DHCP and DNS services. The following addresses
 824 are used:
 825
 826  10.0.2.2   Gateway
 827  10.0.2.3   DNS Server
 828  10.0.2.4   SMB Server
 829
 830 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 831 __EOD__
 832
 833 my $net_fmt = {
 834     macaddr  => get_standard_option('mac-addr', {
 835         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 836     }),
 837     model => {
 838         type => 'string',
 839         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 840         enum => $nic_model_list,
 841         default_key => 1,
 842     },
 843     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 844     bridge => {
 845         type => 'string',
 846         description => $net_fmt_bridge_descr,
 847         format_description => 'bridge',
 848         pattern => '[-_.\w\d]+',
 849         optional => 1,
 850     },
 851     queues => {
 852         type => 'integer',
 853         minimum => 0, maximum => 16,
 854         description => 'Number of packet queues to be used on the device.',
 855         optional => 1,
 856     },
 857     rate => {
 858         type => 'number',
 859         minimum => 0,
 860         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 861         optional => 1,
 862     },
 863     tag => {
 864         type => 'integer',
 865         minimum => 1, maximum => 4094,
 866         description => 'VLAN tag to apply to packets on this interface.',
 867         optional => 1,
 868     },
 869     trunks => {
 870         type => 'string',
 871         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 872         description => 'VLAN trunks to pass through this interface.',
 873         format_description => 'vlanid[;vlanid...]',
 874         optional => 1,
 875     },
 876     firewall => {
 877         type => 'boolean',
 878         description => 'Whether this interface should be protected by the firewall.',
 879         optional => 1,
 880     },
 881     link_down => {
 882         type => 'boolean',
 883         description => 'Whether this interface should be disconnected (like pulling the plug).',
 884         optional => 1,
 885     },
 886     mtu => {
 887         type => 'integer',
 888         minimum => 1, maximum => 65520,
 889         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 890         optional => 1,
 891     },
 892 };
 893
 894 my $netdesc = {
 895     optional => 1,
 896     type => 'string', format => $net_fmt,
 897     description => "Specify network devices.",
 898 };
 899
 900 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 901
 902 my $ipconfig_fmt = {
 903     ip => {
 904         type => 'string',
 905         format => 'pve-ipv4-config',
 906         format_description => 'IPv4Format/CIDR',
 907         description => 'IPv4 address in CIDR format.',
 908         optional => 1,
 909         default => 'dhcp',
 910     },
 911     gw => {
 912         type => 'string',
 913         format => 'ipv4',
 914         format_description => 'GatewayIPv4',
 915         description => 'Default gateway for IPv4 traffic.',
 916         optional => 1,
 917         requires => 'ip',
 918     },
 919     ip6 => {
 920         type => 'string',
 921         format => 'pve-ipv6-config',
 922         format_description => 'IPv6Format/CIDR',
 923         description => 'IPv6 address in CIDR format.',
 924         optional => 1,
 925         default => 'dhcp',
 926     },
 927     gw6 => {
 928         type => 'string',
 929         format => 'ipv6',
 930         format_description => 'GatewayIPv6',
 931         description => 'Default gateway for IPv6 traffic.',
 932         optional => 1,
 933         requires => 'ip6',
 934     },
 935 };
 936 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 937 my $ipconfigdesc = {
 938     optional => 1,
 939     type => 'string', format => 'pve-qm-ipconfig',
 940     description => <<'EODESCR',
 941 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 942
 943 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 944
 945 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 946 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 947
 948 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 949 EODESCR
 950 };
 951 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 952
 953 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 954     $confdesc->{"net$i"} = $netdesc;
 955     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 956 }
 957
 958 foreach my $key (keys %$confdesc_cloudinit) {
 959     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 960 }
 961
 962 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 963 sub verify_volume_id_or_qm_path {
 964     my ($volid, $noerr) = @_;
 965
 966     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 967         return $volid;
 968     }
 969
 970     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 971     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 972     if ($@) {
 973         return undef if $noerr;
 974         die $@;
 975     }
 976     return $volid;
 977 }
 978
 979 my $usb_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string', format => 'pve-qm-usb-device',
 983         format_description => 'HOSTUSBDEVICE|spice',
 984         description => <<EODESCR,
 985 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 986
 987  'bus-port(.port)*' (decimal numbers) or
 988  'vendor_id:product_id' (hexadeciaml numbers) or
 989  'spice'
 990
 991 You can use the 'lsusb -t' command to list existing usb devices.
 992
 993 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 994
 995 The value 'spice' can be used to add a usb redirection devices for spice.
 996 EODESCR
 997     },
 998     usb3 => {
 999         optional => 1,
1000         type => 'boolean',
1001         description => "Specifies whether if given host option is a USB3 device or port.",
1002         default => 0,
1003     },
1004 };
1005
1006 my $usbdesc = {
1007     optional => 1,
1008     type => 'string', format => $usb_fmt,
1009     description => "Configure an USB device (n is 0 to 4).",
1010 };
1011 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1012
1013 my $serialdesc = {
1014         optional => 1,
1015         type => 'string',
1016         pattern => '(/dev/.+|socket)',
1017         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1018         verbose_description =>  <<EODESCR,
1019 Create a serial device inside the VM (n is 0 to 3), and pass through a
1020 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1021 host side (use 'qm terminal' to open a terminal connection).
1022
1023 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1024
1025 CAUTION: Experimental! User reported problems with this option.
1026 EODESCR
1027 };
1028
1029 my $paralleldesc= {
1030         optional => 1,
1031         type => 'string',
1032         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1033         description =>  "Map host parallel devices (n is 0 to 2).",
1034         verbose_description =>  <<EODESCR,
1035 Map host parallel devices (n is 0 to 2).
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1038
1039 CAUTION: Experimental! User reported problems with this option.
1040 EODESCR
1041 };
1042
1043 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1044     $confdesc->{"parallel$i"} = $paralleldesc;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1048     $confdesc->{"serial$i"} = $serialdesc;
1049 }
1050
1051 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1052     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1053 }
1054
1055 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1056     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1057 }
1058
1059 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1060     $confdesc->{"usb$i"} = $usbdesc;
1061 }
1062
1063 my $kvm_api_version = 0;
1064
1065 sub kvm_version {
1066     return $kvm_api_version if $kvm_api_version;
1067
1068     open my $fh, '<', '/dev/kvm'
1069         or return undef;
1070
1071     # 0xae00 => KVM_GET_API_VERSION
1072     $kvm_api_version = ioctl($fh, 0xae00, 0);
1073
1074     return $kvm_api_version;
1075 }
1076
1077 my $kvm_user_version = {};
1078 my $kvm_mtime = {};
1079
1080 sub kvm_user_version {
1081     my ($binary) = @_;
1082
1083     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1084     my $st = stat($binary);
1085
1086     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1087     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1088         $cachedmtime == $st->mtime;
1089
1090     $kvm_user_version->{$binary} = 'unknown';
1091     $kvm_mtime->{$binary} = $st->mtime;
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version->{$binary} = $2;
1097         }
1098     };
1099
1100     eval { run_command([$binary, '--version'], outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version->{$binary};
1104
1105 }
1106
1107 sub kernel_has_vhost_net {
1108     return -c '/dev/vhost-net';
1109 }
1110
1111 sub option_exists {
1112     my $key = shift;
1113     return defined($confdesc->{$key});
1114 }
1115
1116 my $cdrom_path;
1117 sub get_cdrom_path {
1118
1119     return  $cdrom_path if $cdrom_path;
1120
1121     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1122     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1123     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1124 }
1125
1126 sub get_iso_path {
1127     my ($storecfg, $vmid, $cdrom) = @_;
1128
1129     if ($cdrom eq 'cdrom') {
1130         return get_cdrom_path();
1131     } elsif ($cdrom eq 'none') {
1132         return '';
1133     } elsif ($cdrom =~ m|^/|) {
1134         return $cdrom;
1135     } else {
1136         return PVE::Storage::path($storecfg, $cdrom);
1137     }
1138 }
1139
1140 # try to convert old style file names to volume IDs
1141 sub filename_to_volume_id {
1142     my ($vmid, $file, $media) = @_;
1143
1144      if (!($file eq 'none' || $file eq 'cdrom' ||
1145           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1146
1147         return undef if $file =~ m|/|;
1148
1149         if ($media && $media eq 'cdrom') {
1150             $file = "local:iso/$file";
1151         } else {
1152             $file = "local:$vmid/$file";
1153         }
1154     }
1155
1156     return $file;
1157 }
1158
1159 sub verify_media_type {
1160     my ($opt, $vtype, $media) = @_;
1161
1162     return if !$media;
1163
1164     my $etype;
1165     if ($media eq 'disk') {
1166         $etype = 'images';
1167     } elsif ($media eq 'cdrom') {
1168         $etype = 'iso';
1169     } else {
1170         die "internal error";
1171     }
1172
1173     return if ($vtype eq $etype);
1174
1175     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1176 }
1177
1178 sub cleanup_drive_path {
1179     my ($opt, $storecfg, $drive) = @_;
1180
1181     # try to convert filesystem paths to volume IDs
1182
1183     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1184         ($drive->{file} !~ m|^/dev/.+|) &&
1185         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1186         ($drive->{file} !~ m/^\d+$/)) {
1187         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1188         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1189         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1190         verify_media_type($opt, $vtype, $drive->{media});
1191         $drive->{file} = $volid;
1192     }
1193
1194     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1195 }
1196
1197 sub parse_hotplug_features {
1198     my ($data) = @_;
1199
1200     my $res = {};
1201
1202     return $res if $data eq '0';
1203
1204     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1205
1206     foreach my $feature (PVE::Tools::split_list($data)) {
1207         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1208             $res->{$1} = 1;
1209         } else {
1210             die "invalid hotplug feature '$feature'\n";
1211         }
1212     }
1213     return $res;
1214 }
1215
1216 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1217 sub pve_verify_hotplug_features {
1218     my ($value, $noerr) = @_;
1219
1220     return $value if parse_hotplug_features($value);
1221
1222     return undef if $noerr;
1223
1224     die "unable to parse hotplug option\n";
1225 }
1226
1227 sub scsi_inquiry {
1228     my($fh, $noerr) = @_;
1229
1230     my $SG_IO = 0x2285;
1231     my $SG_GET_VERSION_NUM = 0x2282;
1232
1233     my $versionbuf = "\x00" x 8;
1234     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1235     if (!$ret) {
1236         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1237         return undef;
1238     }
1239     my $version = unpack("I", $versionbuf);
1240     if ($version < 30000) {
1241         die "scsi generic interface too old\n"  if !$noerr;
1242         return undef;
1243     }
1244
1245     my $buf = "\x00" x 36;
1246     my $sensebuf = "\x00" x 8;
1247     my $cmd = pack("C x3 C x1", 0x12, 36);
1248
1249     # see /usr/include/scsi/sg.h
1250     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1251
1252     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1253                       length($sensebuf), 0, length($buf), $buf,
1254                       $cmd, $sensebuf, 6000);
1255
1256     $ret = ioctl($fh, $SG_IO, $packet);
1257     if (!$ret) {
1258         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1259         return undef;
1260     }
1261
1262     my @res = unpack($sg_io_hdr_t, $packet);
1263     if ($res[17] || $res[18]) {
1264         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1265         return undef;
1266     }
1267
1268     my $res = {};
1269     (my $byte0, my $byte1, $res->{vendor},
1270      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1271
1272     $res->{removable} = $byte1 & 128 ? 1 : 0;
1273     $res->{type} = $byte0 & 31;
1274
1275     return $res;
1276 }
1277
1278 sub path_is_scsi {
1279     my ($path) = @_;
1280
1281     my $fh = IO::File->new("+<$path") || return undef;
1282     my $res = scsi_inquiry($fh, 1);
1283     close($fh);
1284
1285     return $res;
1286 }
1287
1288 sub print_tabletdevice_full {
1289     my ($conf, $arch) = @_;
1290
1291     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1292
1293     # we use uhci for old VMs because tablet driver was buggy in older qemu
1294     my $usbbus;
1295     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1296         $usbbus = 'ehci';
1297     } else {
1298         $usbbus = 'uhci';
1299     }
1300
1301     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1302 }
1303
1304 sub print_keyboarddevice_full {
1305     my ($conf, $arch, $machine) = @_;
1306
1307     return undef if $arch ne 'aarch64';
1308
1309     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1310 }
1311
1312 sub print_drivedevice_full {
1313     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1314
1315     my $device = '';
1316     my $maxdev = 0;
1317
1318     if ($drive->{interface} eq 'virtio') {
1319         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1320         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1321         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1322     } elsif ($drive->{interface} eq 'scsi') {
1323
1324         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1325         my $unit = $drive->{index} % $maxdev;
1326         my $devicetype = 'hd';
1327         my $path = '';
1328         if (drive_is_cdrom($drive)) {
1329             $devicetype = 'cd';
1330         } else {
1331             if ($drive->{file} =~ m|^/|) {
1332                 $path = $drive->{file};
1333                 if (my $info = path_is_scsi($path)) {
1334                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1335                         $devicetype = 'block';
1336                     } elsif ($info->{type} == 1) { # tape
1337                         $devicetype = 'generic';
1338                     }
1339                 }
1340             } else {
1341                  $path = PVE::Storage::path($storecfg, $drive->{file});
1342             }
1343
1344             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1345             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1346             if ($path =~ m/^iscsi\:\/\// &&
1347                !min_version($version, 4, 1)) {
1348                 $devicetype = 'generic';
1349             }
1350         }
1351
1352         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1353             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1354         } else {
1355             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1356         }
1357
1358         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1359             $device .= ",rotation_rate=1";
1360         }
1361         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1362
1363     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1364         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1365         my $controller = int($drive->{index} / $maxdev);
1366         my $unit = $drive->{index} % $maxdev;
1367         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1368
1369         $device = "ide-$devicetype";
1370         if ($drive->{interface} eq 'ide') {
1371             $device .= ",bus=ide.$controller,unit=$unit";
1372         } else {
1373             $device .= ",bus=ahci$controller.$unit";
1374         }
1375         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1376
1377         if ($devicetype eq 'hd') {
1378             if (my $model = $drive->{model}) {
1379                 $model = URI::Escape::uri_unescape($model);
1380                 $device .= ",model=$model";
1381             }
1382             if ($drive->{ssd}) {
1383                 $device .= ",rotation_rate=1";
1384             }
1385         }
1386         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1387     } elsif ($drive->{interface} eq 'usb') {
1388         die "implement me";
1389         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1390     } else {
1391         die "unsupported interface type";
1392     }
1393
1394     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1395
1396     if (my $serial = $drive->{serial}) {
1397         $serial = URI::Escape::uri_unescape($serial);
1398         $device .= ",serial=$serial";
1399     }
1400
1401
1402     return $device;
1403 }
1404
1405 sub get_initiator_name {
1406     my $initiator;
1407
1408     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1409     while (defined(my $line = <$fh>)) {
1410         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1411         $initiator = $1;
1412         last;
1413     }
1414     $fh->close();
1415
1416     return $initiator;
1417 }
1418
1419 sub print_drive_commandline_full {
1420     my ($storecfg, $vmid, $drive) = @_;
1421
1422     my $path;
1423     my $volid = $drive->{file};
1424     my $format;
1425
1426     if (drive_is_cdrom($drive)) {
1427         $path = get_iso_path($storecfg, $vmid, $volid);
1428     } else {
1429         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1430         if ($storeid) {
1431             $path = PVE::Storage::path($storecfg, $volid);
1432             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1433             $format = qemu_img_format($scfg, $volname);
1434         } else {
1435             $path = $volid;
1436             $format = "raw";
1437         }
1438    }
1439
1440     my $opts = '';
1441     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1442     foreach my $o (@qemu_drive_options) {
1443         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1444     }
1445
1446     # snapshot only accepts on|off
1447     if (defined($drive->{snapshot})) {
1448         my $v = $drive->{snapshot} ? 'on' : 'off';
1449         $opts .= ",snapshot=$v";
1450     }
1451
1452     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1453         my ($dir, $qmpname) = @$type;
1454         if (my $v = $drive->{"mbps$dir"}) {
1455             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1456         }
1457         if (my $v = $drive->{"mbps${dir}_max"}) {
1458             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1459         }
1460         if (my $v = $drive->{"bps${dir}_max_length"}) {
1461             $opts .= ",throttling.bps$qmpname-max-length=$v";
1462         }
1463         if (my $v = $drive->{"iops${dir}"}) {
1464             $opts .= ",throttling.iops$qmpname=$v";
1465         }
1466         if (my $v = $drive->{"iops${dir}_max"}) {
1467             $opts .= ",throttling.iops$qmpname-max=$v";
1468         }
1469         if (my $v = $drive->{"iops${dir}_max_length"}) {
1470             $opts .= ",throttling.iops$qmpname-max-length=$v";
1471         }
1472     }
1473
1474     $opts .= ",format=$format" if $format && !$drive->{format};
1475
1476     my $cache_direct = 0;
1477
1478     if (my $cache = $drive->{cache}) {
1479         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1480     } elsif (!drive_is_cdrom($drive)) {
1481         $opts .= ",cache=none";
1482         $cache_direct = 1;
1483     }
1484
1485     # aio native works only with O_DIRECT
1486     if (!$drive->{aio}) {
1487         if($cache_direct) {
1488             $opts .= ",aio=native";
1489         } else {
1490             $opts .= ",aio=threads";
1491         }
1492     }
1493
1494     if (!drive_is_cdrom($drive)) {
1495         my $detectzeroes;
1496         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1497             $detectzeroes = 'off';
1498         } elsif ($drive->{discard}) {
1499             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1500         } else {
1501             # This used to be our default with discard not being specified:
1502             $detectzeroes = 'on';
1503         }
1504         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1505     }
1506
1507     my $pathinfo = $path ? "file=$path," : '';
1508
1509     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1510 }
1511
1512 sub print_netdevice_full {
1513     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1514
1515     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1516
1517     my $device = $net->{model};
1518     if ($net->{model} eq 'virtio') {
1519          $device = 'virtio-net-pci';
1520      };
1521
1522     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1523     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1524     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1525         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1526         my $vectors = $net->{queues} * 2 + 2;
1527         $tmpstr .= ",vectors=$vectors,mq=on";
1528     }
1529     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1530
1531     if (my $mtu = $net->{mtu}) {
1532         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1533             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1534             if ($mtu == 1) {
1535                  $mtu = $bridge_mtu;
1536             } elsif ($mtu < 576) {
1537                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1538             } elsif ($mtu > $bridge_mtu) {
1539                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1540             }
1541             $tmpstr .= ",host_mtu=$mtu";
1542         } else {
1543             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1544         }
1545     }
1546
1547     if ($use_old_bios_files) {
1548         my $romfile;
1549         if ($device eq 'virtio-net-pci') {
1550             $romfile = 'pxe-virtio.rom';
1551         } elsif ($device eq 'e1000') {
1552             $romfile = 'pxe-e1000.rom';
1553         } elsif ($device eq 'ne2k') {
1554             $romfile = 'pxe-ne2k_pci.rom';
1555         } elsif ($device eq 'pcnet') {
1556             $romfile = 'pxe-pcnet.rom';
1557         } elsif ($device eq 'rtl8139') {
1558             $romfile = 'pxe-rtl8139.rom';
1559         }
1560         $tmpstr .= ",romfile=$romfile" if $romfile;
1561     }
1562
1563     return $tmpstr;
1564 }
1565
1566 sub print_netdev_full {
1567     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1568
1569     my $i = '';
1570     if ($netid =~ m/^net(\d+)$/) {
1571         $i = int($1);
1572     }
1573
1574     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1575
1576     my $ifname = "tap${vmid}i$i";
1577
1578     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1579     die "interface name '$ifname' is too long (max 15 character)\n"
1580         if length($ifname) >= 16;
1581
1582     my $vhostparam = '';
1583     if (is_native($arch)) {
1584         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1585     }
1586
1587     my $vmname = $conf->{name} || "vm$vmid";
1588
1589     my $netdev = "";
1590     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1591
1592     if ($net->{bridge}) {
1593         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1594     } else {
1595         $netdev = "type=user,id=$netid,hostname=$vmname";
1596     }
1597
1598     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1599
1600     return $netdev;
1601 }
1602
1603 my $vga_map = {
1604     'cirrus' => 'cirrus-vga',
1605     'std' => 'VGA',
1606     'vmware' => 'vmware-svga',
1607     'virtio' => 'virtio-vga',
1608 };
1609
1610 sub print_vga_device {
1611     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1612
1613     my $type = $vga_map->{$vga->{type}};
1614     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1615         $type = 'virtio-gpu';
1616     }
1617     my $vgamem_mb = $vga->{memory};
1618
1619     my $max_outputs = '';
1620     if ($qxlnum) {
1621         $type = $id ? 'qxl' : 'qxl-vga';
1622
1623         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1624             # set max outputs so linux can have up to 4 qxl displays with one device
1625             if (min_version($machine_version, 4, 1)) {
1626                 $max_outputs = ",max_outputs=4";
1627             }
1628         }
1629     }
1630
1631     die "no devicetype for $vga->{type}\n" if !$type;
1632
1633     my $memory = "";
1634     if ($vgamem_mb) {
1635         if ($vga->{type} eq 'virtio') {
1636             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1637             $memory = ",max_hostmem=$bytes";
1638         } elsif ($qxlnum) {
1639             # from https://www.spice-space.org/multiple-monitors.html
1640             $memory = ",vgamem_mb=$vga->{memory}";
1641             my $ram = $vgamem_mb * 4;
1642             my $vram = $vgamem_mb * 2;
1643             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1644         } else {
1645             $memory = ",vgamem_mb=$vga->{memory}";
1646         }
1647     } elsif ($qxlnum && $id) {
1648         $memory = ",ram_size=67108864,vram_size=33554432";
1649     }
1650
1651     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1652     my $vgaid = "vga" . ($id // '');
1653     my $pciaddr;
1654
1655     if ($q35 && $vgaid eq 'vga') {
1656         # the first display uses pcie.0 bus on q35 machines
1657         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1658     } else {
1659         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1660     }
1661
1662     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1663 }
1664
1665 sub parse_number_sets {
1666     my ($set) = @_;
1667     my $res = [];
1668     foreach my $part (split(/;/, $set)) {
1669         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1670             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1671             push @$res, [ $1, $2 ];
1672         } else {
1673             die "invalid range: $part\n";
1674         }
1675     }
1676     return $res;
1677 }
1678
1679 sub parse_numa {
1680     my ($data) = @_;
1681
1682     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1683     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1684     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1685     return $res;
1686 }
1687
1688 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1689 sub parse_net {
1690     my ($data) = @_;
1691
1692     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1693     if ($@) {
1694         warn $@;
1695         return undef;
1696     }
1697     if (!defined($res->{macaddr})) {
1698         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1699         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1700     }
1701     return $res;
1702 }
1703
1704 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1705 sub parse_ipconfig {
1706     my ($data) = @_;
1707
1708     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1709     if ($@) {
1710         warn $@;
1711         return undef;
1712     }
1713
1714     if ($res->{gw} && !$res->{ip}) {
1715         warn 'gateway specified without specifying an IP address';
1716         return undef;
1717     }
1718     if ($res->{gw6} && !$res->{ip6}) {
1719         warn 'IPv6 gateway specified without specifying an IPv6 address';
1720         return undef;
1721     }
1722     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1723         warn 'gateway specified together with DHCP';
1724         return undef;
1725     }
1726     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1727         # gw6 + auto/dhcp
1728         warn "IPv6 gateway specified together with $res->{ip6} address";
1729         return undef;
1730     }
1731
1732     if (!$res->{ip} && !$res->{ip6}) {
1733         return { ip => 'dhcp', ip6 => 'dhcp' };
1734     }
1735
1736     return $res;
1737 }
1738
1739 sub print_net {
1740     my $net = shift;
1741
1742     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1743 }
1744
1745 sub add_random_macs {
1746     my ($settings) = @_;
1747
1748     foreach my $opt (keys %$settings) {
1749         next if $opt !~ m/^net(\d+)$/;
1750         my $net = parse_net($settings->{$opt});
1751         next if !$net;
1752         $settings->{$opt} = print_net($net);
1753     }
1754 }
1755
1756 sub vm_is_volid_owner {
1757     my ($storecfg, $vmid, $volid) = @_;
1758
1759     if ($volid !~  m|^/|) {
1760         my ($path, $owner);
1761         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1762         if ($owner && ($owner == $vmid)) {
1763             return 1;
1764         }
1765     }
1766
1767     return undef;
1768 }
1769
1770 sub vmconfig_register_unused_drive {
1771     my ($storecfg, $vmid, $conf, $drive) = @_;
1772
1773     if (drive_is_cloudinit($drive)) {
1774         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1775         warn $@ if $@;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         my $volid = $drive->{file};
1778         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1779             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1780         }
1781     }
1782 }
1783
1784 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1785 my $smbios1_fmt = {
1786     uuid => {
1787         type => 'string',
1788         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1789         format_description => 'UUID',
1790         description => "Set SMBIOS1 UUID.",
1791         optional => 1,
1792     },
1793     version => {
1794         type => 'string',
1795         pattern => '[A-Za-z0-9+\/]+={0,2}',
1796         format_description => 'Base64 encoded string',
1797         description => "Set SMBIOS1 version.",
1798         optional => 1,
1799     },
1800     serial => {
1801         type => 'string',
1802         pattern => '[A-Za-z0-9+\/]+={0,2}',
1803         format_description => 'Base64 encoded string',
1804         description => "Set SMBIOS1 serial number.",
1805         optional => 1,
1806     },
1807     manufacturer => {
1808         type => 'string',
1809         pattern => '[A-Za-z0-9+\/]+={0,2}',
1810         format_description => 'Base64 encoded string',
1811         description => "Set SMBIOS1 manufacturer.",
1812         optional => 1,
1813     },
1814     product => {
1815         type => 'string',
1816         pattern => '[A-Za-z0-9+\/]+={0,2}',
1817         format_description => 'Base64 encoded string',
1818         description => "Set SMBIOS1 product ID.",
1819         optional => 1,
1820     },
1821     sku => {
1822         type => 'string',
1823         pattern => '[A-Za-z0-9+\/]+={0,2}',
1824         format_description => 'Base64 encoded string',
1825         description => "Set SMBIOS1 SKU string.",
1826         optional => 1,
1827     },
1828     family => {
1829         type => 'string',
1830         pattern => '[A-Za-z0-9+\/]+={0,2}',
1831         format_description => 'Base64 encoded string',
1832         description => "Set SMBIOS1 family string.",
1833         optional => 1,
1834     },
1835     base64 => {
1836         type => 'boolean',
1837         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1838         optional => 1,
1839     },
1840 };
1841
1842 sub parse_smbios1 {
1843     my ($data) = @_;
1844
1845     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1846     warn $@ if $@;
1847     return $res;
1848 }
1849
1850 sub print_smbios1 {
1851     my ($smbios1) = @_;
1852     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1853 }
1854
1855 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1856
1857 sub parse_watchdog {
1858     my ($value) = @_;
1859
1860     return undef if !$value;
1861
1862     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1863     warn $@ if $@;
1864     return $res;
1865 }
1866
1867 sub parse_guest_agent {
1868     my ($value) = @_;
1869
1870     return {} if !defined($value->{agent});
1871
1872     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1873     warn $@ if $@;
1874
1875     # if the agent is disabled ignore the other potentially set properties
1876     return {} if !$res->{enabled};
1877     return $res;
1878 }
1879
1880 sub parse_vga {
1881     my ($value) = @_;
1882
1883     return {} if !$value;
1884     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1885     warn $@ if $@;
1886     return $res;
1887 }
1888
1889 sub parse_rng {
1890     my ($value) = @_;
1891
1892     return undef if !$value;
1893
1894     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1895     warn $@ if $@;
1896     return $res;
1897 }
1898
1899 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1900 sub verify_usb_device {
1901     my ($value, $noerr) = @_;
1902
1903     return $value if parse_usb_device($value);
1904
1905     return undef if $noerr;
1906
1907     die "unable to parse usb device\n";
1908 }
1909
1910 # add JSON properties for create and set function
1911 sub json_config_properties {
1912     my $prop = shift;
1913
1914     foreach my $opt (keys %$confdesc) {
1915         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1916             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1917         $prop->{$opt} = $confdesc->{$opt};
1918     }
1919
1920     return $prop;
1921 }
1922
1923 # return copy of $confdesc_cloudinit to generate documentation
1924 sub cloudinit_config_properties {
1925
1926     return dclone($confdesc_cloudinit);
1927 }
1928
1929 sub check_type {
1930     my ($key, $value) = @_;
1931
1932     die "unknown setting '$key'\n" if !$confdesc->{$key};
1933
1934     my $type = $confdesc->{$key}->{type};
1935
1936     if (!defined($value)) {
1937         die "got undefined value\n";
1938     }
1939
1940     if ($value =~ m/[\n\r]/) {
1941         die "property contains a line feed\n";
1942     }
1943
1944     if ($type eq 'boolean') {
1945         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1946         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1947         die "type check ('boolean') failed - got '$value'\n";
1948     } elsif ($type eq 'integer') {
1949         return int($1) if $value =~ m/^(\d+)$/;
1950         die "type check ('integer') failed - got '$value'\n";
1951     } elsif ($type eq 'number') {
1952         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1953         die "type check ('number') failed - got '$value'\n";
1954     } elsif ($type eq 'string') {
1955         if (my $fmt = $confdesc->{$key}->{format}) {
1956             PVE::JSONSchema::check_format($fmt, $value);
1957             return $value;
1958         }
1959         $value =~ s/^\"(.*)\"$/$1/;
1960         return $value;
1961     } else {
1962         die "internal error"
1963     }
1964 }
1965
1966 sub destroy_vm {
1967     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1968
1969     my $conf = PVE::QemuConfig->load_config($vmid);
1970
1971     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1972
1973     if ($conf->{template}) {
1974         # check if any base image is still used by a linked clone
1975         PVE::QemuConfig->foreach_volume($conf, sub {
1976                 my ($ds, $drive) = @_;
1977                 return if drive_is_cdrom($drive);
1978
1979                 my $volid = $drive->{file};
1980                 return if !$volid || $volid =~ m|^/|;
1981
1982                 die "base volume '$volid' is still in use by linked cloned\n"
1983                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
1984
1985         });
1986     }
1987
1988     # only remove disks owned by this VM
1989     PVE::QemuConfig->foreach_volume($conf, sub {
1990         my ($ds, $drive) = @_;
1991         return if drive_is_cdrom($drive, 1);
1992
1993         my $volid = $drive->{file};
1994         return if !$volid || $volid =~ m|^/|;
1995
1996         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1997         return if !$path || !$owner || ($owner != $vmid);
1998
1999         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2000         warn "Could not remove disk '$volid', check manually: $@" if $@;
2001     });
2002
2003     # also remove unused disk
2004     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2005     PVE::Storage::foreach_volid($vmdisks, sub {
2006         my ($volid, $sid, $volname, $d) = @_;
2007         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2008         warn $@ if $@;
2009     });
2010
2011     if (defined $replacement_conf) {
2012         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2013     } else {
2014         PVE::QemuConfig->destroy_config($vmid);
2015     }
2016 }
2017
2018 sub parse_vm_config {
2019     my ($filename, $raw) = @_;
2020
2021     return undef if !defined($raw);
2022
2023     my $res = {
2024         digest => Digest::SHA::sha1_hex($raw),
2025         snapshots => {},
2026         pending => {},
2027     };
2028
2029     $filename =~ m|/qemu-server/(\d+)\.conf$|
2030         || die "got strange filename '$filename'";
2031
2032     my $vmid = $1;
2033
2034     my $conf = $res;
2035     my $descr;
2036     my $section = '';
2037
2038     my @lines = split(/\n/, $raw);
2039     foreach my $line (@lines) {
2040         next if $line =~ m/^\s*$/;
2041
2042         if ($line =~ m/^\[PENDING\]\s*$/i) {
2043             $section = 'pending';
2044             if (defined($descr)) {
2045                 $descr =~ s/\s+$//;
2046                 $conf->{description} = $descr;
2047             }
2048             $descr = undef;
2049             $conf = $res->{$section} = {};
2050             next;
2051
2052         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2053             $section = $1;
2054             if (defined($descr)) {
2055                 $descr =~ s/\s+$//;
2056                 $conf->{description} = $descr;
2057             }
2058             $descr = undef;
2059             $conf = $res->{snapshots}->{$section} = {};
2060             next;
2061         }
2062
2063         if ($line =~ m/^\#(.*)\s*$/) {
2064             $descr = '' if !defined($descr);
2065             $descr .= PVE::Tools::decode_text($1) . "\n";
2066             next;
2067         }
2068
2069         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2070             $descr = '' if !defined($descr);
2071             $descr .= PVE::Tools::decode_text($2);
2072         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2073             $conf->{snapstate} = $1;
2074         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2075             my $key = $1;
2076             my $value = $2;
2077             $conf->{$key} = $value;
2078         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2079             my $value = $1;
2080             if ($section eq 'pending') {
2081                 $conf->{delete} = $value; # we parse this later
2082             } else {
2083                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2084             }
2085         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2086             my $key = $1;
2087             my $value = $2;
2088             eval { $value = check_type($key, $value); };
2089             if ($@) {
2090                 warn "vm $vmid - unable to parse value of '$key' - $@";
2091             } else {
2092                 $key = 'ide2' if $key eq 'cdrom';
2093                 my $fmt = $confdesc->{$key}->{format};
2094                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2095                     my $v = parse_drive($key, $value);
2096                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2097                         $v->{file} = $volid;
2098                         $value = print_drive($v);
2099                     } else {
2100                         warn "vm $vmid - unable to parse value of '$key'\n";
2101                         next;
2102                     }
2103                 }
2104
2105                 $conf->{$key} = $value;
2106             }
2107         }
2108     }
2109
2110     if (defined($descr)) {
2111         $descr =~ s/\s+$//;
2112         $conf->{description} = $descr;
2113     }
2114     delete $res->{snapstate}; # just to be sure
2115
2116     return $res;
2117 }
2118
2119 sub write_vm_config {
2120     my ($filename, $conf) = @_;
2121
2122     delete $conf->{snapstate}; # just to be sure
2123
2124     if ($conf->{cdrom}) {
2125         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2126         $conf->{ide2} = $conf->{cdrom};
2127         delete $conf->{cdrom};
2128     }
2129
2130     # we do not use 'smp' any longer
2131     if ($conf->{sockets}) {
2132         delete $conf->{smp};
2133     } elsif ($conf->{smp}) {
2134         $conf->{sockets} = $conf->{smp};
2135         delete $conf->{cores};
2136         delete $conf->{smp};
2137     }
2138
2139     my $used_volids = {};
2140
2141     my $cleanup_config = sub {
2142         my ($cref, $pending, $snapname) = @_;
2143
2144         foreach my $key (keys %$cref) {
2145             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2146                 $key eq 'snapstate' || $key eq 'pending';
2147             my $value = $cref->{$key};
2148             if ($key eq 'delete') {
2149                 die "propertry 'delete' is only allowed in [PENDING]\n"
2150                     if !$pending;
2151                 # fixme: check syntax?
2152                 next;
2153             }
2154             eval { $value = check_type($key, $value); };
2155             die "unable to parse value of '$key' - $@" if $@;
2156
2157             $cref->{$key} = $value;
2158
2159             if (!$snapname && is_valid_drivename($key)) {
2160                 my $drive = parse_drive($key, $value);
2161                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2162             }
2163         }
2164     };
2165
2166     &$cleanup_config($conf);
2167
2168     &$cleanup_config($conf->{pending}, 1);
2169
2170     foreach my $snapname (keys %{$conf->{snapshots}}) {
2171         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2172         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2173     }
2174
2175     # remove 'unusedX' settings if we re-add a volume
2176     foreach my $key (keys %$conf) {
2177         my $value = $conf->{$key};
2178         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2179             delete $conf->{$key};
2180         }
2181     }
2182
2183     my $generate_raw_config = sub {
2184         my ($conf, $pending) = @_;
2185
2186         my $raw = '';
2187
2188         # add description as comment to top of file
2189         if (defined(my $descr = $conf->{description})) {
2190             if ($descr) {
2191                 foreach my $cl (split(/\n/, $descr)) {
2192                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2193                 }
2194             } else {
2195                 $raw .= "#\n" if $pending;
2196             }
2197         }
2198
2199         foreach my $key (sort keys %$conf) {
2200             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2201             $raw .= "$key: $conf->{$key}\n";
2202         }
2203         return $raw;
2204     };
2205
2206     my $raw = &$generate_raw_config($conf);
2207
2208     if (scalar(keys %{$conf->{pending}})){
2209         $raw .= "\n[PENDING]\n";
2210         $raw .= &$generate_raw_config($conf->{pending}, 1);
2211     }
2212
2213     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2214         $raw .= "\n[$snapname]\n";
2215         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2216     }
2217
2218     return $raw;
2219 }
2220
2221 sub load_defaults {
2222
2223     my $res = {};
2224
2225     # we use static defaults from our JSON schema configuration
2226     foreach my $key (keys %$confdesc) {
2227         if (defined(my $default = $confdesc->{$key}->{default})) {
2228             $res->{$key} = $default;
2229         }
2230     }
2231
2232     return $res;
2233 }
2234
2235 sub config_list {
2236     my $vmlist = PVE::Cluster::get_vmlist();
2237     my $res = {};
2238     return $res if !$vmlist || !$vmlist->{ids};
2239     my $ids = $vmlist->{ids};
2240     my $nodename = nodename();
2241
2242     foreach my $vmid (keys %$ids) {
2243         my $d = $ids->{$vmid};
2244         next if !$d->{node} || $d->{node} ne $nodename;
2245         next if !$d->{type} || $d->{type} ne 'qemu';
2246         $res->{$vmid}->{exists} = 1;
2247     }
2248     return $res;
2249 }
2250
2251 # test if VM uses local resources (to prevent migration)
2252 sub check_local_resources {
2253     my ($conf, $noerr) = @_;
2254
2255     my @loc_res = ();
2256
2257     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2258     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2259
2260     push @loc_res, "ivshmem" if $conf->{ivshmem};
2261
2262     foreach my $k (keys %$conf) {
2263         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2264         # sockets are safe: they will recreated be on the target side post-migrate
2265         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2266         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2267     }
2268
2269     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2270
2271     return \@loc_res;
2272 }
2273
2274 # check if used storages are available on all nodes (use by migrate)
2275 sub check_storage_availability {
2276     my ($storecfg, $conf, $node) = @_;
2277
2278     PVE::QemuConfig->foreach_volume($conf, sub {
2279         my ($ds, $drive) = @_;
2280
2281         my $volid = $drive->{file};
2282         return if !$volid;
2283
2284         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2285         return if !$sid;
2286
2287         # check if storage is available on both nodes
2288         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2289         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2290    });
2291 }
2292
2293 # list nodes where all VM images are available (used by has_feature API)
2294 sub shared_nodes {
2295     my ($conf, $storecfg) = @_;
2296
2297     my $nodelist = PVE::Cluster::get_nodelist();
2298     my $nodehash = { map { $_ => 1 } @$nodelist };
2299     my $nodename = nodename();
2300
2301     PVE::QemuConfig->foreach_volume($conf, sub {
2302         my ($ds, $drive) = @_;
2303
2304         my $volid = $drive->{file};
2305         return if !$volid;
2306
2307         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2308         if ($storeid) {
2309             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2310             if ($scfg->{disable}) {
2311                 $nodehash = {};
2312             } elsif (my $avail = $scfg->{nodes}) {
2313                 foreach my $node (keys %$nodehash) {
2314                     delete $nodehash->{$node} if !$avail->{$node};
2315                 }
2316             } elsif (!$scfg->{shared}) {
2317                 foreach my $node (keys %$nodehash) {
2318                     delete $nodehash->{$node} if $node ne $nodename
2319                 }
2320             }
2321         }
2322     });
2323
2324     return $nodehash
2325 }
2326
2327 sub check_local_storage_availability {
2328     my ($conf, $storecfg) = @_;
2329
2330     my $nodelist = PVE::Cluster::get_nodelist();
2331     my $nodehash = { map { $_ => {} } @$nodelist };
2332
2333     PVE::QemuConfig->foreach_volume($conf, sub {
2334         my ($ds, $drive) = @_;
2335
2336         my $volid = $drive->{file};
2337         return if !$volid;
2338
2339         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2340         if ($storeid) {
2341             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2342
2343             if ($scfg->{disable}) {
2344                 foreach my $node (keys %$nodehash) {
2345                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2346                 }
2347             } elsif (my $avail = $scfg->{nodes}) {
2348                 foreach my $node (keys %$nodehash) {
2349                     if (!$avail->{$node}) {
2350                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2351                     }
2352                 }
2353             }
2354         }
2355     });
2356
2357     foreach my $node (values %$nodehash) {
2358         if (my $unavail = $node->{unavailable_storages}) {
2359             $node->{unavailable_storages} = [ sort keys %$unavail ];
2360         }
2361     }
2362
2363     return $nodehash
2364 }
2365
2366 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2367 sub check_running {
2368     my ($vmid, $nocheck, $node) = @_;
2369
2370     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2371     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2372 }
2373
2374 sub vzlist {
2375
2376     my $vzlist = config_list();
2377
2378     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2379
2380     while (defined(my $de = $fd->read)) {
2381         next if $de !~ m/^(\d+)\.pid$/;
2382         my $vmid = $1;
2383         next if !defined($vzlist->{$vmid});
2384         if (my $pid = check_running($vmid)) {
2385             $vzlist->{$vmid}->{pid} = $pid;
2386         }
2387     }
2388
2389     return $vzlist;
2390 }
2391
2392 our $vmstatus_return_properties = {
2393     vmid => get_standard_option('pve-vmid'),
2394     status => {
2395         description => "Qemu process status.",
2396         type => 'string',
2397         enum => ['stopped', 'running'],
2398     },
2399     maxmem => {
2400         description => "Maximum memory in bytes.",
2401         type => 'integer',
2402         optional => 1,
2403         renderer => 'bytes',
2404     },
2405     maxdisk => {
2406         description => "Root disk size in bytes.",
2407         type => 'integer',
2408         optional => 1,
2409         renderer => 'bytes',
2410     },
2411     name => {
2412         description => "VM name.",
2413         type => 'string',
2414         optional => 1,
2415     },
2416     qmpstatus => {
2417         description => "Qemu QMP agent status.",
2418         type => 'string',
2419         optional => 1,
2420     },
2421     pid => {
2422         description => "PID of running qemu process.",
2423         type => 'integer',
2424         optional => 1,
2425     },
2426     uptime => {
2427         description => "Uptime.",
2428         type => 'integer',
2429         optional => 1,
2430         renderer => 'duration',
2431     },
2432     cpus => {
2433         description => "Maximum usable CPUs.",
2434         type => 'number',
2435         optional => 1,
2436     },
2437     lock => {
2438         description => "The current config lock, if any.",
2439         type => 'string',
2440         optional => 1,
2441     },
2442     tags => {
2443         description  => "The current configured tags, if any",
2444         type => 'string',
2445         optional => 1,
2446     },
2447 };
2448
2449 my $last_proc_pid_stat;
2450
2451 # get VM status information
2452 # This must be fast and should not block ($full == false)
2453 # We only query KVM using QMP if $full == true (this can be slow)
2454 sub vmstatus {
2455     my ($opt_vmid, $full) = @_;
2456
2457     my $res = {};
2458
2459     my $storecfg = PVE::Storage::config();
2460
2461     my $list = vzlist();
2462     my $defaults = load_defaults();
2463
2464     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2465
2466     my $cpucount = $cpuinfo->{cpus} || 1;
2467
2468     foreach my $vmid (keys %$list) {
2469         next if $opt_vmid && ($vmid ne $opt_vmid);
2470
2471         my $conf = PVE::QemuConfig->load_config($vmid);
2472
2473         my $d = { vmid => $vmid };
2474         $d->{pid} = $list->{$vmid}->{pid};
2475
2476         # fixme: better status?
2477         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2478
2479         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2480         if (defined($size)) {
2481             $d->{disk} = 0; # no info available
2482             $d->{maxdisk} = $size;
2483         } else {
2484             $d->{disk} = 0;
2485             $d->{maxdisk} = 0;
2486         }
2487
2488         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2489             * ($conf->{cores} || $defaults->{cores});
2490         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2491         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2492
2493         $d->{name} = $conf->{name} || "VM $vmid";
2494         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2495             : $defaults->{memory}*(1024*1024);
2496
2497         if ($conf->{balloon}) {
2498             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2499             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2500                 : $defaults->{shares};
2501         }
2502
2503         $d->{uptime} = 0;
2504         $d->{cpu} = 0;
2505         $d->{mem} = 0;
2506
2507         $d->{netout} = 0;
2508         $d->{netin} = 0;
2509
2510         $d->{diskread} = 0;
2511         $d->{diskwrite} = 0;
2512
2513         $d->{template} = PVE::QemuConfig->is_template($conf);
2514
2515         $d->{serial} = 1 if conf_has_serial($conf);
2516         $d->{lock} = $conf->{lock} if $conf->{lock};
2517         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2518
2519         $res->{$vmid} = $d;
2520     }
2521
2522     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2523     foreach my $dev (keys %$netdev) {
2524         next if $dev !~ m/^tap([1-9]\d*)i/;
2525         my $vmid = $1;
2526         my $d = $res->{$vmid};
2527         next if !$d;
2528
2529         $d->{netout} += $netdev->{$dev}->{receive};
2530         $d->{netin} += $netdev->{$dev}->{transmit};
2531
2532         if ($full) {
2533             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2534             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2535         }
2536
2537     }
2538
2539     my $ctime = gettimeofday;
2540
2541     foreach my $vmid (keys %$list) {
2542
2543         my $d = $res->{$vmid};
2544         my $pid = $d->{pid};
2545         next if !$pid;
2546
2547         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2548         next if !$pstat; # not running
2549
2550         my $used = $pstat->{utime} + $pstat->{stime};
2551
2552         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2553
2554         if ($pstat->{vsize}) {
2555             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2556         }
2557
2558         my $old = $last_proc_pid_stat->{$pid};
2559         if (!$old) {
2560             $last_proc_pid_stat->{$pid} = {
2561                 time => $ctime,
2562                 used => $used,
2563                 cpu => 0,
2564             };
2565             next;
2566         }
2567
2568         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2569
2570         if ($dtime > 1000) {
2571             my $dutime = $used -  $old->{used};
2572
2573             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2574             $last_proc_pid_stat->{$pid} = {
2575                 time => $ctime,
2576                 used => $used,
2577                 cpu => $d->{cpu},
2578             };
2579         } else {
2580             $d->{cpu} = $old->{cpu};
2581         }
2582     }
2583
2584     return $res if !$full;
2585
2586     my $qmpclient = PVE::QMPClient->new();
2587
2588     my $ballooncb = sub {
2589         my ($vmid, $resp) = @_;
2590
2591         my $info = $resp->{'return'};
2592         return if !$info->{max_mem};
2593
2594         my $d = $res->{$vmid};
2595
2596         # use memory assigned to VM
2597         $d->{maxmem} = $info->{max_mem};
2598         $d->{balloon} = $info->{actual};
2599
2600         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2601             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2602             $d->{freemem} = $info->{free_mem};
2603         }
2604
2605         $d->{ballooninfo} = $info;
2606     };
2607
2608     my $blockstatscb = sub {
2609         my ($vmid, $resp) = @_;
2610         my $data = $resp->{'return'} || [];
2611         my $totalrdbytes = 0;
2612         my $totalwrbytes = 0;
2613
2614         for my $blockstat (@$data) {
2615             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2616             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2617
2618             $blockstat->{device} =~ s/drive-//;
2619             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2620         }
2621         $res->{$vmid}->{diskread} = $totalrdbytes;
2622         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2623     };
2624
2625     my $statuscb = sub {
2626         my ($vmid, $resp) = @_;
2627
2628         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2629         # this fails if ballon driver is not loaded, so this must be
2630         # the last commnand (following command are aborted if this fails).
2631         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2632
2633         my $status = 'unknown';
2634         if (!defined($status = $resp->{'return'}->{status})) {
2635             warn "unable to get VM status\n";
2636             return;
2637         }
2638
2639         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2640     };
2641
2642     foreach my $vmid (keys %$list) {
2643         next if $opt_vmid && ($vmid ne $opt_vmid);
2644         next if !$res->{$vmid}->{pid}; # not running
2645         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2646     }
2647
2648     $qmpclient->queue_execute(undef, 2);
2649
2650     foreach my $vmid (keys %$list) {
2651         next if $opt_vmid && ($vmid ne $opt_vmid);
2652         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2653     }
2654
2655     return $res;
2656 }
2657
2658 sub conf_has_serial {
2659     my ($conf) = @_;
2660
2661     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2662         if ($conf->{"serial$i"}) {
2663             return 1;
2664         }
2665     }
2666
2667     return 0;
2668 }
2669
2670 sub conf_has_audio {
2671     my ($conf, $id) = @_;
2672
2673     $id //= 0;
2674     my $audio = $conf->{"audio$id"};
2675     return undef if !defined($audio);
2676
2677     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2678     my $audiodriver = $audioproperties->{driver} // 'spice';
2679
2680     return {
2681         dev => $audioproperties->{device},
2682         dev_id => "audiodev$id",
2683         backend => $audiodriver,
2684         backend_id => "$audiodriver-backend${id}",
2685     };
2686 }
2687
2688 sub audio_devs {
2689     my ($audio, $audiopciaddr, $machine_version) = @_;
2690
2691     my $devs = [];
2692
2693     my $id = $audio->{dev_id};
2694     my $audiodev = "";
2695     if (min_version($machine_version, 4, 2)) {
2696         $audiodev = ",audiodev=$audio->{backend_id}";
2697     }
2698
2699     if ($audio->{dev} eq 'AC97') {
2700         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2701     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2702         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2703         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2704         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2705     } else {
2706         die "unkown audio device '$audio->{dev}', implement me!";
2707     }
2708
2709     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2710
2711     return $devs;
2712 }
2713
2714 sub vga_conf_has_spice {
2715     my ($vga) = @_;
2716
2717     my $vgaconf = parse_vga($vga);
2718     my $vgatype = $vgaconf->{type};
2719     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2720
2721     return $1 || 1;
2722 }
2723
2724 sub is_native($) {
2725     my ($arch) = @_;
2726     return get_host_arch() eq $arch;
2727 }
2728
2729 sub get_vm_arch {
2730     my ($conf) = @_;
2731     return $conf->{arch} // get_host_arch();
2732 }
2733
2734 my $default_machines = {
2735     x86_64 => 'pc',
2736     aarch64 => 'virt',
2737 };
2738
2739 sub get_vm_machine {
2740     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2741
2742     my $machine = $forcemachine || $conf->{machine};
2743
2744     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2745         $arch //= 'x86_64';
2746         $machine ||= $default_machines->{$arch};
2747         if ($add_pve_version) {
2748             $kvmversion //= kvm_user_version();
2749             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2750             $machine .= "+pve$pvever";
2751         }
2752     }
2753
2754     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2755         # for version-pinned machines that do not include a pve-version (e.g.
2756         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2757         $machine .= '+pve0';
2758     }
2759
2760     return $machine;
2761 }
2762
2763 sub get_ovmf_files($) {
2764     my ($arch) = @_;
2765
2766     my $ovmf = $OVMF->{$arch}
2767         or die "no OVMF images known for architecture '$arch'\n";
2768
2769     return @$ovmf;
2770 }
2771
2772 my $Arch2Qemu = {
2773     aarch64 => '/usr/bin/qemu-system-aarch64',
2774     x86_64 => '/usr/bin/qemu-system-x86_64',
2775 };
2776 sub get_command_for_arch($) {
2777     my ($arch) = @_;
2778     return '/usr/bin/kvm' if is_native($arch);
2779
2780     my $cmd = $Arch2Qemu->{$arch}
2781         or die "don't know how to emulate architecture '$arch'\n";
2782     return $cmd;
2783 }
2784
2785 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2786 # to use in a QEMU command line (-cpu element), first array_intersect the result
2787 # of query_supported_ with query_understood_. This is necessary because:
2788 #
2789 # a) query_understood_ returns flags the host cannot use and
2790 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2791 #    flags, but CPU settings - with most of them being flags. Those settings
2792 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2793 #
2794 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2795 # expensive. If you need the value returned from this, you can get it much
2796 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2797 # $accel being 'kvm' or 'tcg'.
2798 #
2799 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2800 # changes, automatically populating pmxcfs.
2801 #
2802 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2803 # since kvm and tcg machines support different flags
2804 #
2805 sub query_supported_cpu_flags {
2806     my ($arch) = @_;
2807
2808     $arch //= get_host_arch();
2809     my $default_machine = $default_machines->{$arch};
2810
2811     my $flags = {};
2812
2813     # FIXME: Once this is merged, the code below should work for ARM as well:
2814     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2815     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2816         $arch eq "aarch64";
2817
2818     my $kvm_supported = defined(kvm_version());
2819     my $qemu_cmd = get_command_for_arch($arch);
2820     my $fakevmid = -1;
2821     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2822
2823     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2824     my $query_supported_run_qemu = sub {
2825         my ($kvm) = @_;
2826
2827         my $flags = {};
2828         my $cmd = [
2829             $qemu_cmd,
2830             '-machine', $default_machine,
2831             '-display', 'none',
2832             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2833             '-mon', 'chardev=qmp,mode=control',
2834             '-pidfile', $pidfile,
2835             '-S', '-daemonize'
2836         ];
2837
2838         if (!$kvm) {
2839             push @$cmd, '-accel', 'tcg';
2840         }
2841
2842         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2843         die "QEMU flag querying VM exited with code " . $rc if $rc;
2844
2845         eval {
2846             my $cmd_result = mon_cmd(
2847                 $fakevmid,
2848                 'query-cpu-model-expansion',
2849                 type => 'full',
2850                 model => { name => 'host' }
2851             );
2852
2853             my $props = $cmd_result->{model}->{props};
2854             foreach my $prop (keys %$props) {
2855                 next if $props->{$prop} ne '1';
2856                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2857                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2858                 # We only keep those with underscores, to match /proc/cpuinfo
2859                 $prop =~ s/\.|-/_/g;
2860                 $flags->{$prop} = 1;
2861             }
2862         };
2863         my $err = $@;
2864
2865         # force stop with 10 sec timeout and 'nocheck'
2866         # always stop, even if QMP failed
2867         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2868
2869         die $err if $err;
2870
2871         return [ sort keys %$flags ];
2872     };
2873
2874     # We need to query QEMU twice, since KVM and TCG have different supported flags
2875     PVE::QemuConfig->lock_config($fakevmid, sub {
2876         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2877         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2878
2879         if ($kvm_supported) {
2880             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2881             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2882         }
2883     });
2884
2885     return $flags;
2886 }
2887
2888 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2889 my $understood_cpu_flag_dir = "/usr/share/kvm";
2890 sub query_understood_cpu_flags {
2891     my $arch = get_host_arch();
2892     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2893
2894     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2895         if ! -e $filepath;
2896
2897     my $raw = file_get_contents($filepath);
2898     $raw =~ s/^\s+|\s+$//g;
2899     my @flags = split(/\s+/, $raw);
2900
2901     return \@flags;
2902 }
2903
2904 sub config_to_command {
2905     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2906
2907     my $cmd = [];
2908     my $globalFlags = [];
2909     my $machineFlags = [];
2910     my $rtcFlags = [];
2911     my $devices = [];
2912     my $pciaddr = '';
2913     my $bridges = {};
2914     my $ostype = $conf->{ostype};
2915     my $winversion = windows_version($ostype);
2916     my $kvm = $conf->{kvm};
2917     my $nodename = nodename();
2918
2919     my $arch = get_vm_arch($conf);
2920     my $kvm_binary = get_command_for_arch($arch);
2921     my $kvmver = kvm_user_version($kvm_binary);
2922
2923     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2924         $kvmver //= "undefined";
2925         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2926     }
2927
2928     my $add_pve_version = min_version($kvmver, 4, 1);
2929
2930     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2931     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2932     $kvm //= 1 if is_native($arch);
2933
2934     $machine_version =~ m/(\d+)\.(\d+)/;
2935     my ($machine_major, $machine_minor) = ($1, $2);
2936
2937     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2938         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2939     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2940         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2941     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2942         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2943         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2944             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2945     }
2946
2947     # if a specific +pve version is required for a feature, use $version_guard
2948     # instead of min_version to allow machines to be run with the minimum
2949     # required version
2950     my $required_pve_version = 0;
2951     my $version_guard = sub {
2952         my ($major, $minor, $pve) = @_;
2953         return 0 if !min_version($machine_version, $major, $minor, $pve);
2954         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2955         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2956         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2957         return 1;
2958     };
2959
2960     if ($kvm) {
2961         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2962             if !defined kvm_version();
2963     }
2964
2965     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2966     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2967     my $use_old_bios_files = undef;
2968     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2969
2970     my $cpuunits = defined($conf->{cpuunits}) ?
2971             $conf->{cpuunits} : $defaults->{cpuunits};
2972
2973     push @$cmd, $kvm_binary;
2974
2975     push @$cmd, '-id', $vmid;
2976
2977     my $vmname = $conf->{name} || "vm$vmid";
2978
2979     push @$cmd, '-name', $vmname;
2980
2981     my $use_virtio = 0;
2982
2983     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2984     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2985     push @$cmd, '-mon', "chardev=qmp,mode=control";
2986
2987     if (min_version($machine_version, 2, 12)) {
2988         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2989         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2990     }
2991
2992     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
2993
2994     push @$cmd, '-daemonize';
2995
2996     if ($conf->{smbios1}) {
2997         my $smbios_conf = parse_smbios1($conf->{smbios1});
2998         if ($smbios_conf->{base64}) {
2999             # Do not pass base64 flag to qemu
3000             delete $smbios_conf->{base64};
3001             my $smbios_string = "";
3002             foreach my $key (keys %$smbios_conf) {
3003                 my $value;
3004                 if ($key eq "uuid") {
3005                     $value = $smbios_conf->{uuid}
3006                 } else {
3007                     $value = decode_base64($smbios_conf->{$key});
3008                 }
3009                 # qemu accepts any binary data, only commas need escaping by double comma
3010                 $value =~ s/,/,,/g;
3011                 $smbios_string .= "," . $key . "=" . $value if $value;
3012             }
3013             push @$cmd, '-smbios', "type=1" . $smbios_string;
3014         } else {
3015             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3016         }
3017     }
3018
3019     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3020         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3021         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3022
3023         my ($path, $format);
3024         if (my $efidisk = $conf->{efidisk0}) {
3025             my $d = parse_drive('efidisk0', $efidisk);
3026             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3027             $format = $d->{format};
3028             if ($storeid) {
3029                 $path = PVE::Storage::path($storecfg, $d->{file});
3030                 if (!defined($format)) {
3031                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3032                     $format = qemu_img_format($scfg, $volname);
3033                 }
3034             } else {
3035                 $path = $d->{file};
3036                 die "efidisk format must be specified\n"
3037                     if !defined($format);
3038             }
3039         } else {
3040             warn "no efidisk configured! Using temporary efivars disk.\n";
3041             $path = "/tmp/$vmid-ovmf.fd";
3042             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3043             $format = 'raw';
3044         }
3045
3046         my $size_str = "";
3047
3048         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3049             $size_str = ",size=" . (-s $ovmf_vars);
3050         }
3051
3052         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3053         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3054     }
3055
3056     # load q35 config
3057     if ($q35) {
3058         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3059         if (min_version($machine_version, 4, 0)) {
3060             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3061         } else {
3062             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3063         }
3064     }
3065
3066     if ($conf->{vmgenid}) {
3067         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3068     }
3069
3070     # add usb controllers
3071     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3072     push @$devices, @usbcontrollers if @usbcontrollers;
3073     my $vga = parse_vga($conf->{vga});
3074
3075     my $qxlnum = vga_conf_has_spice($conf->{vga});
3076     $vga->{type} = 'qxl' if $qxlnum;
3077
3078     if (!$vga->{type}) {
3079         if ($arch eq 'aarch64') {
3080             $vga->{type} = 'virtio';
3081         } elsif (min_version($machine_version, 2, 9)) {
3082             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3083         } else {
3084             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3085         }
3086     }
3087
3088     # enable absolute mouse coordinates (needed by vnc)
3089     my $tablet;
3090     if (defined($conf->{tablet})) {
3091         $tablet = $conf->{tablet};
3092     } else {
3093         $tablet = $defaults->{tablet};
3094         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3095         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3096     }
3097
3098     if ($tablet) {
3099         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3100         my $kbd = print_keyboarddevice_full($conf, $arch);
3101         push @$devices, '-device', $kbd if defined($kbd);
3102     }
3103
3104     # host pci device passthrough
3105     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3106         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3107
3108     # usb devices
3109     my $usb_dev_features = {};
3110     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3111
3112     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3113     push @$devices, @usbdevices if @usbdevices;
3114     # serial devices
3115     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3116         if (my $path = $conf->{"serial$i"}) {
3117             if ($path eq 'socket') {
3118                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3119                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3120                 # On aarch64, serial0 is the UART device. Qemu only allows
3121                 # connecting UART devices via the '-serial' command line, as
3122                 # the device has a fixed slot on the hardware...
3123                 if ($arch eq 'aarch64' && $i == 0) {
3124                     push @$devices, '-serial', "chardev:serial$i";
3125                 } else {
3126                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3127                 }
3128             } else {
3129                 die "no such serial device\n" if ! -c $path;
3130                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3131                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3132             }
3133         }
3134     }
3135
3136     # parallel devices
3137     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3138         if (my $path = $conf->{"parallel$i"}) {
3139             die "no such parallel device\n" if ! -c $path;
3140             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3141             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3142             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3143         }
3144     }
3145
3146     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3147         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3148         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3149         push @$devices, @$audio_devs;
3150     }
3151
3152     my $sockets = 1;
3153     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3154     $sockets = $conf->{sockets} if  $conf->{sockets};
3155
3156     my $cores = $conf->{cores} || 1;
3157
3158     my $maxcpus = $sockets * $cores;
3159
3160     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3161
3162     my $allowed_vcpus = $cpuinfo->{cpus};
3163
3164     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3165         if ($allowed_vcpus < $maxcpus);
3166
3167     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3168
3169         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3170         for (my $i = 2; $i <= $vcpus; $i++)  {
3171             my $cpustr = print_cpu_device($conf,$i);
3172             push @$cmd, '-device', $cpustr;
3173         }
3174
3175     } else {
3176
3177         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3178     }
3179     push @$cmd, '-nodefaults';
3180
3181     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3182
3183     my $bootindex_hash = {};
3184     my $i = 1;
3185     foreach my $o (split(//, $bootorder)) {
3186         $bootindex_hash->{$o} = $i*100;
3187         $i++;
3188     }
3189
3190     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3191
3192     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3193
3194     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3195
3196     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3197         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3198         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3199         push @$cmd,  '-vnc', "unix:$socket,password";
3200     } else {
3201         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3202         push @$cmd, '-nographic';
3203     }
3204
3205     # time drift fix
3206     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3207     my $useLocaltime = $conf->{localtime};
3208
3209     if ($winversion >= 5) { # windows
3210         $useLocaltime = 1 if !defined($conf->{localtime});
3211
3212         # use time drift fix when acpi is enabled
3213         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3214             $tdf = 1 if !defined($conf->{tdf});
3215         }
3216     }
3217
3218     if ($winversion >= 6) {
3219         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3220         push @$cmd, '-no-hpet';
3221     }
3222
3223     push @$rtcFlags, 'driftfix=slew' if $tdf;
3224
3225     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3226         push @$rtcFlags, "base=$conf->{startdate}";
3227     } elsif ($useLocaltime) {
3228         push @$rtcFlags, 'base=localtime';
3229     }
3230
3231     if ($forcecpu) {
3232         push @$cmd, '-cpu', $forcecpu;
3233     } else {
3234         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3235     }
3236
3237     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3238
3239     push @$cmd, '-S' if $conf->{freeze};
3240
3241     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3242
3243     my $guest_agent = parse_guest_agent($conf);
3244
3245     if ($guest_agent->{enabled}) {
3246         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3247         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3248
3249         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3250             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3251             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3252             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3253         } elsif ($guest_agent->{type} eq 'isa') {
3254             push @$devices, '-device', "isa-serial,chardev=qga0";
3255         }
3256     }
3257
3258     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3259     if ($rng && &$version_guard(4, 1, 2)) {
3260         check_rng_source($rng->{source});
3261
3262         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3263         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3264         my $limiter_str = "";
3265         if ($max_bytes) {
3266             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3267         }
3268
3269         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3270         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3271         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3272     }
3273
3274     my $spice_port;
3275
3276     if ($qxlnum) {
3277         if ($qxlnum > 1) {
3278             if ($winversion){
3279                 for (my $i = 1; $i < $qxlnum; $i++){
3280                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3281                 }
3282             } else {
3283                 # assume other OS works like Linux
3284                 my ($ram, $vram) = ("134217728", "67108864");
3285                 if ($vga->{memory}) {
3286                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3287                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3288                 }
3289                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3290                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3291             }
3292         }
3293
3294         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3295
3296         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3297         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3298         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3299
3300         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3301         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3302         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3303
3304         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3305         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3306
3307         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3308         if ($spice_enhancement->{foldersharing}) {
3309             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3310             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3311         }
3312
3313         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3314         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3315         push @$devices, '-spice', "$spice_opts";
3316     }
3317
3318     # enable balloon by default, unless explicitly disabled
3319     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3320         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3321         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3322     }
3323
3324     if ($conf->{watchdog}) {
3325         my $wdopts = parse_watchdog($conf->{watchdog});
3326         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3327         my $watchdog = $wdopts->{model} || 'i6300esb';
3328         push @$devices, '-device', "$watchdog$pciaddr";
3329         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3330     }
3331
3332     my $vollist = [];
3333     my $scsicontroller = {};
3334     my $ahcicontroller = {};
3335     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3336
3337     # Add iscsi initiator name if available
3338     if (my $initiator = get_initiator_name()) {
3339         push @$devices, '-iscsi', "initiator-name=$initiator";
3340     }
3341
3342     PVE::QemuConfig->foreach_volume($conf, sub {
3343         my ($ds, $drive) = @_;
3344
3345         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3346             push @$vollist, $drive->{file};
3347         }
3348
3349         # ignore efidisk here, already added in bios/fw handling code above
3350         return if $drive->{interface} eq 'efidisk';
3351
3352         $use_virtio = 1 if $ds =~ m/^virtio/;
3353
3354         if (drive_is_cdrom ($drive)) {
3355             if ($bootindex_hash->{d}) {
3356                 $drive->{bootindex} = $bootindex_hash->{d};
3357                 $bootindex_hash->{d} += 1;
3358             }
3359         } else {
3360             if ($bootindex_hash->{c}) {
3361                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3362                 $bootindex_hash->{c} += 1;
3363             }
3364         }
3365
3366         if ($drive->{interface} eq 'virtio'){
3367            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3368         }
3369
3370         if ($drive->{interface} eq 'scsi') {
3371
3372             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3373
3374             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3375                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3376
3377             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3378             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3379
3380             my $iothread = '';
3381             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3382                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3383                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3384             } elsif ($drive->{iothread}) {
3385                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3386             }
3387
3388             my $queues = '';
3389             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3390                 $queues = ",num_queues=$drive->{queues}";
3391             }
3392
3393             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3394             $scsicontroller->{$controller}=1;
3395         }
3396
3397         if ($drive->{interface} eq 'sata') {
3398             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3399             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3400             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3401             $ahcicontroller->{$controller}=1;
3402         }
3403
3404         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3405         push @$devices, '-drive',$drive_cmd;
3406         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3407     });
3408
3409     for (my $i = 0; $i < $MAX_NETS; $i++) {
3410          next if !$conf->{"net$i"};
3411          my $d = parse_net($conf->{"net$i"});
3412          next if !$d;
3413
3414          $use_virtio = 1 if $d->{model} eq 'virtio';
3415
3416          if ($bootindex_hash->{n}) {
3417             $d->{bootindex} = $bootindex_hash->{n};
3418             $bootindex_hash->{n} += 1;
3419          }
3420
3421          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3422          push @$devices, '-netdev', $netdevfull;
3423
3424          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3425          push @$devices, '-device', $netdevicefull;
3426     }
3427
3428     if ($conf->{ivshmem}) {
3429         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3430
3431         my $bus;
3432         if ($q35) {
3433             $bus = print_pcie_addr("ivshmem");
3434         } else {
3435             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3436         }
3437
3438         my $ivshmem_name = $ivshmem->{name} // $vmid;
3439         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3440
3441         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3442         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3443     }
3444
3445     # pci.4 is nested in pci.1
3446     $bridges->{1} = 1 if $bridges->{4};
3447
3448     if (!$q35) {
3449         # add pci bridges
3450         if (min_version($machine_version, 2, 3)) {
3451            $bridges->{1} = 1;
3452            $bridges->{2} = 1;
3453         }
3454
3455         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3456
3457     }
3458
3459     for my $k (sort {$b cmp $a} keys %$bridges) {
3460         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3461
3462         my $k_name = $k;
3463         if ($k == 2 && $legacy_igd) {
3464             $k_name = "$k-igd";
3465         }
3466         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3467
3468         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3469         if ($q35) {
3470             # add after -readconfig pve-q35.cfg
3471             splice @$devices, 2, 0, '-device', $devstr;
3472         } else {
3473             unshift @$devices, '-device', $devstr if $k > 0;
3474         }
3475     }
3476
3477     if (!$kvm) {
3478         push @$machineFlags, 'accel=tcg';
3479     }
3480
3481     my $machine_type_min = $machine_type;
3482     if ($add_pve_version) {
3483         $machine_type_min =~ s/\+pve\d+$//;
3484         $machine_type_min .= "+pve$required_pve_version";
3485     }
3486     push @$machineFlags, "type=${machine_type_min}";
3487
3488     push @$cmd, @$devices;
3489     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3490     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3491     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3492
3493     if (my $vmstate = $conf->{vmstate}) {
3494         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3495         push @$vollist, $vmstate;
3496         push @$cmd, '-loadstate', $statepath;
3497         print "activating and using '$vmstate' as vmstate\n";
3498     }
3499
3500     # add custom args
3501     if ($conf->{args}) {
3502         my $aa = PVE::Tools::split_args($conf->{args});
3503         push @$cmd, @$aa;
3504     }
3505
3506     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3507 }
3508
3509 sub check_rng_source {
3510     my ($source) = @_;
3511
3512     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3513     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3514         if ! -e $source;
3515
3516     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3517     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3518         # Needs to abort, otherwise QEMU crashes on first rng access.
3519         # Note that rng_current cannot be changed to 'none' manually, so
3520         # once the VM is past this point, it is no longer an issue.
3521         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3522             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3523             . " a compatible hardware-RNG is attached to the host.\n";
3524     }
3525 }
3526
3527 sub spice_port {
3528     my ($vmid) = @_;
3529
3530     my $res = mon_cmd($vmid, 'query-spice');
3531
3532     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3533 }
3534
3535 sub vm_devices_list {
3536     my ($vmid) = @_;
3537
3538     my $res = mon_cmd($vmid, 'query-pci');
3539     my $devices_to_check = [];
3540     my $devices = {};
3541     foreach my $pcibus (@$res) {
3542         push @$devices_to_check, @{$pcibus->{devices}},
3543     }
3544
3545     while (@$devices_to_check) {
3546         my $to_check = [];
3547         for my $d (@$devices_to_check) {
3548             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3549             next if !$d->{'pci_bridge'};
3550
3551             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3552             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3553         }
3554         $devices_to_check = $to_check;
3555     }
3556
3557     my $resblock = mon_cmd($vmid, 'query-block');
3558     foreach my $block (@$resblock) {
3559         if($block->{device} =~ m/^drive-(\S+)/){
3560                 $devices->{$1} = 1;
3561         }
3562     }
3563
3564     my $resmice = mon_cmd($vmid, 'query-mice');
3565     foreach my $mice (@$resmice) {
3566         if ($mice->{name} eq 'QEMU HID Tablet') {
3567             $devices->{tablet} = 1;
3568             last;
3569         }
3570     }
3571
3572     # for usb devices there is no query-usb
3573     # but we can iterate over the entries in
3574     # qom-list path=/machine/peripheral
3575     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3576     foreach my $per (@$resperipheral) {
3577         if ($per->{name} =~ m/^usb\d+$/) {
3578             $devices->{$per->{name}} = 1;
3579         }
3580     }
3581
3582     return $devices;
3583 }
3584
3585 sub vm_deviceplug {
3586     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3587
3588     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3589
3590     my $devices_list = vm_devices_list($vmid);
3591     return 1 if defined($devices_list->{$deviceid});
3592
3593     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3594
3595     if ($deviceid eq 'tablet') {
3596
3597         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3598
3599     } elsif ($deviceid eq 'keyboard') {
3600
3601         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3602
3603     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3604
3605         die "usb hotplug currently not reliable\n";
3606         # since we can't reliably hot unplug all added usb devices
3607         # and usb passthrough disables live migration
3608         # we disable usb hotplugging for now
3609         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3610
3611     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3612
3613         qemu_iothread_add($vmid, $deviceid, $device);
3614
3615         qemu_driveadd($storecfg, $vmid, $device);
3616         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3617
3618         qemu_deviceadd($vmid, $devicefull);
3619         eval { qemu_deviceaddverify($vmid, $deviceid); };
3620         if (my $err = $@) {
3621             eval { qemu_drivedel($vmid, $deviceid); };
3622             warn $@ if $@;
3623             die $err;
3624         }
3625
3626     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3627
3628
3629         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3630         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3631         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3632
3633         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3634
3635         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3636             qemu_iothread_add($vmid, $deviceid, $device);
3637             $devicefull .= ",iothread=iothread-$deviceid";
3638         }
3639
3640         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3641             $devicefull .= ",num_queues=$device->{queues}";
3642         }
3643
3644         qemu_deviceadd($vmid, $devicefull);
3645         qemu_deviceaddverify($vmid, $deviceid);
3646
3647     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3648
3649         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3650         qemu_driveadd($storecfg, $vmid, $device);
3651
3652         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3653         eval { qemu_deviceadd($vmid, $devicefull); };
3654         if (my $err = $@) {
3655             eval { qemu_drivedel($vmid, $deviceid); };
3656             warn $@ if $@;
3657             die $err;
3658         }
3659
3660     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3661
3662         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3663
3664         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3665         my $use_old_bios_files = undef;
3666         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3667
3668         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3669         qemu_deviceadd($vmid, $netdevicefull);
3670         eval {
3671             qemu_deviceaddverify($vmid, $deviceid);
3672             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3673         };
3674         if (my $err = $@) {
3675             eval { qemu_netdevdel($vmid, $deviceid); };
3676             warn $@ if $@;
3677             die $err;
3678         }
3679
3680     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3681
3682         my $bridgeid = $2;
3683         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3684         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3685
3686         qemu_deviceadd($vmid, $devicefull);
3687         qemu_deviceaddverify($vmid, $deviceid);
3688
3689     } else {
3690         die "can't hotplug device '$deviceid'\n";
3691     }
3692
3693     return 1;
3694 }
3695
3696 # fixme: this should raise exceptions on error!
3697 sub vm_deviceunplug {
3698     my ($vmid, $conf, $deviceid) = @_;
3699
3700     my $devices_list = vm_devices_list($vmid);
3701     return 1 if !defined($devices_list->{$deviceid});
3702
3703     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3704
3705     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3706
3707         qemu_devicedel($vmid, $deviceid);
3708
3709     } elsif ($deviceid =~ m/^usb\d+$/) {
3710
3711         die "usb hotplug currently not reliable\n";
3712         # when unplugging usb devices this way,
3713         # there may be remaining usb controllers/hubs
3714         # so we disable it for now
3715         qemu_devicedel($vmid, $deviceid);
3716         qemu_devicedelverify($vmid, $deviceid);
3717
3718     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3719
3720         qemu_devicedel($vmid, $deviceid);
3721         qemu_devicedelverify($vmid, $deviceid);
3722         qemu_drivedel($vmid, $deviceid);
3723         qemu_iothread_del($conf, $vmid, $deviceid);
3724
3725     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3726
3727         qemu_devicedel($vmid, $deviceid);
3728         qemu_devicedelverify($vmid, $deviceid);
3729         qemu_iothread_del($conf, $vmid, $deviceid);
3730
3731     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3732
3733         qemu_devicedel($vmid, $deviceid);
3734         qemu_drivedel($vmid, $deviceid);
3735         qemu_deletescsihw($conf, $vmid, $deviceid);
3736
3737     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3738
3739         qemu_devicedel($vmid, $deviceid);
3740         qemu_devicedelverify($vmid, $deviceid);
3741         qemu_netdevdel($vmid, $deviceid);
3742
3743     } else {
3744         die "can't unplug device '$deviceid'\n";
3745     }
3746
3747     return 1;
3748 }
3749
3750 sub qemu_deviceadd {
3751     my ($vmid, $devicefull) = @_;
3752
3753     $devicefull = "driver=".$devicefull;
3754     my %options =  split(/[=,]/, $devicefull);
3755
3756     mon_cmd($vmid, "device_add" , %options);
3757 }
3758
3759 sub qemu_devicedel {
3760     my ($vmid, $deviceid) = @_;
3761
3762     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3763 }
3764
3765 sub qemu_iothread_add {
3766     my($vmid, $deviceid, $device) = @_;
3767
3768     if ($device->{iothread}) {
3769         my $iothreads = vm_iothreads_list($vmid);
3770         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3771     }
3772 }
3773
3774 sub qemu_iothread_del {
3775     my($conf, $vmid, $deviceid) = @_;
3776
3777     my $confid = $deviceid;
3778     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3779         $confid = 'scsi' . $1;
3780     }
3781     my $device = parse_drive($confid, $conf->{$confid});
3782     if ($device->{iothread}) {
3783         my $iothreads = vm_iothreads_list($vmid);
3784         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3785     }
3786 }
3787
3788 sub qemu_objectadd {
3789     my($vmid, $objectid, $qomtype) = @_;
3790
3791     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3792
3793     return 1;
3794 }
3795
3796 sub qemu_objectdel {
3797     my($vmid, $objectid) = @_;
3798
3799     mon_cmd($vmid, "object-del", id => $objectid);
3800
3801     return 1;
3802 }
3803
3804 sub qemu_driveadd {
3805     my ($storecfg, $vmid, $device) = @_;
3806
3807     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3808     $drive =~ s/\\/\\\\/g;
3809     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3810
3811     # If the command succeeds qemu prints: "OK"
3812     return 1 if $ret =~ m/OK/s;
3813
3814     die "adding drive failed: $ret\n";
3815 }
3816
3817 sub qemu_drivedel {
3818     my($vmid, $deviceid) = @_;
3819
3820     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3821     $ret =~ s/^\s+//;
3822
3823     return 1 if $ret eq "";
3824
3825     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3826     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3827
3828     die "deleting drive $deviceid failed : $ret\n";
3829 }
3830
3831 sub qemu_deviceaddverify {
3832     my ($vmid, $deviceid) = @_;
3833
3834     for (my $i = 0; $i <= 5; $i++) {
3835          my $devices_list = vm_devices_list($vmid);
3836          return 1 if defined($devices_list->{$deviceid});
3837          sleep 1;
3838     }
3839
3840     die "error on hotplug device '$deviceid'\n";
3841 }
3842
3843
3844 sub qemu_devicedelverify {
3845     my ($vmid, $deviceid) = @_;
3846
3847     # need to verify that the device is correctly removed as device_del
3848     # is async and empty return is not reliable
3849
3850     for (my $i = 0; $i <= 5; $i++) {
3851          my $devices_list = vm_devices_list($vmid);
3852          return 1 if !defined($devices_list->{$deviceid});
3853          sleep 1;
3854     }
3855
3856     die "error on hot-unplugging device '$deviceid'\n";
3857 }
3858
3859 sub qemu_findorcreatescsihw {
3860     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3861
3862     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3863
3864     my $scsihwid="$controller_prefix$controller";
3865     my $devices_list = vm_devices_list($vmid);
3866
3867     if(!defined($devices_list->{$scsihwid})) {
3868         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3869     }
3870
3871     return 1;
3872 }
3873
3874 sub qemu_deletescsihw {
3875     my ($conf, $vmid, $opt) = @_;
3876
3877     my $device = parse_drive($opt, $conf->{$opt});
3878
3879     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3880         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3881         return 1;
3882     }
3883
3884     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3885
3886     my $devices_list = vm_devices_list($vmid);
3887     foreach my $opt (keys %{$devices_list}) {
3888         if (is_valid_drivename($opt)) {
3889             my $drive = parse_drive($opt, $conf->{$opt});
3890             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3891                 return 1;
3892             }
3893         }
3894     }
3895
3896     my $scsihwid="scsihw$controller";
3897
3898     vm_deviceunplug($vmid, $conf, $scsihwid);
3899
3900     return 1;
3901 }
3902
3903 sub qemu_add_pci_bridge {
3904     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3905
3906     my $bridges = {};
3907
3908     my $bridgeid;
3909
3910     print_pci_addr($device, $bridges, $arch, $machine_type);
3911
3912     while (my ($k, $v) = each %$bridges) {
3913         $bridgeid = $k;
3914     }
3915     return 1 if !defined($bridgeid) || $bridgeid < 1;
3916
3917     my $bridge = "pci.$bridgeid";
3918     my $devices_list = vm_devices_list($vmid);
3919
3920     if (!defined($devices_list->{$bridge})) {
3921         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3922     }
3923
3924     return 1;
3925 }
3926
3927 sub qemu_set_link_status {
3928     my ($vmid, $device, $up) = @_;
3929
3930     mon_cmd($vmid, "set_link", name => $device,
3931                up => $up ? JSON::true : JSON::false);
3932 }
3933
3934 sub qemu_netdevadd {
3935     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3936
3937     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3938     my %options =  split(/[=,]/, $netdev);
3939
3940     if (defined(my $vhost = $options{vhost})) {
3941         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3942     }
3943
3944     if (defined(my $queues = $options{queues})) {
3945         $options{queues} = $queues + 0;
3946     }
3947
3948     mon_cmd($vmid, "netdev_add",  %options);
3949     return 1;
3950 }
3951
3952 sub qemu_netdevdel {
3953     my ($vmid, $deviceid) = @_;
3954
3955     mon_cmd($vmid, "netdev_del", id => $deviceid);
3956 }
3957
3958 sub qemu_usb_hotplug {
3959     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3960
3961     return if !$device;
3962
3963     # remove the old one first
3964     vm_deviceunplug($vmid, $conf, $deviceid);
3965
3966     # check if xhci controller is necessary and available
3967     if ($device->{usb3}) {
3968
3969         my $devicelist = vm_devices_list($vmid);
3970
3971         if (!$devicelist->{xhci}) {
3972             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
3973             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3974         }
3975     }
3976     my $d = parse_usb_device($device->{host});
3977     $d->{usb3} = $device->{usb3};
3978
3979     # add the new one
3980     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3981 }
3982
3983 sub qemu_cpu_hotplug {
3984     my ($vmid, $conf, $vcpus) = @_;
3985
3986     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
3987
3988     my $sockets = 1;
3989     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3990     $sockets = $conf->{sockets} if  $conf->{sockets};
3991     my $cores = $conf->{cores} || 1;
3992     my $maxcpus = $sockets * $cores;
3993
3994     $vcpus = $maxcpus if !$vcpus;
3995
3996     die "you can't add more vcpus than maxcpus\n"
3997         if $vcpus > $maxcpus;
3998
3999     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4000
4001     if ($vcpus < $currentvcpus) {
4002
4003         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4004
4005             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4006                 qemu_devicedel($vmid, "cpu$i");
4007                 my $retry = 0;
4008                 my $currentrunningvcpus = undef;
4009                 while (1) {
4010                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4011                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4012                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4013                     $retry++;
4014                     sleep 1;
4015                 }
4016                 #update conf after each succesfull cpu unplug
4017                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4018                 PVE::QemuConfig->write_config($vmid, $conf);
4019             }
4020         } else {
4021             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4022         }
4023
4024         return;
4025     }
4026
4027     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4028     die "vcpus in running vm does not match its configuration\n"
4029         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4030
4031     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4032
4033         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4034             my $cpustr = print_cpu_device($conf, $i);
4035             qemu_deviceadd($vmid, $cpustr);
4036
4037             my $retry = 0;
4038             my $currentrunningvcpus = undef;
4039             while (1) {
4040                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4041                 last if scalar(@{$currentrunningvcpus}) == $i;
4042                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4043                 sleep 1;
4044                 $retry++;
4045             }
4046             #update conf after each succesfull cpu hotplug
4047             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4048             PVE::QemuConfig->write_config($vmid, $conf);
4049         }
4050     } else {
4051
4052         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4053             mon_cmd($vmid, "cpu-add", id => int($i));
4054         }
4055     }
4056 }
4057
4058 sub qemu_block_set_io_throttle {
4059     my ($vmid, $deviceid,
4060         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4061         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4062         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4063         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4064
4065     return if !check_running($vmid) ;
4066
4067     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4068         bps => int($bps),
4069         bps_rd => int($bps_rd),
4070         bps_wr => int($bps_wr),
4071         iops => int($iops),
4072         iops_rd => int($iops_rd),
4073         iops_wr => int($iops_wr),
4074         bps_max => int($bps_max),
4075         bps_rd_max => int($bps_rd_max),
4076         bps_wr_max => int($bps_wr_max),
4077         iops_max => int($iops_max),
4078         iops_rd_max => int($iops_rd_max),
4079         iops_wr_max => int($iops_wr_max),
4080         bps_max_length => int($bps_max_length),
4081         bps_rd_max_length => int($bps_rd_max_length),
4082         bps_wr_max_length => int($bps_wr_max_length),
4083         iops_max_length => int($iops_max_length),
4084         iops_rd_max_length => int($iops_rd_max_length),
4085         iops_wr_max_length => int($iops_wr_max_length),
4086     );
4087
4088 }
4089
4090 # old code, only used to shutdown old VM after update
4091 sub __read_avail {
4092     my ($fh, $timeout) = @_;
4093
4094     my $sel = new IO::Select;
4095     $sel->add($fh);
4096
4097     my $res = '';
4098     my $buf;
4099
4100     my @ready;
4101     while (scalar (@ready = $sel->can_read($timeout))) {
4102         my $count;
4103         if ($count = $fh->sysread($buf, 8192)) {
4104             if ($buf =~ /^(.*)\(qemu\) $/s) {
4105                 $res .= $1;
4106                 last;
4107             } else {
4108                 $res .= $buf;
4109             }
4110         } else {
4111             if (!defined($count)) {
4112                 die "$!\n";
4113             }
4114             last;
4115         }
4116     }
4117
4118     die "monitor read timeout\n" if !scalar(@ready);
4119
4120     return $res;
4121 }
4122
4123 sub qemu_block_resize {
4124     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4125
4126     my $running = check_running($vmid);
4127
4128     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4129
4130     return if !$running;
4131
4132     my $padding = (1024 - $size % 1024) % 1024;
4133     $size = $size + $padding;
4134
4135     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4136
4137 }
4138
4139 sub qemu_volume_snapshot {
4140     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4141
4142     my $running = check_running($vmid);
4143
4144     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4145         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4146     } else {
4147         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4148     }
4149 }
4150
4151 sub qemu_volume_snapshot_delete {
4152     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4153
4154     my $running = check_running($vmid);
4155
4156     if($running) {
4157
4158         $running = undef;
4159         my $conf = PVE::QemuConfig->load_config($vmid);
4160         PVE::QemuConfig->foreach_volume($conf, sub {
4161             my ($ds, $drive) = @_;
4162             $running = 1 if $drive->{file} eq $volid;
4163         });
4164     }
4165
4166     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4167         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4168     } else {
4169         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4170     }
4171 }
4172
4173 sub set_migration_caps {
4174     my ($vmid) = @_;
4175
4176     my $cap_ref = [];
4177
4178     my $enabled_cap = {
4179         "auto-converge" => 1,
4180         "xbzrle" => 1,
4181         "x-rdma-pin-all" => 0,
4182         "zero-blocks" => 0,
4183         "compress" => 0
4184     };
4185
4186     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4187
4188     for my $supported_capability (@$supported_capabilities) {
4189         push @$cap_ref, {
4190             capability => $supported_capability->{capability},
4191             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4192         };
4193     }
4194
4195     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4196 }
4197
4198 sub foreach_volid {
4199     my ($conf, $func, @param) = @_;
4200
4201     my $volhash = {};
4202
4203     my $test_volid = sub {
4204         my ($key, $drive, $snapname) = @_;
4205
4206         my $volid = $drive->{file};
4207         return if !$volid;
4208
4209         $volhash->{$volid}->{cdrom} //= 1;
4210         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4211
4212         my $replicate = $drive->{replicate} // 1;
4213         $volhash->{$volid}->{replicate} //= 0;
4214         $volhash->{$volid}->{replicate} = 1 if $replicate;
4215
4216         $volhash->{$volid}->{shared} //= 0;
4217         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4218
4219         $volhash->{$volid}->{referenced_in_config} //= 0;
4220         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4221
4222         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4223             if defined($snapname);
4224
4225         my $size = $drive->{size};
4226         $volhash->{$volid}->{size} //= $size if $size;
4227
4228         $volhash->{$volid}->{is_vmstate} //= 0;
4229         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4230
4231         $volhash->{$volid}->{is_unused} //= 0;
4232         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4233     };
4234
4235     my $include_opts = {
4236         extra_keys => ['vmstate'],
4237         include_unused => 1,
4238     };
4239
4240     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4241     foreach my $snapname (keys %{$conf->{snapshots}}) {
4242         my $snap = $conf->{snapshots}->{$snapname};
4243         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4244     }
4245
4246     foreach my $volid (keys %$volhash) {
4247         &$func($volid, $volhash->{$volid}, @param);
4248     }
4249 }
4250
4251 my $fast_plug_option = {
4252     'lock' => 1,
4253     'name' => 1,
4254     'onboot' => 1,
4255     'shares' => 1,
4256     'startup' => 1,
4257     'description' => 1,
4258     'protection' => 1,
4259     'vmstatestorage' => 1,
4260     'hookscript' => 1,
4261     'tags' => 1,
4262 };
4263
4264 # hotplug changes in [PENDING]
4265 # $selection hash can be used to only apply specified options, for
4266 # example: { cores => 1 } (only apply changed 'cores')
4267 # $errors ref is used to return error messages
4268 sub vmconfig_hotplug_pending {
4269     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4270
4271     my $defaults = load_defaults();
4272     my $arch = get_vm_arch($conf);
4273     my $machine_type = get_vm_machine($conf, undef, $arch);
4274
4275     # commit values which do not have any impact on running VM first
4276     # Note: those option cannot raise errors, we we do not care about
4277     # $selection and always apply them.
4278
4279     my $add_error = sub {
4280         my ($opt, $msg) = @_;
4281         $errors->{$opt} = "hotplug problem - $msg";
4282     };
4283
4284     my $changes = 0;
4285     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4286         if ($fast_plug_option->{$opt}) {
4287             $conf->{$opt} = $conf->{pending}->{$opt};
4288             delete $conf->{pending}->{$opt};
4289             $changes = 1;
4290         }
4291     }
4292
4293     if ($changes) {
4294         PVE::QemuConfig->write_config($vmid, $conf);
4295     }
4296
4297     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4298
4299     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4300     foreach my $opt (sort keys %$pending_delete_hash) {
4301         next if $selection && !$selection->{$opt};
4302         my $force = $pending_delete_hash->{$opt}->{force};
4303         eval {
4304             if ($opt eq 'hotplug') {
4305                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4306             } elsif ($opt eq 'tablet') {
4307                 die "skip\n" if !$hotplug_features->{usb};
4308                 if ($defaults->{tablet}) {
4309                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4310                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4311                         if $arch eq 'aarch64';
4312                 } else {
4313                     vm_deviceunplug($vmid, $conf, 'tablet');
4314                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4315                 }
4316             } elsif ($opt =~ m/^usb\d+/) {
4317                 die "skip\n";
4318                 # since we cannot reliably hot unplug usb devices
4319                 # we are disabling it
4320                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4321                 vm_deviceunplug($vmid, $conf, $opt);
4322             } elsif ($opt eq 'vcpus') {
4323                 die "skip\n" if !$hotplug_features->{cpu};
4324                 qemu_cpu_hotplug($vmid, $conf, undef);
4325             } elsif ($opt eq 'balloon') {
4326                 # enable balloon device is not hotpluggable
4327                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4328                 # here we reset the ballooning value to memory
4329                 my $balloon = $conf->{memory} || $defaults->{memory};
4330                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4331             } elsif ($fast_plug_option->{$opt}) {
4332                 # do nothing
4333             } elsif ($opt =~ m/^net(\d+)$/) {
4334                 die "skip\n" if !$hotplug_features->{network};
4335                 vm_deviceunplug($vmid, $conf, $opt);
4336             } elsif (is_valid_drivename($opt)) {
4337                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4338                 vm_deviceunplug($vmid, $conf, $opt);
4339                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4340             } elsif ($opt =~ m/^memory$/) {
4341                 die "skip\n" if !$hotplug_features->{memory};
4342                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4343             } elsif ($opt eq 'cpuunits') {
4344                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4345             } elsif ($opt eq 'cpulimit') {
4346                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4347             } else {
4348                 die "skip\n";
4349             }
4350         };
4351         if (my $err = $@) {
4352             &$add_error($opt, $err) if $err ne "skip\n";
4353         } else {
4354             delete $conf->{$opt};
4355             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4356         }
4357     }
4358
4359     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4360     $apply_pending_cloudinit = sub {
4361         return if $apply_pending_cloudinit_done; # once is enough
4362         $apply_pending_cloudinit_done = 1; # once is enough
4363
4364         my ($key, $value) = @_;
4365
4366         my @cloudinit_opts = keys %$confdesc_cloudinit;
4367         foreach my $opt (keys %{$conf->{pending}}) {
4368             next if !grep { $_ eq $opt } @cloudinit_opts;
4369             $conf->{$opt} = delete $conf->{pending}->{$opt};
4370         }
4371
4372         my $new_conf = { %$conf };
4373         $new_conf->{$key} = $value;
4374         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4375     };
4376
4377     foreach my $opt (keys %{$conf->{pending}}) {
4378         next if $selection && !$selection->{$opt};
4379         my $value = $conf->{pending}->{$opt};
4380         eval {
4381             if ($opt eq 'hotplug') {
4382                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4383             } elsif ($opt eq 'tablet') {
4384                 die "skip\n" if !$hotplug_features->{usb};
4385                 if ($value == 1) {
4386                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4387                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4388                         if $arch eq 'aarch64';
4389                 } elsif ($value == 0) {
4390                     vm_deviceunplug($vmid, $conf, 'tablet');
4391                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4392                 }
4393             } elsif ($opt =~ m/^usb\d+$/) {
4394                 die "skip\n";
4395                 # since we cannot reliably hot unplug usb devices
4396                 # we are disabling it
4397                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4398                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4399                 die "skip\n" if !$d;
4400                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4401             } elsif ($opt eq 'vcpus') {
4402                 die "skip\n" if !$hotplug_features->{cpu};
4403                 qemu_cpu_hotplug($vmid, $conf, $value);
4404             } elsif ($opt eq 'balloon') {
4405                 # enable/disable balloning device is not hotpluggable
4406                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4407                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4408                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4409
4410                 # allow manual ballooning if shares is set to zero
4411                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4412                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4413                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4414                 }
4415             } elsif ($opt =~ m/^net(\d+)$/) {
4416                 # some changes can be done without hotplug
4417                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4418                                     $vmid, $opt, $value, $arch, $machine_type);
4419             } elsif (is_valid_drivename($opt)) {
4420                 die "skip\n" if $opt eq 'efidisk0';
4421                 # some changes can be done without hotplug
4422                 my $drive = parse_drive($opt, $value);
4423                 if (drive_is_cloudinit($drive)) {
4424                     &$apply_pending_cloudinit($opt, $value);
4425                 }
4426                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4427                                      $vmid, $opt, $value, $arch, $machine_type);
4428             } elsif ($opt =~ m/^memory$/) { #dimms
4429                 die "skip\n" if !$hotplug_features->{memory};
4430                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4431             } elsif ($opt eq 'cpuunits') {
4432                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4433             } elsif ($opt eq 'cpulimit') {
4434                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4435                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4436             } else {
4437                 die "skip\n";  # skip non-hot-pluggable options
4438             }
4439         };
4440         if (my $err = $@) {
4441             &$add_error($opt, $err) if $err ne "skip\n";
4442         } else {
4443             $conf->{$opt} = $value;
4444             delete $conf->{pending}->{$opt};
4445         }
4446     }
4447
4448     PVE::QemuConfig->write_config($vmid, $conf);
4449 }
4450
4451 sub try_deallocate_drive {
4452     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4453
4454     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4455         my $volid = $drive->{file};
4456         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4457             my $sid = PVE::Storage::parse_volume_id($volid);
4458             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4459
4460             # check if the disk is really unused
4461             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4462                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4463             PVE::Storage::vdisk_free($storecfg, $volid);
4464             return 1;
4465         } else {
4466             # If vm is not owner of this disk remove from config
4467             return 1;
4468         }
4469     }
4470
4471     return undef;
4472 }
4473
4474 sub vmconfig_delete_or_detach_drive {
4475     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4476
4477     my $drive = parse_drive($opt, $conf->{$opt});
4478
4479     my $rpcenv = PVE::RPCEnvironment::get();
4480     my $authuser = $rpcenv->get_user();
4481
4482     if ($force) {
4483         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4484         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4485     } else {
4486         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4487     }
4488 }
4489
4490
4491
4492 sub vmconfig_apply_pending {
4493     my ($vmid, $conf, $storecfg, $errors) = @_;
4494
4495     my $add_apply_error = sub {
4496         my ($opt, $msg) = @_;
4497         my $err_msg = "unable to apply pending change $opt : $msg";
4498         $errors->{$opt} = $err_msg;
4499         warn $err_msg;
4500     };
4501
4502     # cold plug
4503
4504     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4505     foreach my $opt (sort keys %$pending_delete_hash) {
4506         my $force = $pending_delete_hash->{$opt}->{force};
4507         eval {
4508             if ($opt =~ m/^unused/) {
4509                 die "internal error";
4510             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4511                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4512             }
4513         };
4514         if (my $err = $@) {
4515             $add_apply_error->($opt, $err);
4516         } else {
4517             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4518             delete $conf->{$opt};
4519         }
4520     }
4521
4522     PVE::QemuConfig->cleanup_pending($conf);
4523
4524     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4525         next if $opt eq 'delete'; # just to be sure
4526         eval {
4527             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4528                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4529             }
4530         };
4531         if (my $err = $@) {
4532             $add_apply_error->($opt, $err);
4533         } else {
4534             $conf->{$opt} = delete $conf->{pending}->{$opt};
4535         }
4536     }
4537
4538     # write all changes at once to avoid unnecessary i/o
4539     PVE::QemuConfig->write_config($vmid, $conf);
4540 }
4541
4542 sub vmconfig_update_net {
4543     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4544
4545     my $newnet = parse_net($value);
4546
4547     if ($conf->{$opt}) {
4548         my $oldnet = parse_net($conf->{$opt});
4549
4550         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4551             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4552             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4553             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4554
4555             # for non online change, we try to hot-unplug
4556             die "skip\n" if !$hotplug;
4557             vm_deviceunplug($vmid, $conf, $opt);
4558         } else {
4559
4560             die "internal error" if $opt !~ m/net(\d+)/;
4561             my $iface = "tap${vmid}i$1";
4562
4563             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4564                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4565                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4566                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4567                 PVE::Network::tap_unplug($iface);
4568
4569                 if ($have_sdn) {
4570                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4571                 } else {
4572                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4573                 }
4574             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4575                 # Rate can be applied on its own but any change above needs to
4576                 # include the rate in tap_plug since OVS resets everything.
4577                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4578             }
4579
4580             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4581                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4582             }
4583
4584             return 1;
4585         }
4586     }
4587
4588     if ($hotplug) {
4589         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4590     } else {
4591         die "skip\n";
4592     }
4593 }
4594
4595 sub vmconfig_update_disk {
4596     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4597
4598     my $drive = parse_drive($opt, $value);
4599
4600     if ($conf->{$opt}) {
4601
4602         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4603
4604             my $media = $drive->{media} || 'disk';
4605             my $oldmedia = $old_drive->{media} || 'disk';
4606             die "unable to change media type\n" if $media ne $oldmedia;
4607
4608             if (!drive_is_cdrom($old_drive)) {
4609
4610                 if ($drive->{file} ne $old_drive->{file}) {
4611
4612                     die "skip\n" if !$hotplug;
4613
4614                     # unplug and register as unused
4615                     vm_deviceunplug($vmid, $conf, $opt);
4616                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4617
4618                 } else {
4619                     # update existing disk
4620
4621                     # skip non hotpluggable value
4622                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4623                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4624                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4625                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4626                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4627                         die "skip\n";
4628                     }
4629
4630                     # apply throttle
4631                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4632                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4633                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4634                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4635                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4636                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4637                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4638                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4639                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4640                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4641                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4642                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4643                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4644                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4645                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4646                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4647                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4648                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4649
4650                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4651                                                    ($drive->{mbps} || 0)*1024*1024,
4652                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4653                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4654                                                    $drive->{iops} || 0,
4655                                                    $drive->{iops_rd} || 0,
4656                                                    $drive->{iops_wr} || 0,
4657                                                    ($drive->{mbps_max} || 0)*1024*1024,
4658                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4659                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4660                                                    $drive->{iops_max} || 0,
4661                                                    $drive->{iops_rd_max} || 0,
4662                                                    $drive->{iops_wr_max} || 0,
4663                                                    $drive->{bps_max_length} || 1,
4664                                                    $drive->{bps_rd_max_length} || 1,
4665                                                    $drive->{bps_wr_max_length} || 1,
4666                                                    $drive->{iops_max_length} || 1,
4667                                                    $drive->{iops_rd_max_length} || 1,
4668                                                    $drive->{iops_wr_max_length} || 1);
4669
4670                     }
4671
4672                     return 1;
4673                 }
4674
4675             } else { # cdrom
4676
4677                 if ($drive->{file} eq 'none') {
4678                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4679                     if (drive_is_cloudinit($old_drive)) {
4680                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4681                     }
4682                 } else {
4683                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4684
4685                     # force eject if locked
4686                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4687
4688                     if ($path) {
4689                         mon_cmd($vmid, "blockdev-change-medium",
4690                             id => "$opt", filename => "$path");
4691                     }
4692                 }
4693
4694                 return 1;
4695             }
4696         }
4697     }
4698
4699     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4700     # hotplug new disks
4701     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4702     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4703 }
4704
4705 # called in locked context by incoming migration
4706 sub vm_migrate_get_nbd_disks {
4707     my ($storecfg, $conf, $replicated_volumes) = @_;
4708
4709     my $local_volumes = {};
4710     PVE::QemuConfig->foreach_volume($conf, sub {
4711         my ($ds, $drive) = @_;
4712
4713         return if drive_is_cdrom($drive);
4714
4715         my $volid = $drive->{file};
4716
4717         return if !$volid;
4718
4719         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4720
4721         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4722         return if $scfg->{shared};
4723
4724         # replicated disks re-use existing state via bitmap
4725         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4726         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4727     });
4728     return $local_volumes;
4729 }
4730
4731 # called in locked context by incoming migration
4732 sub vm_migrate_alloc_nbd_disks {
4733     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4734
4735     my $format = undef;
4736
4737     my $nbd = {};
4738     foreach my $opt (sort keys %$source_volumes) {
4739         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4740
4741         if ($use_existing) {
4742             $nbd->{$opt}->{drivestr} = print_drive($drive);
4743             $nbd->{$opt}->{volid} = $volid;
4744             $nbd->{$opt}->{replicated} = 1;
4745             next;
4746         }
4747
4748         # If a remote storage is specified and the format of the original
4749         # volume is not available there, fall back to the default format.
4750         # Otherwise use the same format as the original.
4751         if (!$storagemap->{identity}) {
4752             $storeid = map_storage($storagemap, $storeid);
4753             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4754             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4755             my $fileFormat = qemu_img_format($scfg, $volname);
4756             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4757         } else {
4758             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4759             $format = qemu_img_format($scfg, $volname);
4760         }
4761
4762         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4763         my $newdrive = $drive;
4764         $newdrive->{format} = $format;
4765         $newdrive->{file} = $newvolid;
4766         my $drivestr = print_drive($newdrive);
4767         $nbd->{$opt}->{drivestr} = $drivestr;
4768         $nbd->{$opt}->{volid} = $newvolid;
4769     }
4770
4771     return $nbd;
4772 }
4773
4774 # see vm_start_nolock for parameters, additionally:
4775 # migrate_opts:
4776 #   storagemap = parsed storage map for allocating NBD disks
4777 sub vm_start {
4778     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4779
4780     return PVE::QemuConfig->lock_config($vmid, sub {
4781         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4782
4783         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4784
4785         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4786
4787         PVE::QemuConfig->check_lock($conf)
4788             if !($params->{skiplock} || $has_suspended_lock);
4789
4790         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4791
4792         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4793
4794         if (my $storagemap = $migrate_opts->{storagemap}) {
4795             my $replicated = $migrate_opts->{replicated_volumes};
4796             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4797             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4798
4799             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4800                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4801             }
4802         }
4803
4804         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4805     });
4806 }
4807
4808
4809 # params:
4810 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4811 #   skiplock => 0/1, skip checking for config lock
4812 #   forcemachine => to force Qemu machine (rollback/migration)
4813 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4814 #   timeout => in seconds
4815 #   paused => start VM in paused state (backup)
4816 #   resume => resume from hibernation
4817 # migrate_opts:
4818 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4819 #   migratedfrom => source node
4820 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4821 #   network => CIDR of migration network
4822 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4823 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4824 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4825 sub vm_start_nolock {
4826     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4827
4828     my $statefile = $params->{statefile};
4829     my $resume = $params->{resume};
4830
4831     my $migratedfrom = $migrate_opts->{migratedfrom};
4832     my $migration_type = $migrate_opts->{type};
4833
4834     my $res = {};
4835
4836     # clean up leftover reboot request files
4837     eval { clear_reboot_request($vmid); };
4838     warn $@ if $@;
4839
4840     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4841         vmconfig_apply_pending($vmid, $conf, $storecfg);
4842         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4843     }
4844
4845     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4846
4847     my $defaults = load_defaults();
4848
4849     # set environment variable useful inside network script
4850     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4851
4852     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4853
4854     my $forcemachine = $params->{forcemachine};
4855     my $forcecpu = $params->{forcecpu};
4856     if ($resume) {
4857         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4858         $forcemachine = $conf->{runningmachine};
4859         $forcecpu = $conf->{runningcpu};
4860         print "Resuming suspended VM\n";
4861     }
4862
4863     my ($cmd, $vollist, $spice_port) =
4864         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4865
4866     my $migration_ip;
4867     my $get_migration_ip = sub {
4868         my ($nodename) = @_;
4869
4870         return $migration_ip if defined($migration_ip);
4871
4872         my $cidr = $migrate_opts->{network};
4873
4874         if (!defined($cidr)) {
4875             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4876             $cidr = $dc_conf->{migration}->{network};
4877         }
4878
4879         if (defined($cidr)) {
4880             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4881
4882             die "could not get IP: no address configured on local " .
4883                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4884
4885             die "could not get IP: multiple addresses configured on local " .
4886                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4887
4888             $migration_ip = @$ips[0];
4889         }
4890
4891         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4892             if !defined($migration_ip);
4893
4894         return $migration_ip;
4895     };
4896
4897     my $migrate_uri;
4898     if ($statefile) {
4899         if ($statefile eq 'tcp') {
4900             my $localip = "localhost";
4901             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4902             my $nodename = nodename();
4903
4904             if (!defined($migration_type)) {
4905                 if (defined($datacenterconf->{migration}->{type})) {
4906                     $migration_type = $datacenterconf->{migration}->{type};
4907                 } else {
4908                     $migration_type = 'secure';
4909                 }
4910             }
4911
4912             if ($migration_type eq 'insecure') {
4913                 $localip = $get_migration_ip->($nodename);
4914                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4915             }
4916
4917             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4918             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4919             $migrate_uri = "tcp:${localip}:${migrate_port}";
4920             push @$cmd, '-incoming', $migrate_uri;
4921             push @$cmd, '-S';
4922
4923         } elsif ($statefile eq 'unix') {
4924             # should be default for secure migrations as a ssh TCP forward
4925             # tunnel is not deterministic reliable ready and fails regurarly
4926             # to set up in time, so use UNIX socket forwards
4927             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4928             unlink $socket_addr;
4929
4930             $migrate_uri = "unix:$socket_addr";
4931
4932             push @$cmd, '-incoming', $migrate_uri;
4933             push @$cmd, '-S';
4934
4935         } elsif (-e $statefile) {
4936             push @$cmd, '-loadstate', $statefile;
4937         } else {
4938             my $statepath = PVE::Storage::path($storecfg, $statefile);
4939             push @$vollist, $statefile;
4940             push @$cmd, '-loadstate', $statepath;
4941         }
4942     } elsif ($params->{paused}) {
4943         push @$cmd, '-S';
4944     }
4945
4946     # host pci devices
4947     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
4948       my $d = parse_hostpci($conf->{"hostpci$i"});
4949       next if !$d;
4950       my $pcidevices = $d->{pciid};
4951       foreach my $pcidevice (@$pcidevices) {
4952             my $pciid = $pcidevice->{id};
4953
4954             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4955             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4956             die "no pci device info for device '$pciid'\n" if !$info;
4957
4958             if ($d->{mdev}) {
4959                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4960                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4961             } else {
4962                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4963                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4964                 die "can't reset pci device '$pciid'\n"
4965                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4966             }
4967       }
4968     }
4969
4970     PVE::Storage::activate_volumes($storecfg, $vollist);
4971
4972     eval {
4973         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4974             outfunc => sub {}, errfunc => sub {});
4975     };
4976     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4977     # timeout should be more than enough here...
4978     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4979
4980     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4981                                               : $defaults->{cpuunits};
4982
4983     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4984     my %run_params = (
4985         timeout => $statefile ? undef : $start_timeout,
4986         umask => 0077,
4987         noerr => 1,
4988     );
4989
4990     # when migrating, prefix QEMU output so other side can pick up any
4991     # errors that might occur and show the user
4992     if ($migratedfrom) {
4993         $run_params{quiet} = 1;
4994         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4995     }
4996
4997     my %properties = (
4998         Slice => 'qemu.slice',
4999         KillMode => 'none',
5000         CPUShares => $cpuunits
5001     );
5002
5003     if (my $cpulimit = $conf->{cpulimit}) {
5004         $properties{CPUQuota} = int($cpulimit * 100);
5005     }
5006     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5007
5008     my $run_qemu = sub {
5009         PVE::Tools::run_fork sub {
5010             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5011
5012             my $exitcode = run_command($cmd, %run_params);
5013             die "QEMU exited with code $exitcode\n" if $exitcode;
5014         };
5015     };
5016
5017     if ($conf->{hugepages}) {
5018
5019         my $code = sub {
5020             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5021             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5022
5023             PVE::QemuServer::Memory::hugepages_mount();
5024             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5025
5026             eval { $run_qemu->() };
5027             if (my $err = $@) {
5028                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5029                 die $err;
5030             }
5031
5032             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5033         };
5034         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5035
5036     } else {
5037         eval { $run_qemu->() };
5038     }
5039
5040     if (my $err = $@) {
5041         # deactivate volumes if start fails
5042         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5043         die "start failed: $err";
5044     }
5045
5046     print "migration listens on $migrate_uri\n" if $migrate_uri;
5047     $res->{migrate_uri} = $migrate_uri;
5048
5049     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5050         eval { mon_cmd($vmid, "cont"); };
5051         warn $@ if $@;
5052     }
5053
5054     #start nbd server for storage migration
5055     if (my $nbd = $migrate_opts->{nbd}) {
5056         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5057
5058         my $migrate_storage_uri;
5059         # nbd_protocol_version > 0 for unix socket support
5060         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5061             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5062             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5063             $migrate_storage_uri = "nbd:unix:$socket_path";
5064         } else {
5065             my $nodename = nodename();
5066             my $localip = $get_migration_ip->($nodename);
5067             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5068             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5069
5070             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5071             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5072             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5073         }
5074
5075         $res->{migrate_storage_uri} = $migrate_storage_uri;
5076
5077         foreach my $opt (sort keys %$nbd) {
5078             my $drivestr = $nbd->{$opt}->{drivestr};
5079             my $volid = $nbd->{$opt}->{volid};
5080             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5081             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5082             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5083             print "re-using replicated volume: $opt - $volid\n"
5084                 if $nbd->{$opt}->{replicated};
5085
5086             $res->{drives}->{$opt} = $nbd->{$opt};
5087             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5088         }
5089     }
5090
5091     if ($migratedfrom) {
5092         eval {
5093             set_migration_caps($vmid);
5094         };
5095         warn $@ if $@;
5096
5097         if ($spice_port) {
5098             print "spice listens on port $spice_port\n";
5099             $res->{spice_port} = $spice_port;
5100             if ($migrate_opts->{spice_ticket}) {
5101                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5102                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5103             }
5104         }
5105
5106     } else {
5107         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5108             if !$statefile && $conf->{balloon};
5109
5110         foreach my $opt (keys %$conf) {
5111             next if $opt !~  m/^net\d+$/;
5112             my $nicconf = parse_net($conf->{$opt});
5113             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5114         }
5115     }
5116
5117     mon_cmd($vmid, 'qom-set',
5118                 path => "machine/peripheral/balloon0",
5119                 property => "guest-stats-polling-interval",
5120                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5121
5122     if ($resume) {
5123         print "Resumed VM, removing state\n";
5124         if (my $vmstate = $conf->{vmstate}) {
5125             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5126             PVE::Storage::vdisk_free($storecfg, $vmstate);
5127         }
5128         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5129         PVE::QemuConfig->write_config($vmid, $conf);
5130     }
5131
5132     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5133
5134     return $res;
5135 }
5136
5137 sub vm_commandline {
5138     my ($storecfg, $vmid, $snapname) = @_;
5139
5140     my $conf = PVE::QemuConfig->load_config($vmid);
5141     my $forcemachine;
5142     my $forcecpu;
5143
5144     if ($snapname) {
5145         my $snapshot = $conf->{snapshots}->{$snapname};
5146         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5147
5148         # check for machine or CPU overrides in snapshot
5149         $forcemachine = $snapshot->{runningmachine};
5150         $forcecpu = $snapshot->{runningcpu};
5151
5152         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5153
5154         $conf = $snapshot;
5155     }
5156
5157     my $defaults = load_defaults();
5158
5159     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5160         $forcemachine, $forcecpu);
5161
5162     return PVE::Tools::cmd2string($cmd);
5163 }
5164
5165 sub vm_reset {
5166     my ($vmid, $skiplock) = @_;
5167
5168     PVE::QemuConfig->lock_config($vmid, sub {
5169
5170         my $conf = PVE::QemuConfig->load_config($vmid);
5171
5172         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5173
5174         mon_cmd($vmid, "system_reset");
5175     });
5176 }
5177
5178 sub get_vm_volumes {
5179     my ($conf) = @_;
5180
5181     my $vollist = [];
5182     foreach_volid($conf, sub {
5183         my ($volid, $attr) = @_;
5184
5185         return if $volid =~ m|^/|;
5186
5187         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5188         return if !$sid;
5189
5190         push @$vollist, $volid;
5191     });
5192
5193     return $vollist;
5194 }
5195
5196 sub vm_stop_cleanup {
5197     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5198
5199     eval {
5200
5201         if (!$keepActive) {
5202             my $vollist = get_vm_volumes($conf);
5203             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5204         }
5205
5206         foreach my $ext (qw(mon qmp pid vnc qga)) {
5207             unlink "/var/run/qemu-server/${vmid}.$ext";
5208         }
5209
5210         if ($conf->{ivshmem}) {
5211             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5212             # just delete it for now, VMs which have this already open do not
5213             # are affected, but new VMs will get a separated one. If this
5214             # becomes an issue we either add some sort of ref-counting or just
5215             # add a "don't delete on stop" flag to the ivshmem format.
5216             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5217         }
5218
5219         foreach my $key (keys %$conf) {
5220             next if $key !~ m/^hostpci(\d+)$/;
5221             my $hostpciindex = $1;
5222             my $d = parse_hostpci($conf->{$key});
5223             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5224
5225             foreach my $pci (@{$d->{pciid}}) {
5226                 my $pciid = $pci->{id};
5227                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5228             }
5229         }
5230
5231         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5232     };
5233     warn $@ if $@; # avoid errors - just warn
5234 }
5235
5236 # call only in locked context
5237 sub _do_vm_stop {
5238     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5239
5240     my $pid = check_running($vmid, $nocheck);
5241     return if !$pid;
5242
5243     my $conf;
5244     if (!$nocheck) {
5245         $conf = PVE::QemuConfig->load_config($vmid);
5246         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5247         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5248             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5249             $timeout = $opts->{down} if $opts->{down};
5250         }
5251         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5252     }
5253
5254     eval {
5255         if ($shutdown) {
5256             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5257                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5258             } else {
5259                 mon_cmd($vmid, "system_powerdown");
5260             }
5261         } else {
5262             mon_cmd($vmid, "quit");
5263         }
5264     };
5265     my $err = $@;
5266
5267     if (!$err) {
5268         $timeout = 60 if !defined($timeout);
5269
5270         my $count = 0;
5271         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5272             $count++;
5273             sleep 1;
5274         }
5275
5276         if ($count >= $timeout) {
5277             if ($force) {
5278                 warn "VM still running - terminating now with SIGTERM\n";
5279                 kill 15, $pid;
5280             } else {
5281                 die "VM quit/powerdown failed - got timeout\n";
5282             }
5283         } else {
5284             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5285             return;
5286         }
5287     } else {
5288        if (!check_running($vmid, $nocheck)) {
5289             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5290             return;
5291        }
5292        if ($force) {
5293             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5294             kill 15, $pid;
5295         } else {
5296             die "VM quit/powerdown failed\n";
5297         }
5298     }
5299
5300     # wait again
5301     $timeout = 10;
5302
5303     my $count = 0;
5304     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5305         $count++;
5306         sleep 1;
5307     }
5308
5309     if ($count >= $timeout) {
5310         warn "VM still running - terminating now with SIGKILL\n";
5311         kill 9, $pid;
5312         sleep 1;
5313     }
5314
5315     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5316 }
5317
5318 # Note: use $nocheck to skip tests if VM configuration file exists.
5319 # We need that when migration VMs to other nodes (files already moved)
5320 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5321 sub vm_stop {
5322     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5323
5324     $force = 1 if !defined($force) && !$shutdown;
5325
5326     if ($migratedfrom){
5327         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5328         kill 15, $pid if $pid;
5329         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5330         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5331         return;
5332     }
5333
5334     PVE::QemuConfig->lock_config($vmid, sub {
5335         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5336    });
5337 }
5338
5339 sub vm_reboot {
5340     my ($vmid, $timeout) = @_;
5341
5342     PVE::QemuConfig->lock_config($vmid, sub {
5343         eval {
5344
5345             # only reboot if running, as qmeventd starts it again on a stop event
5346             return if !check_running($vmid);
5347
5348             create_reboot_request($vmid);
5349
5350             my $storecfg = PVE::Storage::config();
5351             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5352
5353         };
5354         if (my $err = $@) {
5355             # avoid that the next normal shutdown will be confused for a reboot
5356             clear_reboot_request($vmid);
5357             die $err;
5358         }
5359    });
5360 }
5361
5362 # note: if using the statestorage parameter, the caller has to check privileges
5363 sub vm_suspend {
5364     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5365
5366     my $conf;
5367     my $path;
5368     my $storecfg;
5369     my $vmstate;
5370
5371     PVE::QemuConfig->lock_config($vmid, sub {
5372
5373         $conf = PVE::QemuConfig->load_config($vmid);
5374
5375         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5376         PVE::QemuConfig->check_lock($conf)
5377             if !($skiplock || $is_backing_up);
5378
5379         die "cannot suspend to disk during backup\n"
5380             if $is_backing_up && $includestate;
5381
5382         if ($includestate) {
5383             $conf->{lock} = 'suspending';
5384             my $date = strftime("%Y-%m-%d", localtime(time()));
5385             $storecfg = PVE::Storage::config();
5386             if (!$statestorage) {
5387                 $statestorage = find_vmstate_storage($conf, $storecfg);
5388                 # check permissions for the storage
5389                 my $rpcenv = PVE::RPCEnvironment::get();
5390                 if ($rpcenv->{type} ne 'cli') {
5391                     my $authuser = $rpcenv->get_user();
5392                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5393                 }
5394             }
5395
5396
5397             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5398             $path = PVE::Storage::path($storecfg, $vmstate);
5399             PVE::QemuConfig->write_config($vmid, $conf);
5400         } else {
5401             mon_cmd($vmid, "stop");
5402         }
5403     });
5404
5405     if ($includestate) {
5406         # save vm state
5407         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5408
5409         eval {
5410             mon_cmd($vmid, "savevm-start", statefile => $path);
5411             for(;;) {
5412                 my $state = mon_cmd($vmid, "query-savevm");
5413                 if (!$state->{status}) {
5414                     die "savevm not active\n";
5415                 } elsif ($state->{status} eq 'active') {
5416                     sleep(1);
5417                     next;
5418                 } elsif ($state->{status} eq 'completed') {
5419                     print "State saved, quitting\n";
5420                     last;
5421                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5422                     die "query-savevm failed with error '$state->{error}'\n"
5423                 } else {
5424                     die "query-savevm returned status '$state->{status}'\n";
5425                 }
5426             }
5427         };
5428         my $err = $@;
5429
5430         PVE::QemuConfig->lock_config($vmid, sub {
5431             $conf = PVE::QemuConfig->load_config($vmid);
5432             if ($err) {
5433                 # cleanup, but leave suspending lock, to indicate something went wrong
5434                 eval {
5435                     mon_cmd($vmid, "savevm-end");
5436                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5437                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5438                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5439                     PVE::QemuConfig->write_config($vmid, $conf);
5440                 };
5441                 warn $@ if $@;
5442                 die $err;
5443             }
5444
5445             die "lock changed unexpectedly\n"
5446                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5447
5448             mon_cmd($vmid, "quit");
5449             $conf->{lock} = 'suspended';
5450             PVE::QemuConfig->write_config($vmid, $conf);
5451         });
5452     }
5453 }
5454
5455 sub vm_resume {
5456     my ($vmid, $skiplock, $nocheck) = @_;
5457
5458     PVE::QemuConfig->lock_config($vmid, sub {
5459         my $res = mon_cmd($vmid, 'query-status');
5460         my $resume_cmd = 'cont';
5461
5462         if ($res->{status} && $res->{status} eq 'suspended') {
5463             $resume_cmd = 'system_wakeup';
5464         }
5465
5466         if (!$nocheck) {
5467
5468             my $conf = PVE::QemuConfig->load_config($vmid);
5469
5470             PVE::QemuConfig->check_lock($conf)
5471                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5472         }
5473
5474         mon_cmd($vmid, $resume_cmd);
5475     });
5476 }
5477
5478 sub vm_sendkey {
5479     my ($vmid, $skiplock, $key) = @_;
5480
5481     PVE::QemuConfig->lock_config($vmid, sub {
5482
5483         my $conf = PVE::QemuConfig->load_config($vmid);
5484
5485         # there is no qmp command, so we use the human monitor command
5486         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5487         die $res if $res ne '';
5488     });
5489 }
5490
5491 # vzdump restore implementaion
5492
5493 sub tar_archive_read_firstfile {
5494     my $archive = shift;
5495
5496     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5497
5498     # try to detect archive type first
5499     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5500         die "unable to open file '$archive'\n";
5501     my $firstfile = <$fh>;
5502     kill 15, $pid;
5503     close $fh;
5504
5505     die "ERROR: archive contaions no data\n" if !$firstfile;
5506     chomp $firstfile;
5507
5508     return $firstfile;
5509 }
5510
5511 sub tar_restore_cleanup {
5512     my ($storecfg, $statfile) = @_;
5513
5514     print STDERR "starting cleanup\n";
5515
5516     if (my $fd = IO::File->new($statfile, "r")) {
5517         while (defined(my $line = <$fd>)) {
5518             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5519                 my $volid = $2;
5520                 eval {
5521                     if ($volid =~ m|^/|) {
5522                         unlink $volid || die 'unlink failed\n';
5523                     } else {
5524                         PVE::Storage::vdisk_free($storecfg, $volid);
5525                     }
5526                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5527                 };
5528                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5529             } else {
5530                 print STDERR "unable to parse line in statfile - $line";
5531             }
5532         }
5533         $fd->close();
5534     }
5535 }
5536
5537 sub restore_file_archive {
5538     my ($archive, $vmid, $user, $opts) = @_;
5539
5540     return restore_vma_archive($archive, $vmid, $user, $opts)
5541         if $archive eq '-';
5542
5543     my $info = PVE::Storage::archive_info($archive);
5544     my $format = $opts->{format} // $info->{format};
5545     my $comp = $info->{compression};
5546
5547     # try to detect archive format
5548     if ($format eq 'tar') {
5549         return restore_tar_archive($archive, $vmid, $user, $opts);
5550     } else {
5551         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5552     }
5553 }
5554
5555 # hepler to remove disks that will not be used after restore
5556 my $restore_cleanup_oldconf = sub {
5557     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5558
5559     PVE::QemuConfig->foreach_volume($oldconf, sub {
5560         my ($ds, $drive) = @_;
5561
5562         return if drive_is_cdrom($drive, 1);
5563
5564         my $volid = $drive->{file};
5565         return if !$volid || $volid =~ m|^/|;
5566
5567         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5568         return if !$path || !$owner || ($owner != $vmid);
5569
5570         # Note: only delete disk we want to restore
5571         # other volumes will become unused
5572         if ($virtdev_hash->{$ds}) {
5573             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5574             if (my $err = $@) {
5575                 warn $err;
5576             }
5577         }
5578     });
5579
5580     # delete vmstate files, after the restore we have no snapshots anymore
5581     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5582         my $snap = $oldconf->{snapshots}->{$snapname};
5583         if ($snap->{vmstate}) {
5584             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5585             if (my $err = $@) {
5586                 warn $err;
5587             }
5588         }
5589     }
5590 };
5591
5592 # Helper to parse vzdump backup device hints
5593 #
5594 # $rpcenv: Environment, used to ckeck storage permissions
5595 # $user: User ID, to check storage permissions
5596 # $storecfg: Storage configuration
5597 # $fh: the file handle for reading the configuration
5598 # $devinfo: should contain device sizes for all backu-up'ed devices
5599 # $options: backup options (pool, default storage)
5600 #
5601 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5602 my $parse_backup_hints = sub {
5603     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5604
5605     my $virtdev_hash = {};
5606
5607     while (defined(my $line = <$fh>)) {
5608         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5609             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5610             die "archive does not contain data for drive '$virtdev'\n"
5611                 if !$devinfo->{$devname};
5612
5613             if (defined($options->{storage})) {
5614                 $storeid = $options->{storage} || 'local';
5615             } elsif (!$storeid) {
5616                 $storeid = 'local';
5617             }
5618             $format = 'raw' if !$format;
5619             $devinfo->{$devname}->{devname} = $devname;
5620             $devinfo->{$devname}->{virtdev} = $virtdev;
5621             $devinfo->{$devname}->{format} = $format;
5622             $devinfo->{$devname}->{storeid} = $storeid;
5623
5624             # check permission on storage
5625             my $pool = $options->{pool}; # todo: do we need that?
5626             if ($user ne 'root@pam') {
5627                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5628             }
5629
5630             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5631         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5632             my $virtdev = $1;
5633             my $drive = parse_drive($virtdev, $2);
5634             if (drive_is_cloudinit($drive)) {
5635                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5636                 $storeid = $options->{storage} if defined ($options->{storage});
5637                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5638                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5639
5640                 $virtdev_hash->{$virtdev} = {
5641                     format => $format,
5642                     storeid => $storeid,
5643                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5644                     is_cloudinit => 1,
5645                 };
5646             }
5647         }
5648     }
5649
5650     return $virtdev_hash;
5651 };
5652
5653 # Helper to allocate and activate all volumes required for a restore
5654 #
5655 # $storecfg: Storage configuration
5656 # $virtdev_hash: as returned by parse_backup_hints()
5657 #
5658 # Returns: { $virtdev => $volid }
5659 my $restore_allocate_devices = sub {
5660     my ($storecfg, $virtdev_hash, $vmid) = @_;
5661
5662     my $map = {};
5663     foreach my $virtdev (sort keys %$virtdev_hash) {
5664         my $d = $virtdev_hash->{$virtdev};
5665         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5666         my $storeid = $d->{storeid};
5667         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5668
5669         # test if requested format is supported
5670         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5671         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5672         $d->{format} = $defFormat if !$supported;
5673
5674         my $name;
5675         if ($d->{is_cloudinit}) {
5676             $name = "vm-$vmid-cloudinit";
5677             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5678         }
5679
5680         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5681
5682         print STDERR "new volume ID is '$volid'\n";
5683         $d->{volid} = $volid;
5684
5685         PVE::Storage::activate_volumes($storecfg, [$volid]);
5686
5687         $map->{$virtdev} = $volid;
5688     }
5689
5690     return $map;
5691 };
5692
5693 my $restore_update_config_line = sub {
5694     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5695
5696     return if $line =~ m/^\#qmdump\#/;
5697     return if $line =~ m/^\#vzdump\#/;
5698     return if $line =~ m/^lock:/;
5699     return if $line =~ m/^unused\d+:/;
5700     return if $line =~ m/^parent:/;
5701
5702     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5703     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5704         # try to convert old 1.X settings
5705         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5706         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5707             my ($model, $macaddr) = split(/\=/, $devconfig);
5708             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5709             my $net = {
5710                 model => $model,
5711                 bridge => "vmbr$ind",
5712                 macaddr => $macaddr,
5713             };
5714             my $netstr = print_net($net);
5715
5716             print $outfd "net$cookie->{netcount}: $netstr\n";
5717             $cookie->{netcount}++;
5718         }
5719     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5720         my ($id, $netstr) = ($1, $2);
5721         my $net = parse_net($netstr);
5722         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5723         $netstr = print_net($net);
5724         print $outfd "$id: $netstr\n";
5725     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5726         my $virtdev = $1;
5727         my $value = $3;
5728         my $di = parse_drive($virtdev, $value);
5729         if (defined($di->{backup}) && !$di->{backup}) {
5730             print $outfd "#$line";
5731         } elsif ($map->{$virtdev}) {
5732             delete $di->{format}; # format can change on restore
5733             $di->{file} = $map->{$virtdev};
5734             $value = print_drive($di);
5735             print $outfd "$virtdev: $value\n";
5736         } else {
5737             print $outfd $line;
5738         }
5739     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5740         my $vmgenid = $1;
5741         if ($vmgenid ne '0') {
5742             # always generate a new vmgenid if there was a valid one setup
5743             $vmgenid = generate_uuid();
5744         }
5745         print $outfd "vmgenid: $vmgenid\n";
5746     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5747         my ($uuid, $uuid_str);
5748         UUID::generate($uuid);
5749         UUID::unparse($uuid, $uuid_str);
5750         my $smbios1 = parse_smbios1($2);
5751         $smbios1->{uuid} = $uuid_str;
5752         print $outfd $1.print_smbios1($smbios1)."\n";
5753     } else {
5754         print $outfd $line;
5755     }
5756 };
5757
5758 my $restore_deactivate_volumes = sub {
5759     my ($storecfg, $devinfo) = @_;
5760
5761     my $vollist = [];
5762     foreach my $devname (keys %$devinfo) {
5763         my $volid = $devinfo->{$devname}->{volid};
5764         push @$vollist, $volid if $volid;
5765     }
5766
5767     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5768 };
5769
5770 my $restore_destroy_volumes = sub {
5771     my ($storecfg, $devinfo) = @_;
5772
5773     foreach my $devname (keys %$devinfo) {
5774         my $volid = $devinfo->{$devname}->{volid};
5775         next if !$volid;
5776         eval {
5777             if ($volid =~ m|^/|) {
5778                 unlink $volid || die 'unlink failed\n';
5779             } else {
5780                 PVE::Storage::vdisk_free($storecfg, $volid);
5781             }
5782             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5783         };
5784         print STDERR "unable to cleanup '$volid' - $@" if $@;
5785     }
5786 };
5787
5788 sub scan_volids {
5789     my ($cfg, $vmid) = @_;
5790
5791     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5792
5793     my $volid_hash = {};
5794     foreach my $storeid (keys %$info) {
5795         foreach my $item (@{$info->{$storeid}}) {
5796             next if !($item->{volid} && $item->{size});
5797             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5798             $volid_hash->{$item->{volid}} = $item;
5799         }
5800     }
5801
5802     return $volid_hash;
5803 }
5804
5805 sub update_disk_config {
5806     my ($vmid, $conf, $volid_hash) = @_;
5807
5808     my $changes;
5809     my $prefix = "VM $vmid";
5810
5811     # used and unused disks
5812     my $referenced = {};
5813
5814     # Note: it is allowed to define multiple storages with same path (alias), so
5815     # we need to check both 'volid' and real 'path' (two different volid can point
5816     # to the same path).
5817
5818     my $referencedpath = {};
5819
5820     # update size info
5821     PVE::QemuConfig->foreach_volume($conf, sub {
5822         my ($opt, $drive) = @_;
5823
5824         my $volid = $drive->{file};
5825         return if !$volid;
5826
5827         # mark volid as "in-use" for next step
5828         $referenced->{$volid} = 1;
5829         if ($volid_hash->{$volid} &&
5830             (my $path = $volid_hash->{$volid}->{path})) {
5831             $referencedpath->{$path} = 1;
5832         }
5833
5834         return if drive_is_cdrom($drive);
5835         return if !$volid_hash->{$volid};
5836
5837         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size});
5838         if (defined($updated)) {
5839             $changes = 1;
5840             $conf->{$opt} = print_drive($updated);
5841             print "$prefix ($opt): $msg\n";
5842         }
5843     });
5844
5845     # remove 'unusedX' entry if volume is used
5846     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5847         my ($opt, $drive) = @_;
5848
5849         my $volid = $drive->{file};
5850         return if !$volid;
5851
5852         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5853         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5854             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5855             $changes = 1;
5856             delete $conf->{$opt};
5857         }
5858
5859         $referenced->{$volid} = 1;
5860         $referencedpath->{$path} = 1 if $path;
5861     });
5862
5863     foreach my $volid (sort keys %$volid_hash) {
5864         next if $volid =~ m/vm-$vmid-state-/;
5865         next if $referenced->{$volid};
5866         my $path = $volid_hash->{$volid}->{path};
5867         next if !$path; # just to be sure
5868         next if $referencedpath->{$path};
5869         $changes = 1;
5870         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5871         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5872         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5873     }
5874
5875     return $changes;
5876 }
5877
5878 sub rescan {
5879     my ($vmid, $nolock, $dryrun) = @_;
5880
5881     my $cfg = PVE::Storage::config();
5882
5883     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5884     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5885     foreach my $stor (keys %{$cfg->{ids}}) {
5886         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5887     }
5888
5889     print "rescan volumes...\n";
5890     my $volid_hash = scan_volids($cfg, $vmid);
5891
5892     my $updatefn =  sub {
5893         my ($vmid) = @_;
5894
5895         my $conf = PVE::QemuConfig->load_config($vmid);
5896
5897         PVE::QemuConfig->check_lock($conf);
5898
5899         my $vm_volids = {};
5900         foreach my $volid (keys %$volid_hash) {
5901             my $info = $volid_hash->{$volid};
5902             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5903         }
5904
5905         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5906
5907         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5908     };
5909
5910     if (defined($vmid)) {
5911         if ($nolock) {
5912             &$updatefn($vmid);
5913         } else {
5914             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5915         }
5916     } else {
5917         my $vmlist = config_list();
5918         foreach my $vmid (keys %$vmlist) {
5919             if ($nolock) {
5920                 &$updatefn($vmid);
5921             } else {
5922                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5923             }
5924         }
5925     }
5926 }
5927
5928 sub restore_proxmox_backup_archive {
5929     my ($archive, $vmid, $user, $options) = @_;
5930
5931     my $storecfg = PVE::Storage::config();
5932
5933     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5934     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5935
5936     my $server = $scfg->{server};
5937     my $datastore = $scfg->{datastore};
5938     my $username = $scfg->{username} // 'root@pam';
5939     my $fingerprint = $scfg->{fingerprint};
5940
5941     my $repo = "$username\@$server:$datastore";
5942
5943     # This is only used for `pbs-restore`!
5944     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5945     local $ENV{PBS_PASSWORD} = $password;
5946     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5947
5948     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5949         PVE::Storage::parse_volname($storecfg, $archive);
5950
5951     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5952
5953     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5954
5955     my $tmpdir = "/var/tmp/vzdumptmp$$";
5956     rmtree $tmpdir;
5957     mkpath $tmpdir;
5958
5959     my $conffile = PVE::QemuConfig->config_file($vmid);
5960     my $tmpfn = "$conffile.$$.tmp";
5961      # disable interrupts (always do cleanups)
5962     local $SIG{INT} =
5963         local $SIG{TERM} =
5964         local $SIG{QUIT} =
5965         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5966
5967     # Note: $oldconf is undef if VM does not exists
5968     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5969     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5970
5971     my $rpcenv = PVE::RPCEnvironment::get();
5972     my $devinfo = {};
5973
5974     eval {
5975         # enable interrupts
5976         local $SIG{INT} =
5977             local $SIG{TERM} =
5978             local $SIG{QUIT} =
5979             local $SIG{HUP} =
5980             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5981
5982         my $cfgfn = "$tmpdir/qemu-server.conf";
5983         my $firewall_config_fn = "$tmpdir/fw.conf";
5984         my $index_fn = "$tmpdir/index.json";
5985
5986         my $cmd = "restore";
5987
5988         my $param = [$pbs_backup_name, "index.json", $index_fn];
5989         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5990         my $index = PVE::Tools::file_get_contents($index_fn);
5991         $index = decode_json($index);
5992
5993         # print Dumper($index);
5994         foreach my $info (@{$index->{files}}) {
5995             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
5996                 my $devname = $1;
5997                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
5998                     $devinfo->{$devname}->{size} = $1;
5999                 } else {
6000                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6001                 }
6002             }
6003         }
6004
6005         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6006         if (!$is_qemu_server_backup) {
6007             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6008         }
6009         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6010
6011         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6012         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6013
6014         if ($has_firewall_config) {
6015             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6016             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6017
6018             my $pve_firewall_dir = '/etc/pve/firewall';
6019             mkdir $pve_firewall_dir; # make sure the dir exists
6020             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6021         }
6022
6023         my $fh = IO::File->new($cfgfn, "r") ||
6024             "unable to read qemu-server.conf - $!\n";
6025
6026         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6027
6028         # fixme: rate limit?
6029
6030         # create empty/temp config
6031         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6032
6033         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6034
6035         # allocate volumes
6036         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6037
6038         foreach my $virtdev (sort keys %$virtdev_hash) {
6039             my $d = $virtdev_hash->{$virtdev};
6040             next if $d->{is_cloudinit}; # no need to restore cloudinit
6041
6042             my $volid = $d->{volid};
6043
6044             my $path = PVE::Storage::path($storecfg, $volid);
6045
6046             # This is the ONLY user of the PBS_ env vars set on top of this function!
6047             my $pbs_restore_cmd = [
6048                 '/usr/bin/pbs-restore',
6049                 '--repository', $repo,
6050                 $pbs_backup_name,
6051                 "$d->{devname}.img.fidx",
6052                 $path,
6053                 '--verbose',
6054                 ];
6055
6056             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6057
6058             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6059                 push @$pbs_restore_cmd, '--skip-zero';
6060             }
6061
6062             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6063             print "restore proxmox backup image: $dbg_cmdstring\n";
6064             run_command($pbs_restore_cmd);
6065         }
6066
6067         $fh->seek(0, 0) || die "seek failed - $!\n";
6068
6069         my $outfd = new IO::File ($tmpfn, "w") ||
6070             die "unable to write config for VM $vmid\n";
6071
6072         my $cookie = { netcount => 0 };
6073         while (defined(my $line = <$fh>)) {
6074             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6075         }
6076
6077         $fh->close();
6078         $outfd->close();
6079     };
6080     my $err = $@;
6081
6082     $restore_deactivate_volumes->($storecfg, $devinfo);
6083
6084     rmtree $tmpdir;
6085
6086     if ($err) {
6087         unlink $tmpfn;
6088         $restore_destroy_volumes->($storecfg, $devinfo);
6089         die $err;
6090     }
6091
6092     rename($tmpfn, $conffile) ||
6093         die "unable to commit configuration file '$conffile'\n";
6094
6095     PVE::Cluster::cfs_update(); # make sure we read new file
6096
6097     eval { rescan($vmid, 1); };
6098     warn $@ if $@;
6099 }
6100
6101 sub restore_vma_archive {
6102     my ($archive, $vmid, $user, $opts, $comp) = @_;
6103
6104     my $readfrom = $archive;
6105
6106     my $cfg = PVE::Storage::config();
6107     my $commands = [];
6108     my $bwlimit = $opts->{bwlimit};
6109
6110     my $dbg_cmdstring = '';
6111     my $add_pipe = sub {
6112         my ($cmd) = @_;
6113         push @$commands, $cmd;
6114         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6115         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6116         $readfrom = '-';
6117     };
6118
6119     my $input = undef;
6120     if ($archive eq '-') {
6121         $input = '<&STDIN';
6122     } else {
6123         # If we use a backup from a PVE defined storage we also consider that
6124         # storage's rate limit:
6125         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6126         if (defined($volid)) {
6127             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6128             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6129             if ($readlimit) {
6130                 print STDERR "applying read rate limit: $readlimit\n";
6131                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6132                 $add_pipe->($cstream);
6133             }
6134         }
6135     }
6136
6137     if ($comp) {
6138         my $info = PVE::Storage::decompressor_info('vma', $comp);
6139         my $cmd = $info->{decompressor};
6140         push @$cmd, $readfrom;
6141         $add_pipe->($cmd);
6142     }
6143
6144     my $tmpdir = "/var/tmp/vzdumptmp$$";
6145     rmtree $tmpdir;
6146
6147     # disable interrupts (always do cleanups)
6148     local $SIG{INT} =
6149         local $SIG{TERM} =
6150         local $SIG{QUIT} =
6151         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6152
6153     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6154     POSIX::mkfifo($mapfifo, 0600);
6155     my $fifofh;
6156
6157     my $openfifo = sub {
6158         open($fifofh, '>', $mapfifo) || die $!;
6159     };
6160
6161     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6162
6163     my $oldtimeout;
6164     my $timeout = 5;
6165
6166     my $devinfo = {};
6167
6168     my $rpcenv = PVE::RPCEnvironment::get();
6169
6170     my $conffile = PVE::QemuConfig->config_file($vmid);
6171     my $tmpfn = "$conffile.$$.tmp";
6172
6173     # Note: $oldconf is undef if VM does not exist
6174     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6175     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6176
6177     my %storage_limits;
6178
6179     my $print_devmap = sub {
6180         my $cfgfn = "$tmpdir/qemu-server.conf";
6181
6182         # we can read the config - that is already extracted
6183         my $fh = IO::File->new($cfgfn, "r") ||
6184             "unable to read qemu-server.conf - $!\n";
6185
6186         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6187         if (-f $fwcfgfn) {
6188             my $pve_firewall_dir = '/etc/pve/firewall';
6189             mkdir $pve_firewall_dir; # make sure the dir exists
6190             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6191         }
6192
6193         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6194
6195         foreach my $key (keys %storage_limits) {
6196             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6197             next if !$limit;
6198             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6199             $storage_limits{$key} = $limit * 1024;
6200         }
6201
6202         foreach my $devname (keys %$devinfo) {
6203             die "found no device mapping information for device '$devname'\n"
6204                 if !$devinfo->{$devname}->{virtdev};
6205         }
6206
6207         # create empty/temp config
6208         if ($oldconf) {
6209             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6210             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6211         }
6212
6213         # allocate volumes
6214         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6215
6216         # print restore information to $fifofh
6217         foreach my $virtdev (sort keys %$virtdev_hash) {
6218             my $d = $virtdev_hash->{$virtdev};
6219             next if $d->{is_cloudinit}; # no need to restore cloudinit
6220
6221             my $storeid = $d->{storeid};
6222             my $volid = $d->{volid};
6223
6224             my $map_opts = '';
6225             if (my $limit = $storage_limits{$storeid}) {
6226                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6227             }
6228
6229             my $write_zeros = 1;
6230             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6231                 $write_zeros = 0;
6232             }
6233
6234             my $path = PVE::Storage::path($cfg, $volid);
6235
6236             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6237
6238             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6239         }
6240
6241         $fh->seek(0, 0) || die "seek failed - $!\n";
6242
6243         my $outfd = new IO::File ($tmpfn, "w") ||
6244             die "unable to write config for VM $vmid\n";
6245
6246         my $cookie = { netcount => 0 };
6247         while (defined(my $line = <$fh>)) {
6248             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6249         }
6250
6251         $fh->close();
6252         $outfd->close();
6253     };
6254
6255     eval {
6256         # enable interrupts
6257         local $SIG{INT} =
6258             local $SIG{TERM} =
6259             local $SIG{QUIT} =
6260             local $SIG{HUP} =
6261             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6262         local $SIG{ALRM} = sub { die "got timeout\n"; };
6263
6264         $oldtimeout = alarm($timeout);
6265
6266         my $parser = sub {
6267             my $line = shift;
6268
6269             print "$line\n";
6270
6271             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6272                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6273                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6274             } elsif ($line =~ m/^CTIME: /) {
6275                 # we correctly received the vma config, so we can disable
6276                 # the timeout now for disk allocation (set to 10 minutes, so
6277                 # that we always timeout if something goes wrong)
6278                 alarm(600);
6279                 &$print_devmap();
6280                 print $fifofh "done\n";
6281                 my $tmp = $oldtimeout || 0;
6282                 $oldtimeout = undef;
6283                 alarm($tmp);
6284                 close($fifofh);
6285             }
6286         };
6287
6288         print "restore vma archive: $dbg_cmdstring\n";
6289         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6290     };
6291     my $err = $@;
6292
6293     alarm($oldtimeout) if $oldtimeout;
6294
6295     $restore_deactivate_volumes->($cfg, $devinfo);
6296
6297     unlink $mapfifo;
6298     rmtree $tmpdir;
6299
6300     if ($err) {
6301         unlink $tmpfn;
6302         $restore_destroy_volumes->($cfg, $devinfo);
6303         die $err;
6304     }
6305
6306     rename($tmpfn, $conffile) ||
6307         die "unable to commit configuration file '$conffile'\n";
6308
6309     PVE::Cluster::cfs_update(); # make sure we read new file
6310
6311     eval { rescan($vmid, 1); };
6312     warn $@ if $@;
6313 }
6314
6315 sub restore_tar_archive {
6316     my ($archive, $vmid, $user, $opts) = @_;
6317
6318     if ($archive ne '-') {
6319         my $firstfile = tar_archive_read_firstfile($archive);
6320         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6321             if $firstfile ne 'qemu-server.conf';
6322     }
6323
6324     my $storecfg = PVE::Storage::config();
6325
6326     # avoid zombie disks when restoring over an existing VM -> cleanup first
6327     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6328     # skiplock=1 because qmrestore has set the 'create' lock itself already
6329     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6330     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6331
6332     my $tocmd = "/usr/lib/qemu-server/qmextract";
6333
6334     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6335     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6336     $tocmd .= ' --prealloc' if $opts->{prealloc};
6337     $tocmd .= ' --info' if $opts->{info};
6338
6339     # tar option "xf" does not autodetect compression when read from STDIN,
6340     # so we pipe to zcat
6341     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6342         PVE::Tools::shellquote("--to-command=$tocmd");
6343
6344     my $tmpdir = "/var/tmp/vzdumptmp$$";
6345     mkpath $tmpdir;
6346
6347     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6348     local $ENV{VZDUMP_VMID} = $vmid;
6349     local $ENV{VZDUMP_USER} = $user;
6350
6351     my $conffile = PVE::QemuConfig->config_file($vmid);
6352     my $tmpfn = "$conffile.$$.tmp";
6353
6354     # disable interrupts (always do cleanups)
6355     local $SIG{INT} =
6356         local $SIG{TERM} =
6357         local $SIG{QUIT} =
6358         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6359
6360     eval {
6361         # enable interrupts
6362         local $SIG{INT} =
6363             local $SIG{TERM} =
6364             local $SIG{QUIT} =
6365             local $SIG{HUP} =
6366             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6367
6368         if ($archive eq '-') {
6369             print "extracting archive from STDIN\n";
6370             run_command($cmd, input => "<&STDIN");
6371         } else {
6372             print "extracting archive '$archive'\n";
6373             run_command($cmd);
6374         }
6375
6376         return if $opts->{info};
6377
6378         # read new mapping
6379         my $map = {};
6380         my $statfile = "$tmpdir/qmrestore.stat";
6381         if (my $fd = IO::File->new($statfile, "r")) {
6382             while (defined (my $line = <$fd>)) {
6383                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6384                     $map->{$1} = $2 if $1;
6385                 } else {
6386                     print STDERR "unable to parse line in statfile - $line\n";
6387                 }
6388             }
6389             $fd->close();
6390         }
6391
6392         my $confsrc = "$tmpdir/qemu-server.conf";
6393
6394         my $srcfd = new IO::File($confsrc, "r") ||
6395             die "unable to open file '$confsrc'\n";
6396
6397         my $outfd = new IO::File ($tmpfn, "w") ||
6398             die "unable to write config for VM $vmid\n";
6399
6400         my $cookie = { netcount => 0 };
6401         while (defined (my $line = <$srcfd>)) {
6402             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6403         }
6404
6405         $srcfd->close();
6406         $outfd->close();
6407     };
6408     if (my $err = $@) {
6409         unlink $tmpfn;
6410         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6411         die $err;
6412     }
6413
6414     rmtree $tmpdir;
6415
6416     rename $tmpfn, $conffile ||
6417         die "unable to commit configuration file '$conffile'\n";
6418
6419     PVE::Cluster::cfs_update(); # make sure we read new file
6420
6421     eval { rescan($vmid, 1); };
6422     warn $@ if $@;
6423 };
6424
6425 sub foreach_storage_used_by_vm {
6426     my ($conf, $func) = @_;
6427
6428     my $sidhash = {};
6429
6430     PVE::QemuConfig->foreach_volume($conf, sub {
6431         my ($ds, $drive) = @_;
6432         return if drive_is_cdrom($drive);
6433
6434         my $volid = $drive->{file};
6435
6436         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6437         $sidhash->{$sid} = $sid if $sid;
6438     });
6439
6440     foreach my $sid (sort keys %$sidhash) {
6441         &$func($sid);
6442     }
6443 }
6444
6445 my $qemu_snap_storage = {
6446     rbd => 1,
6447 };
6448 sub do_snapshots_with_qemu {
6449     my ($storecfg, $volid) = @_;
6450
6451     my $storage_name = PVE::Storage::parse_volume_id($volid);
6452     my $scfg = $storecfg->{ids}->{$storage_name};
6453
6454     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6455         return 1;
6456     }
6457
6458     if ($volid =~ m/\.(qcow2|qed)$/){
6459         return 1;
6460     }
6461
6462     return undef;
6463 }
6464
6465 sub qga_check_running {
6466     my ($vmid, $nowarn) = @_;
6467
6468     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6469     if ($@) {
6470         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6471         return 0;
6472     }
6473     return 1;
6474 }
6475
6476 sub template_create {
6477     my ($vmid, $conf, $disk) = @_;
6478
6479     my $storecfg = PVE::Storage::config();
6480
6481     PVE::QemuConfig->foreach_volume($conf, sub {
6482         my ($ds, $drive) = @_;
6483
6484         return if drive_is_cdrom($drive);
6485         return if $disk && $ds ne $disk;
6486
6487         my $volid = $drive->{file};
6488         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6489
6490         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6491         $drive->{file} = $voliddst;
6492         $conf->{$ds} = print_drive($drive);
6493         PVE::QemuConfig->write_config($vmid, $conf);
6494     });
6495 }
6496
6497 sub convert_iscsi_path {
6498     my ($path) = @_;
6499
6500     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6501         my $portal = $1;
6502         my $target = $2;
6503         my $lun = $3;
6504
6505         my $initiator_name = get_initiator_name();
6506
6507         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6508                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6509     }
6510
6511     die "cannot convert iscsi path '$path', unkown format\n";
6512 }
6513
6514 sub qemu_img_convert {
6515     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6516
6517     my $storecfg = PVE::Storage::config();
6518     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6519     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6520
6521     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6522
6523     my $cachemode;
6524     my $src_path;
6525     my $src_is_iscsi = 0;
6526     my $src_format;
6527
6528     if ($src_storeid) {
6529         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6530         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6531         $src_format = qemu_img_format($src_scfg, $src_volname);
6532         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6533         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6534         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6535     } elsif (-f $src_volid) {
6536         $src_path = $src_volid;
6537         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6538             $src_format = $1;
6539         }
6540     }
6541
6542     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6543
6544     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6545     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6546     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6547     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6548
6549     my $cmd = [];
6550     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6551     push @$cmd, '-l', "snapshot.name=$snapname"
6552         if $snapname && $src_format && $src_format eq "qcow2";
6553     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6554     push @$cmd, '-T', $cachemode if defined($cachemode);
6555
6556     if ($src_is_iscsi) {
6557         push @$cmd, '--image-opts';
6558         $src_path = convert_iscsi_path($src_path);
6559     } elsif ($src_format) {
6560         push @$cmd, '-f', $src_format;
6561     }
6562
6563     if ($dst_is_iscsi) {
6564         push @$cmd, '--target-image-opts';
6565         $dst_path = convert_iscsi_path($dst_path);
6566     } else {
6567         push @$cmd, '-O', $dst_format;
6568     }
6569
6570     push @$cmd, $src_path;
6571
6572     if (!$dst_is_iscsi && $is_zero_initialized) {
6573         push @$cmd, "zeroinit:$dst_path";
6574     } else {
6575         push @$cmd, $dst_path;
6576     }
6577
6578     my $parser = sub {
6579         my $line = shift;
6580         if($line =~ m/\((\S+)\/100\%\)/){
6581             my $percent = $1;
6582             my $transferred = int($size * $percent / 100);
6583             my $remaining = $size - $transferred;
6584
6585             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6586         }
6587
6588     };
6589
6590     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6591     my $err = $@;
6592     die "copy failed: $err" if $err;
6593 }
6594
6595 sub qemu_img_format {
6596     my ($scfg, $volname) = @_;
6597
6598     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6599         return $1;
6600     } else {
6601         return "raw";
6602     }
6603 }
6604
6605 sub qemu_drive_mirror {
6606     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6607
6608     $jobs = {} if !$jobs;
6609
6610     my $qemu_target;
6611     my $format;
6612     $jobs->{"drive-$drive"} = {};
6613
6614     if ($dst_volid =~ /^nbd:/) {
6615         $qemu_target = $dst_volid;
6616         $format = "nbd";
6617     } else {
6618         my $storecfg = PVE::Storage::config();
6619         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6620
6621         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6622
6623         $format = qemu_img_format($dst_scfg, $dst_volname);
6624
6625         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6626
6627         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6628     }
6629
6630     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6631     $opts->{format} = $format if $format;
6632
6633     if (defined($src_bitmap)) {
6634         $opts->{sync} = 'incremental';
6635         $opts->{bitmap} = $src_bitmap;
6636         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6637     }
6638
6639     if (defined($bwlimit)) {
6640         $opts->{speed} = $bwlimit * 1024;
6641         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6642     } else {
6643         print "drive mirror is starting for drive-$drive\n";
6644     }
6645
6646     # if a job already runs for this device we get an error, catch it for cleanup
6647     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6648     if (my $err = $@) {
6649         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6650         warn "$@\n" if $@;
6651         die "mirroring error: $err\n";
6652     }
6653
6654     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6655 }
6656
6657 # $completion can be either
6658 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6659 # 'cancel': wait until all jobs are ready, block-job-cancel them
6660 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6661 sub qemu_drive_mirror_monitor {
6662     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6663
6664     $completion //= 'complete';
6665
6666     eval {
6667         my $err_complete = 0;
6668
6669         while (1) {
6670             die "storage migration timed out\n" if $err_complete > 300;
6671
6672             my $stats = mon_cmd($vmid, "query-block-jobs");
6673
6674             my $running_mirror_jobs = {};
6675             foreach my $stat (@$stats) {
6676                 next if $stat->{type} ne 'mirror';
6677                 $running_mirror_jobs->{$stat->{device}} = $stat;
6678             }
6679
6680             my $readycounter = 0;
6681
6682             foreach my $job (keys %$jobs) {
6683
6684                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6685                     print "$job : finished\n";
6686                     delete $jobs->{$job};
6687                     next;
6688                 }
6689
6690                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6691
6692                 my $busy = $running_mirror_jobs->{$job}->{busy};
6693                 my $ready = $running_mirror_jobs->{$job}->{ready};
6694                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6695                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6696                     my $remaining = $total - $transferred;
6697                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6698
6699                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6700                 }
6701
6702                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6703             }
6704
6705             last if scalar(keys %$jobs) == 0;
6706
6707             if ($readycounter == scalar(keys %$jobs)) {
6708                 print "all mirroring jobs are ready \n";
6709                 last if $completion eq 'skip'; #do the complete later
6710
6711                 if ($vmiddst && $vmiddst != $vmid) {
6712                     my $agent_running = $qga && qga_check_running($vmid);
6713                     if ($agent_running) {
6714                         print "freeze filesystem\n";
6715                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6716                     } else {
6717                         print "suspend vm\n";
6718                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6719                     }
6720
6721                     # if we clone a disk for a new target vm, we don't switch the disk
6722                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6723
6724                     if ($agent_running) {
6725                         print "unfreeze filesystem\n";
6726                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6727                     } else {
6728                         print "resume vm\n";
6729                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6730                     }
6731
6732                     last;
6733                 } else {
6734
6735                     foreach my $job (keys %$jobs) {
6736                         # try to switch the disk if source and destination are on the same guest
6737                         print "$job: Completing block job...\n";
6738
6739                         my $op;
6740                         if ($completion eq 'complete') {
6741                             $op = 'block-job-complete';
6742                         } elsif ($completion eq 'cancel') {
6743                             $op = 'block-job-cancel';
6744                         } else {
6745                             die "invalid completion value: $completion\n";
6746                         }
6747                         eval { mon_cmd($vmid, $op, device => $job) };
6748                         if ($@ =~ m/cannot be completed/) {
6749                             print "$job: Block job cannot be completed, try again.\n";
6750                             $err_complete++;
6751                         }else {
6752                             print "$job: Completed successfully.\n";
6753                             $jobs->{$job}->{complete} = 1;
6754                         }
6755                     }
6756                 }
6757             }
6758             sleep 1;
6759         }
6760     };
6761     my $err = $@;
6762
6763     if ($err) {
6764         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6765         die "mirroring error: $err";
6766     }
6767
6768 }
6769
6770 sub qemu_blockjobs_cancel {
6771     my ($vmid, $jobs) = @_;
6772
6773     foreach my $job (keys %$jobs) {
6774         print "$job: Cancelling block job\n";
6775         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6776         $jobs->{$job}->{cancel} = 1;
6777     }
6778
6779     while (1) {
6780         my $stats = mon_cmd($vmid, "query-block-jobs");
6781
6782         my $running_jobs = {};
6783         foreach my $stat (@$stats) {
6784             $running_jobs->{$stat->{device}} = $stat;
6785         }
6786
6787         foreach my $job (keys %$jobs) {
6788
6789             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6790                 print "$job: Done.\n";
6791                 delete $jobs->{$job};
6792             }
6793         }
6794
6795         last if scalar(keys %$jobs) == 0;
6796
6797         sleep 1;
6798     }
6799 }
6800
6801 sub clone_disk {
6802     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6803         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6804
6805     my $newvolid;
6806
6807     if (!$full) {
6808         print "create linked clone of drive $drivename ($drive->{file})\n";
6809         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6810         push @$newvollist, $newvolid;
6811     } else {
6812
6813         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6814         $storeid = $storage if $storage;
6815
6816         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6817         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6818
6819         print "create full clone of drive $drivename ($drive->{file})\n";
6820         my $name = undef;
6821         if (drive_is_cloudinit($drive)) {
6822             $name = "vm-$newvmid-cloudinit";
6823             $name .= ".$dst_format" if $dst_format ne 'raw';
6824             $snapname = undef;
6825             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6826         } elsif ($drivename eq 'efidisk0') {
6827             $size = get_efivars_size($conf);
6828         }
6829         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6830         push @$newvollist, $newvolid;
6831
6832         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6833
6834         if (drive_is_cloudinit($drive)) {
6835             goto no_data_clone;
6836         }
6837
6838         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6839         if (!$running || $snapname) {
6840             # TODO: handle bwlimits
6841             if ($drivename eq 'efidisk0') {
6842                 # the relevant data on the efidisk may be smaller than the source
6843                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6844                 # that is given by the OVMF_VARS.fd
6845                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6846                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6847                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6848             } else {
6849                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6850             }
6851         } else {
6852
6853             my $kvmver = get_running_qemu_version ($vmid);
6854             if (!min_version($kvmver, 2, 7)) {
6855                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6856                     if $drive->{iothread};
6857             }
6858
6859             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6860         }
6861     }
6862
6863 no_data_clone:
6864     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6865
6866     my $disk = $drive;
6867     $disk->{format} = undef;
6868     $disk->{file} = $newvolid;
6869     $disk->{size} = $size;
6870
6871     return $disk;
6872 }
6873
6874 sub get_running_qemu_version {
6875     my ($vmid) = @_;
6876     my $res = mon_cmd($vmid, "query-version");
6877     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6878 }
6879
6880 sub qemu_use_old_bios_files {
6881     my ($machine_type) = @_;
6882
6883     return if !$machine_type;
6884
6885     my $use_old_bios_files = undef;
6886
6887     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6888         $machine_type = $1;
6889         $use_old_bios_files = 1;
6890     } else {
6891         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6892         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6893         # load new efi bios files on migration. So this hack is required to allow
6894         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6895         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6896         $use_old_bios_files = !min_version($version, 2, 4);
6897     }
6898
6899     return ($use_old_bios_files, $machine_type);
6900 }
6901
6902 sub get_efivars_size {
6903     my ($conf) = @_;
6904     my $arch = get_vm_arch($conf);
6905     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6906     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6907     return -s $ovmf_vars;
6908 }
6909
6910 sub update_efidisk_size {
6911     my ($conf) = @_;
6912
6913     return if !defined($conf->{efidisk0});
6914
6915     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6916     $disk->{size} = get_efivars_size($conf);
6917     $conf->{efidisk0} = print_drive($disk);
6918
6919     return;
6920 }
6921
6922 sub create_efidisk($$$$$) {
6923     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6924
6925     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6926     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6927
6928     my $vars_size_b = -s $ovmf_vars;
6929     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6930     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6931     PVE::Storage::activate_volumes($storecfg, [$volid]);
6932
6933     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6934     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6935
6936     return ($volid, $size/1024);
6937 }
6938
6939 sub vm_iothreads_list {
6940     my ($vmid) = @_;
6941
6942     my $res = mon_cmd($vmid, 'query-iothreads');
6943
6944     my $iothreads = {};
6945     foreach my $iothread (@$res) {
6946         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6947     }
6948
6949     return $iothreads;
6950 }
6951
6952 sub scsihw_infos {
6953     my ($conf, $drive) = @_;
6954
6955     my $maxdev = 0;
6956
6957     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6958         $maxdev = 7;
6959     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6960         $maxdev = 1;
6961     } else {
6962         $maxdev = 256;
6963     }
6964
6965     my $controller = int($drive->{index} / $maxdev);
6966     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6967
6968     return ($maxdev, $controller, $controller_prefix);
6969 }
6970
6971 sub windows_version {
6972     my ($ostype) = @_;
6973
6974     return 0 if !$ostype;
6975
6976     my $winversion = 0;
6977
6978     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6979         $winversion = 5;
6980     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6981         $winversion = 6;
6982     } elsif ($ostype =~ m/^win(\d+)$/) {
6983         $winversion = $1;
6984     }
6985
6986     return $winversion;
6987 }
6988
6989 sub resolve_dst_disk_format {
6990         my ($storecfg, $storeid, $src_volname, $format) = @_;
6991         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6992
6993         if (!$format) {
6994             # if no target format is specified, use the source disk format as hint
6995             if ($src_volname) {
6996                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6997                 $format = qemu_img_format($scfg, $src_volname);
6998             } else {
6999                 return $defFormat;
7000             }
7001         }
7002
7003         # test if requested format is supported - else use default
7004         my $supported = grep { $_ eq $format } @$validFormats;
7005         $format = $defFormat if !$supported;
7006         return $format;
7007 }
7008
7009 # NOTE: if this logic changes, please update docs & possibly gui logic
7010 sub find_vmstate_storage {
7011     my ($conf, $storecfg) = @_;
7012
7013     # first, return storage from conf if set
7014     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7015
7016     my ($target, $shared, $local);
7017
7018     foreach_storage_used_by_vm($conf, sub {
7019         my ($sid) = @_;
7020         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7021         my $dst = $scfg->{shared} ? \$shared : \$local;
7022         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7023     });
7024
7025     # second, use shared storage where VM has at least one disk
7026     # third, use local storage where VM has at least one disk
7027     # fall back to local storage
7028     $target = $shared // $local // 'local';
7029
7030     return $target;
7031 }
7032
7033 sub generate_uuid {
7034     my ($uuid, $uuid_str);
7035     UUID::generate($uuid);
7036     UUID::unparse($uuid, $uuid_str);
7037     return $uuid_str;
7038 }
7039
7040 sub generate_smbios1_uuid {
7041     return "uuid=".generate_uuid();
7042 }
7043
7044 sub nbd_stop {
7045     my ($vmid) = @_;
7046
7047     mon_cmd($vmid, 'nbd-server-stop');
7048 }
7049
7050 sub create_reboot_request {
7051     my ($vmid) = @_;
7052     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7053         or die "failed to create reboot trigger file: $!\n";
7054     close($fh);
7055 }
7056
7057 sub clear_reboot_request {
7058     my ($vmid) = @_;
7059     my $path = "/run/qemu-server/$vmid.reboot";
7060     my $res = 0;
7061
7062     $res = unlink($path);
7063     die "could not remove reboot request for $vmid: $!"
7064         if !$res && $! != POSIX::ENOENT;
7065
7066     return $res;
7067 }
7068
7069 # bash completion helper
7070
7071 sub complete_backup_archives {
7072     my ($cmdname, $pname, $cvalue) = @_;
7073
7074     my $cfg = PVE::Storage::config();
7075
7076     my $storeid;
7077
7078     if ($cvalue =~ m/^([^:]+):/) {
7079         $storeid = $1;
7080     }
7081
7082     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7083
7084     my $res = [];
7085     foreach my $id (keys %$data) {
7086         foreach my $item (@{$data->{$id}}) {
7087             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7088             push @$res, $item->{volid} if defined($item->{volid});
7089         }
7090     }
7091
7092     return $res;
7093 }
7094
7095 my $complete_vmid_full = sub {
7096     my ($running) = @_;
7097
7098     my $idlist = vmstatus();
7099
7100     my $res = [];
7101
7102     foreach my $id (keys %$idlist) {
7103         my $d = $idlist->{$id};
7104         if (defined($running)) {
7105             next if $d->{template};
7106             next if $running && $d->{status} ne 'running';
7107             next if !$running && $d->{status} eq 'running';
7108         }
7109         push @$res, $id;
7110
7111     }
7112     return $res;
7113 };
7114
7115 sub complete_vmid {
7116     return &$complete_vmid_full();
7117 }
7118
7119 sub complete_vmid_stopped {
7120     return &$complete_vmid_full(0);
7121 }
7122
7123 sub complete_vmid_running {
7124     return &$complete_vmid_full(1);
7125 }
7126
7127 sub complete_storage {
7128
7129     my $cfg = PVE::Storage::config();
7130     my $ids = $cfg->{ids};
7131
7132     my $res = [];
7133     foreach my $sid (keys %$ids) {
7134         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7135         next if !$ids->{$sid}->{content}->{images};
7136         push @$res, $sid;
7137     }
7138
7139     return $res;
7140 }
7141
7142 sub complete_migration_storage {
7143     my ($cmd, $param, $current_value, $all_args) = @_;
7144
7145     my $targetnode = @$all_args[1];
7146
7147     my $cfg = PVE::Storage::config();
7148     my $ids = $cfg->{ids};
7149
7150     my $res = [];
7151     foreach my $sid (keys %$ids) {
7152         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7153         next if !$ids->{$sid}->{content}->{images};
7154         push @$res, $sid;
7155     }
7156
7157     return $res;
7158 }
7159
7160 1;