PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if defined($map->{entries}) && $map->{entries}->{$source};
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     machine => get_standard_option('pve-qemu-machine'),
 595     arch => {
 596         description => "Virtual processor architecture. Defaults to the host.",
 597         optional => 1,
 598         type => 'string',
 599         enum => [qw(x86_64 aarch64)],
 600     },
 601     smbios1 => {
 602         description => "Specify SMBIOS type 1 fields.",
 603         type => 'string', format => 'pve-qm-smbios1',
 604         maxLength => 512,
 605         optional => 1,
 606     },
 607     protection => {
 608         optional => 1,
 609         type => 'boolean',
 610         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 611         default => 0,
 612     },
 613     bios => {
 614         optional => 1,
 615         type => 'string',
 616         enum => [ qw(seabios ovmf) ],
 617         description => "Select BIOS implementation.",
 618         default => 'seabios',
 619     },
 620     vmgenid => {
 621         type => 'string',
 622         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 623         format_description => 'UUID',
 624         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 625         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 626             " 128-bit integer value identifier to the guest OS. This allows to".
 627             " notify the guest operating system when the virtual machine is".
 628             " executed with a different configuration (e.g. snapshot execution".
 629             " or creation from a template). The guest operating system notices".
 630             " the change, and is then able to react as appropriate by marking".
 631             " its copies of distributed databases as dirty, re-initializing its".
 632             " random number generator, etc.\n".
 633             "Note that auto-creation only works when done throug API/CLI create".
 634             " or update methods, but not when manually editing the config file.",
 635         default => "1 (autogenerated)",
 636         optional => 1,
 637     },
 638     hookscript => {
 639         type => 'string',
 640         format => 'pve-volume-id',
 641         optional => 1,
 642         description => "Script that will be executed during various steps in the vms lifetime.",
 643     },
 644     ivshmem => {
 645         type => 'string',
 646         format => $ivshmem_fmt,
 647         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 648         optional => 1,
 649     },
 650     audio0 => {
 651         type => 'string',
 652         format => $audio_fmt,
 653         description => "Configure a audio device, useful in combination with QXL/Spice.",
 654         optional => 1
 655     },
 656     spice_enhancements => {
 657         type => 'string',
 658         format => $spice_enhancements_fmt,
 659         description => "Configure additional enhancements for SPICE.",
 660         optional => 1
 661     },
 662     tags => {
 663         type => 'string', format => 'pve-tag-list',
 664         description => 'Tags of the VM. This is only meta information.',
 665         optional => 1,
 666     },
 667     rng0 => {
 668         type => 'string',
 669         format => $rng_fmt,
 670         description => "Configure a VirtIO-based Random Number Generator.",
 671         optional => 1,
 672     },
 673 };
 674
 675 my $cicustom_fmt = {
 676     meta => {
 677         type => 'string',
 678         optional => 1,
 679         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 680         format => 'pve-volume-id',
 681         format_description => 'volume',
 682     },
 683     network => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     user => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697 };
 698 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 699
 700 my $confdesc_cloudinit = {
 701     citype => {
 702         optional => 1,
 703         type => 'string',
 704         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 705         enum => ['configdrive2', 'nocloud'],
 706     },
 707     ciuser => {
 708         optional => 1,
 709         type => 'string',
 710         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 711     },
 712     cipassword => {
 713         optional => 1,
 714         type => 'string',
 715         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 716     },
 717     cicustom => {
 718         optional => 1,
 719         type => 'string',
 720         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 721         format => 'pve-qm-cicustom',
 722     },
 723     searchdomain => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 727     },
 728     nameserver => {
 729         optional => 1,
 730         type => 'string', format => 'address-list',
 731         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 732     },
 733     sshkeys => {
 734         optional => 1,
 735         type => 'string',
 736         format => 'urlencoded',
 737         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 738     },
 739 };
 740
 741 # what about other qemu settings ?
 742 #cpu => 'string',
 743 #machine => 'string',
 744 #fda => 'file',
 745 #fdb => 'file',
 746 #mtdblock => 'file',
 747 #sd => 'file',
 748 #pflash => 'file',
 749 #snapshot => 'bool',
 750 #bootp => 'file',
 751 ##tftp => 'dir',
 752 ##smb => 'dir',
 753 #kernel => 'file',
 754 #append => 'string',
 755 #initrd => 'file',
 756 ##soundhw => 'string',
 757
 758 while (my ($k, $v) = each %$confdesc) {
 759     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 760 }
 761
 762 my $MAX_USB_DEVICES = 5;
 763 my $MAX_NETS = 32;
 764 my $MAX_HOSTPCI_DEVICES = 16;
 765 my $MAX_SERIAL_PORTS = 4;
 766 my $MAX_PARALLEL_PORTS = 3;
 767 my $MAX_NUMA = 8;
 768
 769 my $numa_fmt = {
 770     cpus => {
 771         type => "string",
 772         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 773         description => "CPUs accessing this NUMA node.",
 774         format_description => "id[-id];...",
 775     },
 776     memory => {
 777         type => "number",
 778         description => "Amount of memory this NUMA node provides.",
 779         optional => 1,
 780     },
 781     hostnodes => {
 782         type => "string",
 783         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 784         description => "Host NUMA nodes to use.",
 785         format_description => "id[-id];...",
 786         optional => 1,
 787     },
 788     policy => {
 789         type => 'string',
 790         enum => [qw(preferred bind interleave)],
 791         description => "NUMA allocation policy.",
 792         optional => 1,
 793     },
 794 };
 795 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 796 my $numadesc = {
 797     optional => 1,
 798     type => 'string', format => $numa_fmt,
 799     description => "NUMA topology.",
 800 };
 801 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 802
 803 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 804     $confdesc->{"numa$i"} = $numadesc;
 805 }
 806
 807 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 808                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 809                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 810 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 811
 812 my $net_fmt_bridge_descr = <<__EOD__;
 813 Bridge to attach the network device to. The Proxmox VE standard bridge
 814 is called 'vmbr0'.
 815
 816 If you do not specify a bridge, we create a kvm user (NATed) network
 817 device, which provides DHCP and DNS services. The following addresses
 818 are used:
 819
 820  10.0.2.2   Gateway
 821  10.0.2.3   DNS Server
 822  10.0.2.4   SMB Server
 823
 824 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 825 __EOD__
 826
 827 my $net_fmt = {
 828     macaddr  => get_standard_option('mac-addr', {
 829         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 830     }),
 831     model => {
 832         type => 'string',
 833         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 834         enum => $nic_model_list,
 835         default_key => 1,
 836     },
 837     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 838     bridge => {
 839         type => 'string',
 840         description => $net_fmt_bridge_descr,
 841         format_description => 'bridge',
 842         optional => 1,
 843     },
 844     queues => {
 845         type => 'integer',
 846         minimum => 0, maximum => 16,
 847         description => 'Number of packet queues to be used on the device.',
 848         optional => 1,
 849     },
 850     rate => {
 851         type => 'number',
 852         minimum => 0,
 853         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 854         optional => 1,
 855     },
 856     tag => {
 857         type => 'integer',
 858         minimum => 1, maximum => 4094,
 859         description => 'VLAN tag to apply to packets on this interface.',
 860         optional => 1,
 861     },
 862     trunks => {
 863         type => 'string',
 864         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 865         description => 'VLAN trunks to pass through this interface.',
 866         format_description => 'vlanid[;vlanid...]',
 867         optional => 1,
 868     },
 869     firewall => {
 870         type => 'boolean',
 871         description => 'Whether this interface should be protected by the firewall.',
 872         optional => 1,
 873     },
 874     link_down => {
 875         type => 'boolean',
 876         description => 'Whether this interface should be disconnected (like pulling the plug).',
 877         optional => 1,
 878     },
 879 };
 880
 881 my $netdesc = {
 882     optional => 1,
 883     type => 'string', format => $net_fmt,
 884     description => "Specify network devices.",
 885 };
 886
 887 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 888
 889 my $ipconfig_fmt = {
 890     ip => {
 891         type => 'string',
 892         format => 'pve-ipv4-config',
 893         format_description => 'IPv4Format/CIDR',
 894         description => 'IPv4 address in CIDR format.',
 895         optional => 1,
 896         default => 'dhcp',
 897     },
 898     gw => {
 899         type => 'string',
 900         format => 'ipv4',
 901         format_description => 'GatewayIPv4',
 902         description => 'Default gateway for IPv4 traffic.',
 903         optional => 1,
 904         requires => 'ip',
 905     },
 906     ip6 => {
 907         type => 'string',
 908         format => 'pve-ipv6-config',
 909         format_description => 'IPv6Format/CIDR',
 910         description => 'IPv6 address in CIDR format.',
 911         optional => 1,
 912         default => 'dhcp',
 913     },
 914     gw6 => {
 915         type => 'string',
 916         format => 'ipv6',
 917         format_description => 'GatewayIPv6',
 918         description => 'Default gateway for IPv6 traffic.',
 919         optional => 1,
 920         requires => 'ip6',
 921     },
 922 };
 923 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 924 my $ipconfigdesc = {
 925     optional => 1,
 926     type => 'string', format => 'pve-qm-ipconfig',
 927     description => <<'EODESCR',
 928 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 929
 930 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 931
 932 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 933 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 934
 935 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 936 EODESCR
 937 };
 938 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 939
 940 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 941     $confdesc->{"net$i"} = $netdesc;
 942     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 943 }
 944
 945 foreach my $key (keys %$confdesc_cloudinit) {
 946     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 947 }
 948
 949 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 950 sub verify_volume_id_or_qm_path {
 951     my ($volid, $noerr) = @_;
 952
 953     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 954         return $volid;
 955     }
 956
 957     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 958     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 959     if ($@) {
 960         return undef if $noerr;
 961         die $@;
 962     }
 963     return $volid;
 964 }
 965
 966 my $usb_fmt = {
 967     host => {
 968         default_key => 1,
 969         type => 'string', format => 'pve-qm-usb-device',
 970         format_description => 'HOSTUSBDEVICE|spice',
 971         description => <<EODESCR,
 972 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 973
 974  'bus-port(.port)*' (decimal numbers) or
 975  'vendor_id:product_id' (hexadeciaml numbers) or
 976  'spice'
 977
 978 You can use the 'lsusb -t' command to list existing usb devices.
 979
 980 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 981
 982 The value 'spice' can be used to add a usb redirection devices for spice.
 983 EODESCR
 984     },
 985     usb3 => {
 986         optional => 1,
 987         type => 'boolean',
 988         description => "Specifies whether if given host option is a USB3 device or port.",
 989         default => 0,
 990     },
 991 };
 992
 993 my $usbdesc = {
 994     optional => 1,
 995     type => 'string', format => $usb_fmt,
 996     description => "Configure an USB device (n is 0 to 4).",
 997 };
 998 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 999
1000 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1001 my $hostpci_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string',
1005         pattern => qr/$PCIRE(;$PCIRE)*/,
1006         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1007         description => <<EODESCR,
1008 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1009 of PCI virtual functions of the host. HOSTPCIID syntax is:
1010
1011 'bus:dev.func' (hexadecimal numbers)
1012
1013 You can us the 'lspci' command to list existing PCI devices.
1014 EODESCR
1015     },
1016     rombar => {
1017         type => 'boolean',
1018         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1019         optional => 1,
1020         default => 1,
1021     },
1022     romfile => {
1023         type => 'string',
1024         pattern => '[^,;]+',
1025         format_description => 'string',
1026         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1027         optional => 1,
1028     },
1029     pcie => {
1030         type => 'boolean',
1031         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1032         optional => 1,
1033         default => 0,
1034     },
1035     'x-vga' => {
1036         type => 'boolean',
1037         description =>  "Enable vfio-vga device support.",
1038         optional => 1,
1039         default => 0,
1040     },
1041     'mdev' => {
1042         type => 'string',
1043         format_description => 'string',
1044         pattern => '[^/\.:]+',
1045         optional => 1,
1046         description => <<EODESCR
1047 The type of mediated device to use.
1048 An instance of this type will be created on startup of the VM and
1049 will be cleaned up when the VM stops.
1050 EODESCR
1051     }
1052 };
1053 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1054
1055 my $hostpcidesc = {
1056         optional => 1,
1057         type => 'string', format => 'pve-qm-hostpci',
1058         description => "Map host PCI devices into guest.",
1059         verbose_description =>  <<EODESCR,
1060 Map host PCI devices into guest.
1061
1062 NOTE: This option allows direct access to host hardware. So it is no longer
1063 possible to migrate such machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1069
1070 my $serialdesc = {
1071         optional => 1,
1072         type => 'string',
1073         pattern => '(/dev/.+|socket)',
1074         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1075         verbose_description =>  <<EODESCR,
1076 Create a serial device inside the VM (n is 0 to 3), and pass through a
1077 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1078 host side (use 'qm terminal' to open a terminal connection).
1079
1080 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1081
1082 CAUTION: Experimental! User reported problems with this option.
1083 EODESCR
1084 };
1085
1086 my $paralleldesc= {
1087         optional => 1,
1088         type => 'string',
1089         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1090         description =>  "Map host parallel devices (n is 0 to 2).",
1091         verbose_description =>  <<EODESCR,
1092 Map host parallel devices (n is 0 to 2).
1093
1094 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1095
1096 CAUTION: Experimental! User reported problems with this option.
1097 EODESCR
1098 };
1099
1100 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1101     $confdesc->{"parallel$i"} = $paralleldesc;
1102 }
1103
1104 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1105     $confdesc->{"serial$i"} = $serialdesc;
1106 }
1107
1108 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1109     $confdesc->{"hostpci$i"} = $hostpcidesc;
1110 }
1111
1112 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1113     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1114 }
1115
1116 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1117     $confdesc->{"usb$i"} = $usbdesc;
1118 }
1119
1120 my $kvm_api_version = 0;
1121
1122 sub kvm_version {
1123     return $kvm_api_version if $kvm_api_version;
1124
1125     open my $fh, '<', '/dev/kvm'
1126         or return undef;
1127
1128     # 0xae00 => KVM_GET_API_VERSION
1129     $kvm_api_version = ioctl($fh, 0xae00, 0);
1130
1131     return $kvm_api_version;
1132 }
1133
1134 my $kvm_user_version = {};
1135 my $kvm_mtime = {};
1136
1137 sub kvm_user_version {
1138     my ($binary) = @_;
1139
1140     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1141     my $st = stat($binary);
1142
1143     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1144     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1145         $cachedmtime == $st->mtime;
1146
1147     $kvm_user_version->{$binary} = 'unknown';
1148     $kvm_mtime->{$binary} = $st->mtime;
1149
1150     my $code = sub {
1151         my $line = shift;
1152         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1153             $kvm_user_version->{$binary} = $2;
1154         }
1155     };
1156
1157     eval { run_command([$binary, '--version'], outfunc => $code); };
1158     warn $@ if $@;
1159
1160     return $kvm_user_version->{$binary};
1161
1162 }
1163
1164 sub kernel_has_vhost_net {
1165     return -c '/dev/vhost-net';
1166 }
1167
1168 sub option_exists {
1169     my $key = shift;
1170     return defined($confdesc->{$key});
1171 }
1172
1173 my $cdrom_path;
1174 sub get_cdrom_path {
1175
1176     return  $cdrom_path if $cdrom_path;
1177
1178     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1179     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1180     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1181 }
1182
1183 sub get_iso_path {
1184     my ($storecfg, $vmid, $cdrom) = @_;
1185
1186     if ($cdrom eq 'cdrom') {
1187         return get_cdrom_path();
1188     } elsif ($cdrom eq 'none') {
1189         return '';
1190     } elsif ($cdrom =~ m|^/|) {
1191         return $cdrom;
1192     } else {
1193         return PVE::Storage::path($storecfg, $cdrom);
1194     }
1195 }
1196
1197 # try to convert old style file names to volume IDs
1198 sub filename_to_volume_id {
1199     my ($vmid, $file, $media) = @_;
1200
1201      if (!($file eq 'none' || $file eq 'cdrom' ||
1202           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1203
1204         return undef if $file =~ m|/|;
1205
1206         if ($media && $media eq 'cdrom') {
1207             $file = "local:iso/$file";
1208         } else {
1209             $file = "local:$vmid/$file";
1210         }
1211     }
1212
1213     return $file;
1214 }
1215
1216 sub verify_media_type {
1217     my ($opt, $vtype, $media) = @_;
1218
1219     return if !$media;
1220
1221     my $etype;
1222     if ($media eq 'disk') {
1223         $etype = 'images';
1224     } elsif ($media eq 'cdrom') {
1225         $etype = 'iso';
1226     } else {
1227         die "internal error";
1228     }
1229
1230     return if ($vtype eq $etype);
1231
1232     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1233 }
1234
1235 sub cleanup_drive_path {
1236     my ($opt, $storecfg, $drive) = @_;
1237
1238     # try to convert filesystem paths to volume IDs
1239
1240     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1241         ($drive->{file} !~ m|^/dev/.+|) &&
1242         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1243         ($drive->{file} !~ m/^\d+$/)) {
1244         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1245         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1246         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1247         verify_media_type($opt, $vtype, $drive->{media});
1248         $drive->{file} = $volid;
1249     }
1250
1251     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1252 }
1253
1254 sub parse_hotplug_features {
1255     my ($data) = @_;
1256
1257     my $res = {};
1258
1259     return $res if $data eq '0';
1260
1261     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1262
1263     foreach my $feature (PVE::Tools::split_list($data)) {
1264         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1265             $res->{$1} = 1;
1266         } else {
1267             die "invalid hotplug feature '$feature'\n";
1268         }
1269     }
1270     return $res;
1271 }
1272
1273 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1274 sub pve_verify_hotplug_features {
1275     my ($value, $noerr) = @_;
1276
1277     return $value if parse_hotplug_features($value);
1278
1279     return undef if $noerr;
1280
1281     die "unable to parse hotplug option\n";
1282 }
1283
1284 sub scsi_inquiry {
1285     my($fh, $noerr) = @_;
1286
1287     my $SG_IO = 0x2285;
1288     my $SG_GET_VERSION_NUM = 0x2282;
1289
1290     my $versionbuf = "\x00" x 8;
1291     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1292     if (!$ret) {
1293         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1294         return undef;
1295     }
1296     my $version = unpack("I", $versionbuf);
1297     if ($version < 30000) {
1298         die "scsi generic interface too old\n"  if !$noerr;
1299         return undef;
1300     }
1301
1302     my $buf = "\x00" x 36;
1303     my $sensebuf = "\x00" x 8;
1304     my $cmd = pack("C x3 C x1", 0x12, 36);
1305
1306     # see /usr/include/scsi/sg.h
1307     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1308
1309     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1310                       length($sensebuf), 0, length($buf), $buf,
1311                       $cmd, $sensebuf, 6000);
1312
1313     $ret = ioctl($fh, $SG_IO, $packet);
1314     if (!$ret) {
1315         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1316         return undef;
1317     }
1318
1319     my @res = unpack($sg_io_hdr_t, $packet);
1320     if ($res[17] || $res[18]) {
1321         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1322         return undef;
1323     }
1324
1325     my $res = {};
1326     (my $byte0, my $byte1, $res->{vendor},
1327      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1328
1329     $res->{removable} = $byte1 & 128 ? 1 : 0;
1330     $res->{type} = $byte0 & 31;
1331
1332     return $res;
1333 }
1334
1335 sub path_is_scsi {
1336     my ($path) = @_;
1337
1338     my $fh = IO::File->new("+<$path") || return undef;
1339     my $res = scsi_inquiry($fh, 1);
1340     close($fh);
1341
1342     return $res;
1343 }
1344
1345 sub print_tabletdevice_full {
1346     my ($conf, $arch) = @_;
1347
1348     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1349
1350     # we use uhci for old VMs because tablet driver was buggy in older qemu
1351     my $usbbus;
1352     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1353         $usbbus = 'ehci';
1354     } else {
1355         $usbbus = 'uhci';
1356     }
1357
1358     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1359 }
1360
1361 sub print_keyboarddevice_full {
1362     my ($conf, $arch, $machine) = @_;
1363
1364     return undef if $arch ne 'aarch64';
1365
1366     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1367 }
1368
1369 sub print_drivedevice_full {
1370     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1371
1372     my $device = '';
1373     my $maxdev = 0;
1374
1375     if ($drive->{interface} eq 'virtio') {
1376         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1377         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1378         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1379     } elsif ($drive->{interface} eq 'scsi') {
1380
1381         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1382         my $unit = $drive->{index} % $maxdev;
1383         my $devicetype = 'hd';
1384         my $path = '';
1385         if (drive_is_cdrom($drive)) {
1386             $devicetype = 'cd';
1387         } else {
1388             if ($drive->{file} =~ m|^/|) {
1389                 $path = $drive->{file};
1390                 if (my $info = path_is_scsi($path)) {
1391                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1392                         $devicetype = 'block';
1393                     } elsif ($info->{type} == 1) { # tape
1394                         $devicetype = 'generic';
1395                     }
1396                 }
1397             } else {
1398                  $path = PVE::Storage::path($storecfg, $drive->{file});
1399             }
1400
1401             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1402             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1403             if ($path =~ m/^iscsi\:\/\// &&
1404                !min_version($version, 4, 1)) {
1405                 $devicetype = 'generic';
1406             }
1407         }
1408
1409         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1410             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1411         } else {
1412             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1413         }
1414
1415         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1416             $device .= ",rotation_rate=1";
1417         }
1418         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1419
1420     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1421         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1422         my $controller = int($drive->{index} / $maxdev);
1423         my $unit = $drive->{index} % $maxdev;
1424         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1425
1426         $device = "ide-$devicetype";
1427         if ($drive->{interface} eq 'ide') {
1428             $device .= ",bus=ide.$controller,unit=$unit";
1429         } else {
1430             $device .= ",bus=ahci$controller.$unit";
1431         }
1432         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1433
1434         if ($devicetype eq 'hd') {
1435             if (my $model = $drive->{model}) {
1436                 $model = URI::Escape::uri_unescape($model);
1437                 $device .= ",model=$model";
1438             }
1439             if ($drive->{ssd}) {
1440                 $device .= ",rotation_rate=1";
1441             }
1442         }
1443         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1444     } elsif ($drive->{interface} eq 'usb') {
1445         die "implement me";
1446         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1447     } else {
1448         die "unsupported interface type";
1449     }
1450
1451     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1452
1453     if (my $serial = $drive->{serial}) {
1454         $serial = URI::Escape::uri_unescape($serial);
1455         $device .= ",serial=$serial";
1456     }
1457
1458
1459     return $device;
1460 }
1461
1462 sub get_initiator_name {
1463     my $initiator;
1464
1465     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1466     while (defined(my $line = <$fh>)) {
1467         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1468         $initiator = $1;
1469         last;
1470     }
1471     $fh->close();
1472
1473     return $initiator;
1474 }
1475
1476 sub print_drive_commandline_full {
1477     my ($storecfg, $vmid, $drive) = @_;
1478
1479     my $path;
1480     my $volid = $drive->{file};
1481     my $format;
1482
1483     if (drive_is_cdrom($drive)) {
1484         $path = get_iso_path($storecfg, $vmid, $volid);
1485     } else {
1486         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1487         if ($storeid) {
1488             $path = PVE::Storage::path($storecfg, $volid);
1489             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1490             $format = qemu_img_format($scfg, $volname);
1491         } else {
1492             $path = $volid;
1493             $format = "raw";
1494         }
1495    }
1496
1497     my $opts = '';
1498     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1499     foreach my $o (@qemu_drive_options) {
1500         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1501     }
1502
1503     # snapshot only accepts on|off
1504     if (defined($drive->{snapshot})) {
1505         my $v = $drive->{snapshot} ? 'on' : 'off';
1506         $opts .= ",snapshot=$v";
1507     }
1508
1509     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1510         my ($dir, $qmpname) = @$type;
1511         if (my $v = $drive->{"mbps$dir"}) {
1512             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1513         }
1514         if (my $v = $drive->{"mbps${dir}_max"}) {
1515             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1516         }
1517         if (my $v = $drive->{"bps${dir}_max_length"}) {
1518             $opts .= ",throttling.bps$qmpname-max-length=$v";
1519         }
1520         if (my $v = $drive->{"iops${dir}"}) {
1521             $opts .= ",throttling.iops$qmpname=$v";
1522         }
1523         if (my $v = $drive->{"iops${dir}_max"}) {
1524             $opts .= ",throttling.iops$qmpname-max=$v";
1525         }
1526         if (my $v = $drive->{"iops${dir}_max_length"}) {
1527             $opts .= ",throttling.iops$qmpname-max-length=$v";
1528         }
1529     }
1530
1531     $opts .= ",format=$format" if $format && !$drive->{format};
1532
1533     my $cache_direct = 0;
1534
1535     if (my $cache = $drive->{cache}) {
1536         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1537     } elsif (!drive_is_cdrom($drive)) {
1538         $opts .= ",cache=none";
1539         $cache_direct = 1;
1540     }
1541
1542     # aio native works only with O_DIRECT
1543     if (!$drive->{aio}) {
1544         if($cache_direct) {
1545             $opts .= ",aio=native";
1546         } else {
1547             $opts .= ",aio=threads";
1548         }
1549     }
1550
1551     if (!drive_is_cdrom($drive)) {
1552         my $detectzeroes;
1553         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1554             $detectzeroes = 'off';
1555         } elsif ($drive->{discard}) {
1556             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1557         } else {
1558             # This used to be our default with discard not being specified:
1559             $detectzeroes = 'on';
1560         }
1561         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1562     }
1563
1564     my $pathinfo = $path ? "file=$path," : '';
1565
1566     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1567 }
1568
1569 sub print_netdevice_full {
1570     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1571
1572     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1573
1574     my $device = $net->{model};
1575     if ($net->{model} eq 'virtio') {
1576          $device = 'virtio-net-pci';
1577      };
1578
1579     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1580     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1581     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1582         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1583         my $vectors = $net->{queues} * 2 + 2;
1584         $tmpstr .= ",vectors=$vectors,mq=on";
1585     }
1586     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1587
1588     if ($use_old_bios_files) {
1589         my $romfile;
1590         if ($device eq 'virtio-net-pci') {
1591             $romfile = 'pxe-virtio.rom';
1592         } elsif ($device eq 'e1000') {
1593             $romfile = 'pxe-e1000.rom';
1594         } elsif ($device eq 'ne2k') {
1595             $romfile = 'pxe-ne2k_pci.rom';
1596         } elsif ($device eq 'pcnet') {
1597             $romfile = 'pxe-pcnet.rom';
1598         } elsif ($device eq 'rtl8139') {
1599             $romfile = 'pxe-rtl8139.rom';
1600         }
1601         $tmpstr .= ",romfile=$romfile" if $romfile;
1602     }
1603
1604     return $tmpstr;
1605 }
1606
1607 sub print_netdev_full {
1608     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1609
1610     my $i = '';
1611     if ($netid =~ m/^net(\d+)$/) {
1612         $i = int($1);
1613     }
1614
1615     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1616
1617     my $ifname = "tap${vmid}i$i";
1618
1619     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1620     die "interface name '$ifname' is too long (max 15 character)\n"
1621         if length($ifname) >= 16;
1622
1623     my $vhostparam = '';
1624     if (is_native($arch)) {
1625         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1626     }
1627
1628     my $vmname = $conf->{name} || "vm$vmid";
1629
1630     my $netdev = "";
1631     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1632
1633     if ($net->{bridge}) {
1634         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1635     } else {
1636         $netdev = "type=user,id=$netid,hostname=$vmname";
1637     }
1638
1639     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1640
1641     return $netdev;
1642 }
1643
1644 my $vga_map = {
1645     'cirrus' => 'cirrus-vga',
1646     'std' => 'VGA',
1647     'vmware' => 'vmware-svga',
1648     'virtio' => 'virtio-vga',
1649 };
1650
1651 sub print_vga_device {
1652     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1653
1654     my $type = $vga_map->{$vga->{type}};
1655     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1656         $type = 'virtio-gpu';
1657     }
1658     my $vgamem_mb = $vga->{memory};
1659
1660     my $max_outputs = '';
1661     if ($qxlnum) {
1662         $type = $id ? 'qxl' : 'qxl-vga';
1663
1664         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1665             # set max outputs so linux can have up to 4 qxl displays with one device
1666             if (min_version($machine_version, 4, 1)) {
1667                 $max_outputs = ",max_outputs=4";
1668             }
1669         }
1670     }
1671
1672     die "no devicetype for $vga->{type}\n" if !$type;
1673
1674     my $memory = "";
1675     if ($vgamem_mb) {
1676         if ($vga->{type} eq 'virtio') {
1677             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1678             $memory = ",max_hostmem=$bytes";
1679         } elsif ($qxlnum) {
1680             # from https://www.spice-space.org/multiple-monitors.html
1681             $memory = ",vgamem_mb=$vga->{memory}";
1682             my $ram = $vgamem_mb * 4;
1683             my $vram = $vgamem_mb * 2;
1684             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1685         } else {
1686             $memory = ",vgamem_mb=$vga->{memory}";
1687         }
1688     } elsif ($qxlnum && $id) {
1689         $memory = ",ram_size=67108864,vram_size=33554432";
1690     }
1691
1692     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1693     my $vgaid = "vga" . ($id // '');
1694     my $pciaddr;
1695
1696     if ($q35 && $vgaid eq 'vga') {
1697         # the first display uses pcie.0 bus on q35 machines
1698         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1699     } else {
1700         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1701     }
1702
1703     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1704 }
1705
1706 sub parse_number_sets {
1707     my ($set) = @_;
1708     my $res = [];
1709     foreach my $part (split(/;/, $set)) {
1710         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1711             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1712             push @$res, [ $1, $2 ];
1713         } else {
1714             die "invalid range: $part\n";
1715         }
1716     }
1717     return $res;
1718 }
1719
1720 sub parse_numa {
1721     my ($data) = @_;
1722
1723     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1724     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1725     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1726     return $res;
1727 }
1728
1729 sub parse_hostpci {
1730     my ($value) = @_;
1731
1732     return undef if !$value;
1733
1734     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1735
1736     my @idlist = split(/;/, $res->{host});
1737     delete $res->{host};
1738     foreach my $id (@idlist) {
1739         my $devs = PVE::SysFSTools::lspci($id);
1740         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1741         push @{$res->{pciid}}, @$devs;
1742     }
1743     return $res;
1744 }
1745
1746 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1747 sub parse_net {
1748     my ($data) = @_;
1749
1750     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1751     if ($@) {
1752         warn $@;
1753         return undef;
1754     }
1755     if (!defined($res->{macaddr})) {
1756         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1757         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1758     }
1759     return $res;
1760 }
1761
1762 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1763 sub parse_ipconfig {
1764     my ($data) = @_;
1765
1766     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1767     if ($@) {
1768         warn $@;
1769         return undef;
1770     }
1771
1772     if ($res->{gw} && !$res->{ip}) {
1773         warn 'gateway specified without specifying an IP address';
1774         return undef;
1775     }
1776     if ($res->{gw6} && !$res->{ip6}) {
1777         warn 'IPv6 gateway specified without specifying an IPv6 address';
1778         return undef;
1779     }
1780     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1781         warn 'gateway specified together with DHCP';
1782         return undef;
1783     }
1784     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1785         # gw6 + auto/dhcp
1786         warn "IPv6 gateway specified together with $res->{ip6} address";
1787         return undef;
1788     }
1789
1790     if (!$res->{ip} && !$res->{ip6}) {
1791         return { ip => 'dhcp', ip6 => 'dhcp' };
1792     }
1793
1794     return $res;
1795 }
1796
1797 sub print_net {
1798     my $net = shift;
1799
1800     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1801 }
1802
1803 sub add_random_macs {
1804     my ($settings) = @_;
1805
1806     foreach my $opt (keys %$settings) {
1807         next if $opt !~ m/^net(\d+)$/;
1808         my $net = parse_net($settings->{$opt});
1809         next if !$net;
1810         $settings->{$opt} = print_net($net);
1811     }
1812 }
1813
1814 sub vm_is_volid_owner {
1815     my ($storecfg, $vmid, $volid) = @_;
1816
1817     if ($volid !~  m|^/|) {
1818         my ($path, $owner);
1819         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1820         if ($owner && ($owner == $vmid)) {
1821             return 1;
1822         }
1823     }
1824
1825     return undef;
1826 }
1827
1828 sub vmconfig_register_unused_drive {
1829     my ($storecfg, $vmid, $conf, $drive) = @_;
1830
1831     if (drive_is_cloudinit($drive)) {
1832         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1833         warn $@ if $@;
1834     } elsif (!drive_is_cdrom($drive)) {
1835         my $volid = $drive->{file};
1836         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1837             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1838         }
1839     }
1840 }
1841
1842 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1843 my $smbios1_fmt = {
1844     uuid => {
1845         type => 'string',
1846         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1847         format_description => 'UUID',
1848         description => "Set SMBIOS1 UUID.",
1849         optional => 1,
1850     },
1851     version => {
1852         type => 'string',
1853         pattern => '[A-Za-z0-9+\/]+={0,2}',
1854         format_description => 'Base64 encoded string',
1855         description => "Set SMBIOS1 version.",
1856         optional => 1,
1857     },
1858     serial => {
1859         type => 'string',
1860         pattern => '[A-Za-z0-9+\/]+={0,2}',
1861         format_description => 'Base64 encoded string',
1862         description => "Set SMBIOS1 serial number.",
1863         optional => 1,
1864     },
1865     manufacturer => {
1866         type => 'string',
1867         pattern => '[A-Za-z0-9+\/]+={0,2}',
1868         format_description => 'Base64 encoded string',
1869         description => "Set SMBIOS1 manufacturer.",
1870         optional => 1,
1871     },
1872     product => {
1873         type => 'string',
1874         pattern => '[A-Za-z0-9+\/]+={0,2}',
1875         format_description => 'Base64 encoded string',
1876         description => "Set SMBIOS1 product ID.",
1877         optional => 1,
1878     },
1879     sku => {
1880         type => 'string',
1881         pattern => '[A-Za-z0-9+\/]+={0,2}',
1882         format_description => 'Base64 encoded string',
1883         description => "Set SMBIOS1 SKU string.",
1884         optional => 1,
1885     },
1886     family => {
1887         type => 'string',
1888         pattern => '[A-Za-z0-9+\/]+={0,2}',
1889         format_description => 'Base64 encoded string',
1890         description => "Set SMBIOS1 family string.",
1891         optional => 1,
1892     },
1893     base64 => {
1894         type => 'boolean',
1895         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1896         optional => 1,
1897     },
1898 };
1899
1900 sub parse_smbios1 {
1901     my ($data) = @_;
1902
1903     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1904     warn $@ if $@;
1905     return $res;
1906 }
1907
1908 sub print_smbios1 {
1909     my ($smbios1) = @_;
1910     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1911 }
1912
1913 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1914
1915 sub parse_watchdog {
1916     my ($value) = @_;
1917
1918     return undef if !$value;
1919
1920     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1921     warn $@ if $@;
1922     return $res;
1923 }
1924
1925 sub parse_guest_agent {
1926     my ($value) = @_;
1927
1928     return {} if !defined($value->{agent});
1929
1930     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1931     warn $@ if $@;
1932
1933     # if the agent is disabled ignore the other potentially set properties
1934     return {} if !$res->{enabled};
1935     return $res;
1936 }
1937
1938 sub parse_vga {
1939     my ($value) = @_;
1940
1941     return {} if !$value;
1942     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1943     warn $@ if $@;
1944     return $res;
1945 }
1946
1947 sub parse_rng {
1948     my ($value) = @_;
1949
1950     return undef if !$value;
1951
1952     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1953     warn $@ if $@;
1954     return $res;
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1958 sub verify_usb_device {
1959     my ($value, $noerr) = @_;
1960
1961     return $value if parse_usb_device($value);
1962
1963     return undef if $noerr;
1964
1965     die "unable to parse usb device\n";
1966 }
1967
1968 # add JSON properties for create and set function
1969 sub json_config_properties {
1970     my $prop = shift;
1971
1972     foreach my $opt (keys %$confdesc) {
1973         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1974         $prop->{$opt} = $confdesc->{$opt};
1975     }
1976
1977     return $prop;
1978 }
1979
1980 # return copy of $confdesc_cloudinit to generate documentation
1981 sub cloudinit_config_properties {
1982
1983     return dclone($confdesc_cloudinit);
1984 }
1985
1986 sub check_type {
1987     my ($key, $value) = @_;
1988
1989     die "unknown setting '$key'\n" if !$confdesc->{$key};
1990
1991     my $type = $confdesc->{$key}->{type};
1992
1993     if (!defined($value)) {
1994         die "got undefined value\n";
1995     }
1996
1997     if ($value =~ m/[\n\r]/) {
1998         die "property contains a line feed\n";
1999     }
2000
2001     if ($type eq 'boolean') {
2002         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2003         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2004         die "type check ('boolean') failed - got '$value'\n";
2005     } elsif ($type eq 'integer') {
2006         return int($1) if $value =~ m/^(\d+)$/;
2007         die "type check ('integer') failed - got '$value'\n";
2008     } elsif ($type eq 'number') {
2009         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2010         die "type check ('number') failed - got '$value'\n";
2011     } elsif ($type eq 'string') {
2012         if (my $fmt = $confdesc->{$key}->{format}) {
2013             PVE::JSONSchema::check_format($fmt, $value);
2014             return $value;
2015         }
2016         $value =~ s/^\"(.*)\"$/$1/;
2017         return $value;
2018     } else {
2019         die "internal error"
2020     }
2021 }
2022
2023 sub destroy_vm {
2024     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2025
2026     my $conf = PVE::QemuConfig->load_config($vmid);
2027
2028     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2029
2030     if ($conf->{template}) {
2031         # check if any base image is still used by a linked clone
2032         foreach_drive($conf, sub {
2033                 my ($ds, $drive) = @_;
2034                 return if drive_is_cdrom($drive);
2035
2036                 my $volid = $drive->{file};
2037                 return if !$volid || $volid =~ m|^/|;
2038
2039                 die "base volume '$volid' is still in use by linked cloned\n"
2040                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2041
2042         });
2043     }
2044
2045     # only remove disks owned by this VM
2046     foreach_drive($conf, sub {
2047         my ($ds, $drive) = @_;
2048         return if drive_is_cdrom($drive, 1);
2049
2050         my $volid = $drive->{file};
2051         return if !$volid || $volid =~ m|^/|;
2052
2053         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2054         return if !$path || !$owner || ($owner != $vmid);
2055
2056         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2057         warn "Could not remove disk '$volid', check manually: $@" if $@;
2058     });
2059
2060     # also remove unused disk
2061     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2062     PVE::Storage::foreach_volid($vmdisks, sub {
2063         my ($volid, $sid, $volname, $d) = @_;
2064         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2065         warn $@ if $@;
2066     });
2067
2068     if (defined $replacement_conf) {
2069         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2070     } else {
2071         PVE::QemuConfig->destroy_config($vmid);
2072     }
2073 }
2074
2075 sub parse_vm_config {
2076     my ($filename, $raw) = @_;
2077
2078     return undef if !defined($raw);
2079
2080     my $res = {
2081         digest => Digest::SHA::sha1_hex($raw),
2082         snapshots => {},
2083         pending => {},
2084     };
2085
2086     $filename =~ m|/qemu-server/(\d+)\.conf$|
2087         || die "got strange filename '$filename'";
2088
2089     my $vmid = $1;
2090
2091     my $conf = $res;
2092     my $descr;
2093     my $section = '';
2094
2095     my @lines = split(/\n/, $raw);
2096     foreach my $line (@lines) {
2097         next if $line =~ m/^\s*$/;
2098
2099         if ($line =~ m/^\[PENDING\]\s*$/i) {
2100             $section = 'pending';
2101             if (defined($descr)) {
2102                 $descr =~ s/\s+$//;
2103                 $conf->{description} = $descr;
2104             }
2105             $descr = undef;
2106             $conf = $res->{$section} = {};
2107             next;
2108
2109         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2110             $section = $1;
2111             if (defined($descr)) {
2112                 $descr =~ s/\s+$//;
2113                 $conf->{description} = $descr;
2114             }
2115             $descr = undef;
2116             $conf = $res->{snapshots}->{$section} = {};
2117             next;
2118         }
2119
2120         if ($line =~ m/^\#(.*)\s*$/) {
2121             $descr = '' if !defined($descr);
2122             $descr .= PVE::Tools::decode_text($1) . "\n";
2123             next;
2124         }
2125
2126         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2127             $descr = '' if !defined($descr);
2128             $descr .= PVE::Tools::decode_text($2);
2129         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2130             $conf->{snapstate} = $1;
2131         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2132             my $key = $1;
2133             my $value = $2;
2134             $conf->{$key} = $value;
2135         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2136             my $value = $1;
2137             if ($section eq 'pending') {
2138                 $conf->{delete} = $value; # we parse this later
2139             } else {
2140                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2141             }
2142         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2143             my $key = $1;
2144             my $value = $2;
2145             eval { $value = check_type($key, $value); };
2146             if ($@) {
2147                 warn "vm $vmid - unable to parse value of '$key' - $@";
2148             } else {
2149                 $key = 'ide2' if $key eq 'cdrom';
2150                 my $fmt = $confdesc->{$key}->{format};
2151                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2152                     my $v = parse_drive($key, $value);
2153                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2154                         $v->{file} = $volid;
2155                         $value = print_drive($v);
2156                     } else {
2157                         warn "vm $vmid - unable to parse value of '$key'\n";
2158                         next;
2159                     }
2160                 }
2161
2162                 $conf->{$key} = $value;
2163             }
2164         }
2165     }
2166
2167     if (defined($descr)) {
2168         $descr =~ s/\s+$//;
2169         $conf->{description} = $descr;
2170     }
2171     delete $res->{snapstate}; # just to be sure
2172
2173     return $res;
2174 }
2175
2176 sub write_vm_config {
2177     my ($filename, $conf) = @_;
2178
2179     delete $conf->{snapstate}; # just to be sure
2180
2181     if ($conf->{cdrom}) {
2182         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2183         $conf->{ide2} = $conf->{cdrom};
2184         delete $conf->{cdrom};
2185     }
2186
2187     # we do not use 'smp' any longer
2188     if ($conf->{sockets}) {
2189         delete $conf->{smp};
2190     } elsif ($conf->{smp}) {
2191         $conf->{sockets} = $conf->{smp};
2192         delete $conf->{cores};
2193         delete $conf->{smp};
2194     }
2195
2196     my $used_volids = {};
2197
2198     my $cleanup_config = sub {
2199         my ($cref, $pending, $snapname) = @_;
2200
2201         foreach my $key (keys %$cref) {
2202             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2203                 $key eq 'snapstate' || $key eq 'pending';
2204             my $value = $cref->{$key};
2205             if ($key eq 'delete') {
2206                 die "propertry 'delete' is only allowed in [PENDING]\n"
2207                     if !$pending;
2208                 # fixme: check syntax?
2209                 next;
2210             }
2211             eval { $value = check_type($key, $value); };
2212             die "unable to parse value of '$key' - $@" if $@;
2213
2214             $cref->{$key} = $value;
2215
2216             if (!$snapname && is_valid_drivename($key)) {
2217                 my $drive = parse_drive($key, $value);
2218                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2219             }
2220         }
2221     };
2222
2223     &$cleanup_config($conf);
2224
2225     &$cleanup_config($conf->{pending}, 1);
2226
2227     foreach my $snapname (keys %{$conf->{snapshots}}) {
2228         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2229         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2230     }
2231
2232     # remove 'unusedX' settings if we re-add a volume
2233     foreach my $key (keys %$conf) {
2234         my $value = $conf->{$key};
2235         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2236             delete $conf->{$key};
2237         }
2238     }
2239
2240     my $generate_raw_config = sub {
2241         my ($conf, $pending) = @_;
2242
2243         my $raw = '';
2244
2245         # add description as comment to top of file
2246         if (defined(my $descr = $conf->{description})) {
2247             if ($descr) {
2248                 foreach my $cl (split(/\n/, $descr)) {
2249                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2250                 }
2251             } else {
2252                 $raw .= "#\n" if $pending;
2253             }
2254         }
2255
2256         foreach my $key (sort keys %$conf) {
2257             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2258             $raw .= "$key: $conf->{$key}\n";
2259         }
2260         return $raw;
2261     };
2262
2263     my $raw = &$generate_raw_config($conf);
2264
2265     if (scalar(keys %{$conf->{pending}})){
2266         $raw .= "\n[PENDING]\n";
2267         $raw .= &$generate_raw_config($conf->{pending}, 1);
2268     }
2269
2270     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2271         $raw .= "\n[$snapname]\n";
2272         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2273     }
2274
2275     return $raw;
2276 }
2277
2278 sub load_defaults {
2279
2280     my $res = {};
2281
2282     # we use static defaults from our JSON schema configuration
2283     foreach my $key (keys %$confdesc) {
2284         if (defined(my $default = $confdesc->{$key}->{default})) {
2285             $res->{$key} = $default;
2286         }
2287     }
2288
2289     return $res;
2290 }
2291
2292 sub config_list {
2293     my $vmlist = PVE::Cluster::get_vmlist();
2294     my $res = {};
2295     return $res if !$vmlist || !$vmlist->{ids};
2296     my $ids = $vmlist->{ids};
2297     my $nodename = nodename();
2298
2299     foreach my $vmid (keys %$ids) {
2300         my $d = $ids->{$vmid};
2301         next if !$d->{node} || $d->{node} ne $nodename;
2302         next if !$d->{type} || $d->{type} ne 'qemu';
2303         $res->{$vmid}->{exists} = 1;
2304     }
2305     return $res;
2306 }
2307
2308 # test if VM uses local resources (to prevent migration)
2309 sub check_local_resources {
2310     my ($conf, $noerr) = @_;
2311
2312     my @loc_res = ();
2313
2314     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2315     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2316
2317     push @loc_res, "ivshmem" if $conf->{ivshmem};
2318
2319     foreach my $k (keys %$conf) {
2320         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2321         # sockets are safe: they will recreated be on the target side post-migrate
2322         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2323         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2324     }
2325
2326     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2327
2328     return \@loc_res;
2329 }
2330
2331 # check if used storages are available on all nodes (use by migrate)
2332 sub check_storage_availability {
2333     my ($storecfg, $conf, $node) = @_;
2334
2335     foreach_drive($conf, sub {
2336         my ($ds, $drive) = @_;
2337
2338         my $volid = $drive->{file};
2339         return if !$volid;
2340
2341         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2342         return if !$sid;
2343
2344         # check if storage is available on both nodes
2345         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2346         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2347    });
2348 }
2349
2350 # list nodes where all VM images are available (used by has_feature API)
2351 sub shared_nodes {
2352     my ($conf, $storecfg) = @_;
2353
2354     my $nodelist = PVE::Cluster::get_nodelist();
2355     my $nodehash = { map { $_ => 1 } @$nodelist };
2356     my $nodename = nodename();
2357
2358     foreach_drive($conf, sub {
2359         my ($ds, $drive) = @_;
2360
2361         my $volid = $drive->{file};
2362         return if !$volid;
2363
2364         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2365         if ($storeid) {
2366             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2367             if ($scfg->{disable}) {
2368                 $nodehash = {};
2369             } elsif (my $avail = $scfg->{nodes}) {
2370                 foreach my $node (keys %$nodehash) {
2371                     delete $nodehash->{$node} if !$avail->{$node};
2372                 }
2373             } elsif (!$scfg->{shared}) {
2374                 foreach my $node (keys %$nodehash) {
2375                     delete $nodehash->{$node} if $node ne $nodename
2376                 }
2377             }
2378         }
2379     });
2380
2381     return $nodehash
2382 }
2383
2384 sub check_local_storage_availability {
2385     my ($conf, $storecfg) = @_;
2386
2387     my $nodelist = PVE::Cluster::get_nodelist();
2388     my $nodehash = { map { $_ => {} } @$nodelist };
2389
2390     foreach_drive($conf, sub {
2391         my ($ds, $drive) = @_;
2392
2393         my $volid = $drive->{file};
2394         return if !$volid;
2395
2396         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2397         if ($storeid) {
2398             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2399
2400             if ($scfg->{disable}) {
2401                 foreach my $node (keys %$nodehash) {
2402                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2403                 }
2404             } elsif (my $avail = $scfg->{nodes}) {
2405                 foreach my $node (keys %$nodehash) {
2406                     if (!$avail->{$node}) {
2407                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2408                     }
2409                 }
2410             }
2411         }
2412     });
2413
2414     foreach my $node (values %$nodehash) {
2415         if (my $unavail = $node->{unavailable_storages}) {
2416             $node->{unavailable_storages} = [ sort keys %$unavail ];
2417         }
2418     }
2419
2420     return $nodehash
2421 }
2422
2423 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2424 sub check_running {
2425     my ($vmid, $nocheck, $node) = @_;
2426
2427     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2428     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2429 }
2430
2431 sub vzlist {
2432
2433     my $vzlist = config_list();
2434
2435     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2436
2437     while (defined(my $de = $fd->read)) {
2438         next if $de !~ m/^(\d+)\.pid$/;
2439         my $vmid = $1;
2440         next if !defined($vzlist->{$vmid});
2441         if (my $pid = check_running($vmid)) {
2442             $vzlist->{$vmid}->{pid} = $pid;
2443         }
2444     }
2445
2446     return $vzlist;
2447 }
2448
2449 our $vmstatus_return_properties = {
2450     vmid => get_standard_option('pve-vmid'),
2451     status => {
2452         description => "Qemu process status.",
2453         type => 'string',
2454         enum => ['stopped', 'running'],
2455     },
2456     maxmem => {
2457         description => "Maximum memory in bytes.",
2458         type => 'integer',
2459         optional => 1,
2460         renderer => 'bytes',
2461     },
2462     maxdisk => {
2463         description => "Root disk size in bytes.",
2464         type => 'integer',
2465         optional => 1,
2466         renderer => 'bytes',
2467     },
2468     name => {
2469         description => "VM name.",
2470         type => 'string',
2471         optional => 1,
2472     },
2473     qmpstatus => {
2474         description => "Qemu QMP agent status.",
2475         type => 'string',
2476         optional => 1,
2477     },
2478     pid => {
2479         description => "PID of running qemu process.",
2480         type => 'integer',
2481         optional => 1,
2482     },
2483     uptime => {
2484         description => "Uptime.",
2485         type => 'integer',
2486         optional => 1,
2487         renderer => 'duration',
2488     },
2489     cpus => {
2490         description => "Maximum usable CPUs.",
2491         type => 'number',
2492         optional => 1,
2493     },
2494     lock => {
2495         description => "The current config lock, if any.",
2496         type => 'string',
2497         optional => 1,
2498     },
2499     tags => {
2500         description  => "The current configured tags, if any",
2501         type => 'string',
2502         optional => 1,
2503     },
2504 };
2505
2506 my $last_proc_pid_stat;
2507
2508 # get VM status information
2509 # This must be fast and should not block ($full == false)
2510 # We only query KVM using QMP if $full == true (this can be slow)
2511 sub vmstatus {
2512     my ($opt_vmid, $full) = @_;
2513
2514     my $res = {};
2515
2516     my $storecfg = PVE::Storage::config();
2517
2518     my $list = vzlist();
2519     my $defaults = load_defaults();
2520
2521     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2522
2523     my $cpucount = $cpuinfo->{cpus} || 1;
2524
2525     foreach my $vmid (keys %$list) {
2526         next if $opt_vmid && ($vmid ne $opt_vmid);
2527
2528         my $conf = PVE::QemuConfig->load_config($vmid);
2529
2530         my $d = { vmid => $vmid };
2531         $d->{pid} = $list->{$vmid}->{pid};
2532
2533         # fixme: better status?
2534         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2535
2536         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2537         if (defined($size)) {
2538             $d->{disk} = 0; # no info available
2539             $d->{maxdisk} = $size;
2540         } else {
2541             $d->{disk} = 0;
2542             $d->{maxdisk} = 0;
2543         }
2544
2545         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2546             * ($conf->{cores} || $defaults->{cores});
2547         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2548         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2549
2550         $d->{name} = $conf->{name} || "VM $vmid";
2551         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2552             : $defaults->{memory}*(1024*1024);
2553
2554         if ($conf->{balloon}) {
2555             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2556             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2557                 : $defaults->{shares};
2558         }
2559
2560         $d->{uptime} = 0;
2561         $d->{cpu} = 0;
2562         $d->{mem} = 0;
2563
2564         $d->{netout} = 0;
2565         $d->{netin} = 0;
2566
2567         $d->{diskread} = 0;
2568         $d->{diskwrite} = 0;
2569
2570         $d->{template} = PVE::QemuConfig->is_template($conf);
2571
2572         $d->{serial} = 1 if conf_has_serial($conf);
2573         $d->{lock} = $conf->{lock} if $conf->{lock};
2574         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2575
2576         $res->{$vmid} = $d;
2577     }
2578
2579     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2580     foreach my $dev (keys %$netdev) {
2581         next if $dev !~ m/^tap([1-9]\d*)i/;
2582         my $vmid = $1;
2583         my $d = $res->{$vmid};
2584         next if !$d;
2585
2586         $d->{netout} += $netdev->{$dev}->{receive};
2587         $d->{netin} += $netdev->{$dev}->{transmit};
2588
2589         if ($full) {
2590             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2591             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2592         }
2593
2594     }
2595
2596     my $ctime = gettimeofday;
2597
2598     foreach my $vmid (keys %$list) {
2599
2600         my $d = $res->{$vmid};
2601         my $pid = $d->{pid};
2602         next if !$pid;
2603
2604         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2605         next if !$pstat; # not running
2606
2607         my $used = $pstat->{utime} + $pstat->{stime};
2608
2609         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2610
2611         if ($pstat->{vsize}) {
2612             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2613         }
2614
2615         my $old = $last_proc_pid_stat->{$pid};
2616         if (!$old) {
2617             $last_proc_pid_stat->{$pid} = {
2618                 time => $ctime,
2619                 used => $used,
2620                 cpu => 0,
2621             };
2622             next;
2623         }
2624
2625         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2626
2627         if ($dtime > 1000) {
2628             my $dutime = $used -  $old->{used};
2629
2630             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2631             $last_proc_pid_stat->{$pid} = {
2632                 time => $ctime,
2633                 used => $used,
2634                 cpu => $d->{cpu},
2635             };
2636         } else {
2637             $d->{cpu} = $old->{cpu};
2638         }
2639     }
2640
2641     return $res if !$full;
2642
2643     my $qmpclient = PVE::QMPClient->new();
2644
2645     my $ballooncb = sub {
2646         my ($vmid, $resp) = @_;
2647
2648         my $info = $resp->{'return'};
2649         return if !$info->{max_mem};
2650
2651         my $d = $res->{$vmid};
2652
2653         # use memory assigned to VM
2654         $d->{maxmem} = $info->{max_mem};
2655         $d->{balloon} = $info->{actual};
2656
2657         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2658             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2659             $d->{freemem} = $info->{free_mem};
2660         }
2661
2662         $d->{ballooninfo} = $info;
2663     };
2664
2665     my $blockstatscb = sub {
2666         my ($vmid, $resp) = @_;
2667         my $data = $resp->{'return'} || [];
2668         my $totalrdbytes = 0;
2669         my $totalwrbytes = 0;
2670
2671         for my $blockstat (@$data) {
2672             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2673             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2674
2675             $blockstat->{device} =~ s/drive-//;
2676             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2677         }
2678         $res->{$vmid}->{diskread} = $totalrdbytes;
2679         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2680     };
2681
2682     my $statuscb = sub {
2683         my ($vmid, $resp) = @_;
2684
2685         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2686         # this fails if ballon driver is not loaded, so this must be
2687         # the last commnand (following command are aborted if this fails).
2688         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2689
2690         my $status = 'unknown';
2691         if (!defined($status = $resp->{'return'}->{status})) {
2692             warn "unable to get VM status\n";
2693             return;
2694         }
2695
2696         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2697     };
2698
2699     foreach my $vmid (keys %$list) {
2700         next if $opt_vmid && ($vmid ne $opt_vmid);
2701         next if !$res->{$vmid}->{pid}; # not running
2702         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2703     }
2704
2705     $qmpclient->queue_execute(undef, 2);
2706
2707     foreach my $vmid (keys %$list) {
2708         next if $opt_vmid && ($vmid ne $opt_vmid);
2709         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2710     }
2711
2712     return $res;
2713 }
2714
2715 sub conf_has_serial {
2716     my ($conf) = @_;
2717
2718     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2719         if ($conf->{"serial$i"}) {
2720             return 1;
2721         }
2722     }
2723
2724     return 0;
2725 }
2726
2727 sub conf_has_audio {
2728     my ($conf, $id) = @_;
2729
2730     $id //= 0;
2731     my $audio = $conf->{"audio$id"};
2732     return undef if !defined($audio);
2733
2734     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2735     my $audiodriver = $audioproperties->{driver} // 'spice';
2736
2737     return {
2738         dev => $audioproperties->{device},
2739         dev_id => "audiodev$id",
2740         backend => $audiodriver,
2741         backend_id => "$audiodriver-backend${id}",
2742     };
2743 }
2744
2745 sub vga_conf_has_spice {
2746     my ($vga) = @_;
2747
2748     my $vgaconf = parse_vga($vga);
2749     my $vgatype = $vgaconf->{type};
2750     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2751
2752     return $1 || 1;
2753 }
2754
2755 sub is_native($) {
2756     my ($arch) = @_;
2757     return get_host_arch() eq $arch;
2758 }
2759
2760 sub get_vm_arch {
2761     my ($conf) = @_;
2762     return $conf->{arch} // get_host_arch();
2763 }
2764
2765 my $default_machines = {
2766     x86_64 => 'pc',
2767     aarch64 => 'virt',
2768 };
2769
2770 sub get_vm_machine {
2771     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2772
2773     my $machine = $forcemachine || $conf->{machine};
2774
2775     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2776         $arch //= 'x86_64';
2777         $machine ||= $default_machines->{$arch};
2778         if ($add_pve_version) {
2779             $kvmversion //= kvm_user_version();
2780             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2781             $machine .= "+pve$pvever";
2782         }
2783     }
2784
2785     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2786         # for version-pinned machines that do not include a pve-version (e.g.
2787         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2788         $machine .= '+pve0';
2789     }
2790
2791     return $machine;
2792 }
2793
2794 sub get_ovmf_files($) {
2795     my ($arch) = @_;
2796
2797     my $ovmf = $OVMF->{$arch}
2798         or die "no OVMF images known for architecture '$arch'\n";
2799
2800     return @$ovmf;
2801 }
2802
2803 my $Arch2Qemu = {
2804     aarch64 => '/usr/bin/qemu-system-aarch64',
2805     x86_64 => '/usr/bin/qemu-system-x86_64',
2806 };
2807 sub get_command_for_arch($) {
2808     my ($arch) = @_;
2809     return '/usr/bin/kvm' if is_native($arch);
2810
2811     my $cmd = $Arch2Qemu->{$arch}
2812         or die "don't know how to emulate architecture '$arch'\n";
2813     return $cmd;
2814 }
2815
2816 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2817 # to use in a QEMU command line (-cpu element), first array_intersect the result
2818 # of query_supported_ with query_understood_. This is necessary because:
2819 #
2820 # a) query_understood_ returns flags the host cannot use and
2821 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2822 #    flags, but CPU settings - with most of them being flags. Those settings
2823 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2824 #
2825 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2826 # expensive. If you need the value returned from this, you can get it much
2827 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2828 # $accel being 'kvm' or 'tcg'.
2829 #
2830 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2831 # changes, automatically populating pmxcfs.
2832 #
2833 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2834 # since kvm and tcg machines support different flags
2835 #
2836 sub query_supported_cpu_flags {
2837     my ($arch) = @_;
2838
2839     $arch //= get_host_arch();
2840     my $default_machine = $default_machines->{$arch};
2841
2842     my $flags = {};
2843
2844     # FIXME: Once this is merged, the code below should work for ARM as well:
2845     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2846     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2847         $arch eq "aarch64";
2848
2849     my $kvm_supported = defined(kvm_version());
2850     my $qemu_cmd = get_command_for_arch($arch);
2851     my $fakevmid = -1;
2852     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2853
2854     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2855     my $query_supported_run_qemu = sub {
2856         my ($kvm) = @_;
2857
2858         my $flags = {};
2859         my $cmd = [
2860             $qemu_cmd,
2861             '-machine', $default_machine,
2862             '-display', 'none',
2863             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2864             '-mon', 'chardev=qmp,mode=control',
2865             '-pidfile', $pidfile,
2866             '-S', '-daemonize'
2867         ];
2868
2869         if (!$kvm) {
2870             push @$cmd, '-accel', 'tcg';
2871         }
2872
2873         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2874         die "QEMU flag querying VM exited with code " . $rc if $rc;
2875
2876         eval {
2877             my $cmd_result = mon_cmd(
2878                 $fakevmid,
2879                 'query-cpu-model-expansion',
2880                 type => 'full',
2881                 model => { name => 'host' }
2882             );
2883
2884             my $props = $cmd_result->{model}->{props};
2885             foreach my $prop (keys %$props) {
2886                 next if $props->{$prop} ne '1';
2887                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2888                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2889                 # We only keep those with underscores, to match /proc/cpuinfo
2890                 $prop =~ s/\.|-/_/g;
2891                 $flags->{$prop} = 1;
2892             }
2893         };
2894         my $err = $@;
2895
2896         # force stop with 10 sec timeout and 'nocheck'
2897         # always stop, even if QMP failed
2898         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2899
2900         die $err if $err;
2901
2902         return [ sort keys %$flags ];
2903     };
2904
2905     # We need to query QEMU twice, since KVM and TCG have different supported flags
2906     PVE::QemuConfig->lock_config($fakevmid, sub {
2907         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2908         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2909
2910         if ($kvm_supported) {
2911             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2912             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2913         }
2914     });
2915
2916     return $flags;
2917 }
2918
2919 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2920 my $understood_cpu_flag_dir = "/usr/share/kvm";
2921 sub query_understood_cpu_flags {
2922     my $arch = get_host_arch();
2923     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2924
2925     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2926         if ! -e $filepath;
2927
2928     my $raw = file_get_contents($filepath);
2929     $raw =~ s/^\s+|\s+$//g;
2930     my @flags = split(/\s+/, $raw);
2931
2932     return \@flags;
2933 }
2934
2935 sub config_to_command {
2936     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2937
2938     my $cmd = [];
2939     my $globalFlags = [];
2940     my $machineFlags = [];
2941     my $rtcFlags = [];
2942     my $devices = [];
2943     my $pciaddr = '';
2944     my $bridges = {};
2945     my $ostype = $conf->{ostype};
2946     my $winversion = windows_version($ostype);
2947     my $kvm = $conf->{kvm};
2948     my $nodename = nodename();
2949
2950     my $arch = get_vm_arch($conf);
2951     my $kvm_binary = get_command_for_arch($arch);
2952     my $kvmver = kvm_user_version($kvm_binary);
2953
2954     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2955         $kvmver //= "undefined";
2956         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2957     }
2958
2959     my $add_pve_version = min_version($kvmver, 4, 1);
2960
2961     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2962     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2963     $kvm //= 1 if is_native($arch);
2964
2965     $machine_version =~ m/(\d+)\.(\d+)/;
2966     my ($machine_major, $machine_minor) = ($1, $2);
2967     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2968         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2969
2970     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2971         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2972         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2973           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2974     }
2975
2976     # if a specific +pve version is required for a feature, use $version_guard
2977     # instead of min_version to allow machines to be run with the minimum
2978     # required version
2979     my $required_pve_version = 0;
2980     my $version_guard = sub {
2981         my ($major, $minor, $pve) = @_;
2982         return 0 if !min_version($machine_version, $major, $minor, $pve);
2983         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2984         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2985         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2986         return 1;
2987     };
2988
2989     if ($kvm) {
2990         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2991             if !defined kvm_version();
2992     }
2993
2994     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2995     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2996     my $use_old_bios_files = undef;
2997     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2998
2999     my $cpuunits = defined($conf->{cpuunits}) ?
3000             $conf->{cpuunits} : $defaults->{cpuunits};
3001
3002     push @$cmd, $kvm_binary;
3003
3004     push @$cmd, '-id', $vmid;
3005
3006     my $vmname = $conf->{name} || "vm$vmid";
3007
3008     push @$cmd, '-name', $vmname;
3009
3010     my $use_virtio = 0;
3011
3012     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3013     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3014     push @$cmd, '-mon', "chardev=qmp,mode=control";
3015
3016     if (min_version($machine_version, 2, 12)) {
3017         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3018         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3019     }
3020
3021     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3022
3023     push @$cmd, '-daemonize';
3024
3025     if ($conf->{smbios1}) {
3026         my $smbios_conf = parse_smbios1($conf->{smbios1});
3027         if ($smbios_conf->{base64}) {
3028             # Do not pass base64 flag to qemu
3029             delete $smbios_conf->{base64};
3030             my $smbios_string = "";
3031             foreach my $key (keys %$smbios_conf) {
3032                 my $value;
3033                 if ($key eq "uuid") {
3034                     $value = $smbios_conf->{uuid}
3035                 } else {
3036                     $value = decode_base64($smbios_conf->{$key});
3037                 }
3038                 # qemu accepts any binary data, only commas need escaping by double comma
3039                 $value =~ s/,/,,/g;
3040                 $smbios_string .= "," . $key . "=" . $value if $value;
3041             }
3042             push @$cmd, '-smbios', "type=1" . $smbios_string;
3043         } else {
3044             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3045         }
3046     }
3047
3048     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3049     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3050         die "uefi base image not found\n" if ! -f $ovmf_code;
3051
3052         my $path;
3053         my $format;
3054         if (my $efidisk = $conf->{efidisk0}) {
3055             my $d = parse_drive('efidisk0', $efidisk);
3056             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3057             $format = $d->{format};
3058             if ($storeid) {
3059                 $path = PVE::Storage::path($storecfg, $d->{file});
3060                 if (!defined($format)) {
3061                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3062                     $format = qemu_img_format($scfg, $volname);
3063                 }
3064             } else {
3065                 $path = $d->{file};
3066                 die "efidisk format must be specified\n"
3067                     if !defined($format);
3068             }
3069         } else {
3070             warn "no efidisk configured! Using temporary efivars disk.\n";
3071             $path = "/tmp/$vmid-ovmf.fd";
3072             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3073             $format = 'raw';
3074         }
3075
3076         my $size_str = "";
3077
3078         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3079             $size_str = ",size=" . (-s $ovmf_vars);
3080         }
3081
3082         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3083         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3084     }
3085
3086     # load q35 config
3087     if ($q35) {
3088         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3089         if (min_version($machine_version, 4, 0)) {
3090             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3091         } else {
3092             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3093         }
3094     }
3095
3096     if ($conf->{vmgenid}) {
3097         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3098     }
3099
3100     # add usb controllers
3101     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3102     push @$devices, @usbcontrollers if @usbcontrollers;
3103     my $vga = parse_vga($conf->{vga});
3104
3105     my $qxlnum = vga_conf_has_spice($conf->{vga});
3106     $vga->{type} = 'qxl' if $qxlnum;
3107
3108     if (!$vga->{type}) {
3109         if ($arch eq 'aarch64') {
3110             $vga->{type} = 'virtio';
3111         } elsif (min_version($machine_version, 2, 9)) {
3112             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3113         } else {
3114             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3115         }
3116     }
3117
3118     # enable absolute mouse coordinates (needed by vnc)
3119     my $tablet;
3120     if (defined($conf->{tablet})) {
3121         $tablet = $conf->{tablet};
3122     } else {
3123         $tablet = $defaults->{tablet};
3124         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3125         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3126     }
3127
3128     if ($tablet) {
3129         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3130         my $kbd = print_keyboarddevice_full($conf, $arch);
3131         push @$devices, '-device', $kbd if defined($kbd);
3132     }
3133
3134     my $kvm_off = 0;
3135     my $gpu_passthrough;
3136
3137     # host pci devices
3138     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3139         my $id = "hostpci$i";
3140         my $d = parse_hostpci($conf->{$id});
3141         next if !$d;
3142
3143         if (my $pcie = $d->{pcie}) {
3144             die "q35 machine model is not enabled" if !$q35;
3145             # win7 wants to have the pcie devices directly on the pcie bus
3146             # instead of in the root port
3147             if ($winversion == 7) {
3148                 $pciaddr = print_pcie_addr("${id}bus0");
3149             } else {
3150                 # add more root ports if needed, 4 are present by default
3151                 # by pve-q35 cfgs, rest added here on demand.
3152                 if ($i > 3) {
3153                     push @$devices, '-device', print_pcie_root_port($i);
3154                 }
3155                 $pciaddr = print_pcie_addr($id);
3156             }
3157         } else {
3158             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3159         }
3160
3161         my $xvga = '';
3162         if ($d->{'x-vga'}) {
3163             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3164             $kvm_off = 1;
3165             $vga->{type} = 'none' if !defined($conf->{vga});
3166             $gpu_passthrough = 1;
3167         }
3168
3169         my $pcidevices = $d->{pciid};
3170         my $multifunction = 1 if @$pcidevices > 1;
3171
3172         my $sysfspath;
3173         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3174             my $pci_id = $pcidevices->[0]->{id};
3175             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3176             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3177         } elsif ($d->{mdev}) {
3178             warn "ignoring mediated device '$id' with multifunction device\n";
3179         }
3180
3181         my $j=0;
3182         foreach my $pcidevice (@$pcidevices) {
3183             my $devicestr = "vfio-pci";
3184
3185             if ($sysfspath) {
3186                 $devicestr .= ",sysfsdev=$sysfspath";
3187             } else {
3188                 $devicestr .= ",host=$pcidevice->{id}";
3189             }
3190
3191             my $mf_addr = $multifunction ? ".$j" : '';
3192             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3193
3194             if ($j == 0) {
3195                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3196                 $devicestr .= "$xvga";
3197                 $devicestr .= ",multifunction=on" if $multifunction;
3198                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3199             }
3200
3201             push @$devices, '-device', $devicestr;
3202             $j++;
3203         }
3204     }
3205
3206     # usb devices
3207     my $usb_dev_features = {};
3208     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3209
3210     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3211     push @$devices, @usbdevices if @usbdevices;
3212     # serial devices
3213     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3214         if (my $path = $conf->{"serial$i"}) {
3215             if ($path eq 'socket') {
3216                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3217                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3218                 # On aarch64, serial0 is the UART device. Qemu only allows
3219                 # connecting UART devices via the '-serial' command line, as
3220                 # the device has a fixed slot on the hardware...
3221                 if ($arch eq 'aarch64' && $i == 0) {
3222                     push @$devices, '-serial', "chardev:serial$i";
3223                 } else {
3224                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3225                 }
3226             } else {
3227                 die "no such serial device\n" if ! -c $path;
3228                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3229                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3230             }
3231         }
3232     }
3233
3234     # parallel devices
3235     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3236         if (my $path = $conf->{"parallel$i"}) {
3237             die "no such parallel device\n" if ! -c $path;
3238             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3239             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3240             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3241         }
3242     }
3243
3244     if (my $audio = conf_has_audio($conf)) {
3245
3246         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3247
3248         my $id = $audio->{dev_id};
3249         if ($audio->{dev} eq 'AC97') {
3250             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3251         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3252             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3253             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3254             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3255         } else {
3256             die "unkown audio device '$audio->{dev}', implement me!";
3257         }
3258
3259         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3260     }
3261
3262     my $sockets = 1;
3263     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3264     $sockets = $conf->{sockets} if  $conf->{sockets};
3265
3266     my $cores = $conf->{cores} || 1;
3267
3268     my $maxcpus = $sockets * $cores;
3269
3270     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3271
3272     my $allowed_vcpus = $cpuinfo->{cpus};
3273
3274     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3275         if ($allowed_vcpus < $maxcpus);
3276
3277     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3278
3279         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3280         for (my $i = 2; $i <= $vcpus; $i++)  {
3281             my $cpustr = print_cpu_device($conf,$i);
3282             push @$cmd, '-device', $cpustr;
3283         }
3284
3285     } else {
3286
3287         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3288     }
3289     push @$cmd, '-nodefaults';
3290
3291     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3292
3293     my $bootindex_hash = {};
3294     my $i = 1;
3295     foreach my $o (split(//, $bootorder)) {
3296         $bootindex_hash->{$o} = $i*100;
3297         $i++;
3298     }
3299
3300     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3301
3302     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3303
3304     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3305
3306     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3307         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3308         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3309         push @$cmd,  '-vnc', "unix:$socket,password";
3310     } else {
3311         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3312         push @$cmd, '-nographic';
3313     }
3314
3315     # time drift fix
3316     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3317
3318     my $useLocaltime = $conf->{localtime};
3319
3320     if ($winversion >= 5) { # windows
3321         $useLocaltime = 1 if !defined($conf->{localtime});
3322
3323         # use time drift fix when acpi is enabled
3324         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3325             $tdf = 1 if !defined($conf->{tdf});
3326         }
3327     }
3328
3329     if ($winversion >= 6) {
3330         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3331         push @$cmd, '-no-hpet';
3332     }
3333
3334     push @$rtcFlags, 'driftfix=slew' if $tdf;
3335
3336     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3337         push @$rtcFlags, "base=$conf->{startdate}";
3338     } elsif ($useLocaltime) {
3339         push @$rtcFlags, 'base=localtime';
3340     }
3341
3342     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3343
3344     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3345
3346     push @$cmd, '-S' if $conf->{freeze};
3347
3348     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3349
3350     my $guest_agent = parse_guest_agent($conf);
3351
3352     if ($guest_agent->{enabled}) {
3353         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3354         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3355
3356         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3357             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3358             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3359             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3360         } elsif ($guest_agent->{type} eq 'isa') {
3361             push @$devices, '-device', "isa-serial,chardev=qga0";
3362         }
3363     }
3364
3365     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3366     if ($rng && &$version_guard(4, 1, 2)) {
3367         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3368         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3369
3370         my $limiter_str = "";
3371         if ($max_bytes) {
3372             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3373         }
3374
3375         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3376         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3377             if ! -e $rng->{source};
3378
3379         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3380
3381         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3382         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3383     }
3384
3385     my $spice_port;
3386
3387     if ($qxlnum) {
3388         if ($qxlnum > 1) {
3389             if ($winversion){
3390                 for(my $i = 1; $i < $qxlnum; $i++){
3391                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3392                 }
3393             } else {
3394                 # assume other OS works like Linux
3395                 my ($ram, $vram) = ("134217728", "67108864");
3396                 if ($vga->{memory}) {
3397                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3398                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3399                 }
3400                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3401                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3402             }
3403         }
3404
3405         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3406
3407         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3408         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3409         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3410
3411         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3412         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3413         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3414
3415         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3416         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3417
3418         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3419         if ($spice_enhancement->{foldersharing}) {
3420             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3421             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3422         }
3423
3424         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3425         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3426         push @$devices, '-spice', "$spice_opts";
3427     }
3428
3429     # enable balloon by default, unless explicitly disabled
3430     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3431         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3432         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3433     }
3434
3435     if ($conf->{watchdog}) {
3436         my $wdopts = parse_watchdog($conf->{watchdog});
3437         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3438         my $watchdog = $wdopts->{model} || 'i6300esb';
3439         push @$devices, '-device', "$watchdog$pciaddr";
3440         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3441     }
3442
3443     my $vollist = [];
3444     my $scsicontroller = {};
3445     my $ahcicontroller = {};
3446     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3447
3448     # Add iscsi initiator name if available
3449     if (my $initiator = get_initiator_name()) {
3450         push @$devices, '-iscsi', "initiator-name=$initiator";
3451     }
3452
3453     foreach_drive($conf, sub {
3454         my ($ds, $drive) = @_;
3455
3456         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3457             push @$vollist, $drive->{file};
3458         }
3459
3460         # ignore efidisk here, already added in bios/fw handling code above
3461         return if $drive->{interface} eq 'efidisk';
3462
3463         $use_virtio = 1 if $ds =~ m/^virtio/;
3464
3465         if (drive_is_cdrom ($drive)) {
3466             if ($bootindex_hash->{d}) {
3467                 $drive->{bootindex} = $bootindex_hash->{d};
3468                 $bootindex_hash->{d} += 1;
3469             }
3470         } else {
3471             if ($bootindex_hash->{c}) {
3472                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3473                 $bootindex_hash->{c} += 1;
3474             }
3475         }
3476
3477         if($drive->{interface} eq 'virtio'){
3478            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3479         }
3480
3481         if ($drive->{interface} eq 'scsi') {
3482
3483             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3484
3485             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3486                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3487
3488             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3489             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3490
3491             my $iothread = '';
3492             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3493                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3494                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3495             } elsif ($drive->{iothread}) {
3496                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3497             }
3498
3499             my $queues = '';
3500             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3501                 $queues = ",num_queues=$drive->{queues}";
3502             }
3503
3504             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3505             $scsicontroller->{$controller}=1;
3506         }
3507
3508         if ($drive->{interface} eq 'sata') {
3509            my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3510            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3511            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3512            $ahcicontroller->{$controller}=1;
3513         }
3514
3515         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3516         push @$devices, '-drive',$drive_cmd;
3517         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3518     });
3519
3520     for (my $i = 0; $i < $MAX_NETS; $i++) {
3521          next if !$conf->{"net$i"};
3522          my $d = parse_net($conf->{"net$i"});
3523          next if !$d;
3524
3525          $use_virtio = 1 if $d->{model} eq 'virtio';
3526
3527          if ($bootindex_hash->{n}) {
3528             $d->{bootindex} = $bootindex_hash->{n};
3529             $bootindex_hash->{n} += 1;
3530          }
3531
3532          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3533          push @$devices, '-netdev', $netdevfull;
3534
3535          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3536          push @$devices, '-device', $netdevicefull;
3537     }
3538
3539     if ($conf->{ivshmem}) {
3540         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3541
3542         my $bus;
3543         if ($q35) {
3544             $bus = print_pcie_addr("ivshmem");
3545         } else {
3546             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3547         }
3548
3549         my $ivshmem_name = $ivshmem->{name} // $vmid;
3550         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3551
3552         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3553         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3554     }
3555
3556     # pci.4 is nested in pci.1
3557     $bridges->{1} = 1 if $bridges->{4};
3558
3559     if (!$q35) {
3560         # add pci bridges
3561         if (min_version($machine_version, 2, 3)) {
3562            $bridges->{1} = 1;
3563            $bridges->{2} = 1;
3564         }
3565
3566         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3567
3568     }
3569
3570     for my $k (sort {$b cmp $a} keys %$bridges) {
3571         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3572         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3573         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3574         if ($q35) {
3575             # add after -readconfig pve-q35.cfg
3576             splice @$devices, 2, 0, '-device', $devstr;
3577         } else {
3578             unshift @$devices, '-device', $devstr if $k > 0;
3579         }
3580     }
3581
3582     if (!$kvm) {
3583         push @$machineFlags, 'accel=tcg';
3584     }
3585
3586     my $machine_type_min = $machine_type;
3587     if ($add_pve_version) {
3588         $machine_type_min =~ s/\+pve\d+$//;
3589         $machine_type_min .= "+pve$required_pve_version";
3590     }
3591     push @$machineFlags, "type=${machine_type_min}";
3592
3593     push @$cmd, @$devices;
3594     push @$cmd, '-rtc', join(',', @$rtcFlags)
3595         if scalar(@$rtcFlags);
3596     push @$cmd, '-machine', join(',', @$machineFlags)
3597         if scalar(@$machineFlags);
3598     push @$cmd, '-global', join(',', @$globalFlags)
3599         if scalar(@$globalFlags);
3600
3601     if (my $vmstate = $conf->{vmstate}) {
3602         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3603         push @$vollist, $vmstate;
3604         push @$cmd, '-loadstate', $statepath;
3605         print "activating and using '$vmstate' as vmstate\n";
3606     }
3607
3608     # add custom args
3609     if ($conf->{args}) {
3610         my $aa = PVE::Tools::split_args($conf->{args});
3611         push @$cmd, @$aa;
3612     }
3613
3614     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3615 }
3616
3617 sub spice_port {
3618     my ($vmid) = @_;
3619
3620     my $res = mon_cmd($vmid, 'query-spice');
3621
3622     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3623 }
3624
3625 sub vm_devices_list {
3626     my ($vmid) = @_;
3627
3628     my $res = mon_cmd($vmid, 'query-pci');
3629     my $devices_to_check = [];
3630     my $devices = {};
3631     foreach my $pcibus (@$res) {
3632         push @$devices_to_check, @{$pcibus->{devices}},
3633     }
3634
3635     while (@$devices_to_check) {
3636         my $to_check = [];
3637         for my $d (@$devices_to_check) {
3638             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3639             next if !$d->{'pci_bridge'};
3640
3641             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3642             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3643         }
3644         $devices_to_check = $to_check;
3645     }
3646
3647     my $resblock = mon_cmd($vmid, 'query-block');
3648     foreach my $block (@$resblock) {
3649         if($block->{device} =~ m/^drive-(\S+)/){
3650                 $devices->{$1} = 1;
3651         }
3652     }
3653
3654     my $resmice = mon_cmd($vmid, 'query-mice');
3655     foreach my $mice (@$resmice) {
3656         if ($mice->{name} eq 'QEMU HID Tablet') {
3657             $devices->{tablet} = 1;
3658             last;
3659         }
3660     }
3661
3662     # for usb devices there is no query-usb
3663     # but we can iterate over the entries in
3664     # qom-list path=/machine/peripheral
3665     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3666     foreach my $per (@$resperipheral) {
3667         if ($per->{name} =~ m/^usb\d+$/) {
3668             $devices->{$per->{name}} = 1;
3669         }
3670     }
3671
3672     return $devices;
3673 }
3674
3675 sub vm_deviceplug {
3676     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3677
3678     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3679
3680     my $devices_list = vm_devices_list($vmid);
3681     return 1 if defined($devices_list->{$deviceid});
3682
3683     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3684
3685     if ($deviceid eq 'tablet') {
3686
3687         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3688
3689     } elsif ($deviceid eq 'keyboard') {
3690
3691         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3692
3693     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3694
3695         die "usb hotplug currently not reliable\n";
3696         # since we can't reliably hot unplug all added usb devices
3697         # and usb passthrough disables live migration
3698         # we disable usb hotplugging for now
3699         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3700
3701     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3702
3703         qemu_iothread_add($vmid, $deviceid, $device);
3704
3705         qemu_driveadd($storecfg, $vmid, $device);
3706         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3707
3708         qemu_deviceadd($vmid, $devicefull);
3709         eval { qemu_deviceaddverify($vmid, $deviceid); };
3710         if (my $err = $@) {
3711             eval { qemu_drivedel($vmid, $deviceid); };
3712             warn $@ if $@;
3713             die $err;
3714         }
3715
3716     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3717
3718
3719         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3720         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3721         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3722
3723         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3724
3725         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3726             qemu_iothread_add($vmid, $deviceid, $device);
3727             $devicefull .= ",iothread=iothread-$deviceid";
3728         }
3729
3730         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3731             $devicefull .= ",num_queues=$device->{queues}";
3732         }
3733
3734         qemu_deviceadd($vmid, $devicefull);
3735         qemu_deviceaddverify($vmid, $deviceid);
3736
3737     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3738
3739         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3740         qemu_driveadd($storecfg, $vmid, $device);
3741
3742         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3743         eval { qemu_deviceadd($vmid, $devicefull); };
3744         if (my $err = $@) {
3745             eval { qemu_drivedel($vmid, $deviceid); };
3746             warn $@ if $@;
3747             die $err;
3748         }
3749
3750     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3751
3752         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3753
3754         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3755         my $use_old_bios_files = undef;
3756         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3757
3758         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3759         qemu_deviceadd($vmid, $netdevicefull);
3760         eval {
3761             qemu_deviceaddverify($vmid, $deviceid);
3762             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3763         };
3764         if (my $err = $@) {
3765             eval { qemu_netdevdel($vmid, $deviceid); };
3766             warn $@ if $@;
3767             die $err;
3768         }
3769
3770     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3771
3772         my $bridgeid = $2;
3773         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3774         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3775
3776         qemu_deviceadd($vmid, $devicefull);
3777         qemu_deviceaddverify($vmid, $deviceid);
3778
3779     } else {
3780         die "can't hotplug device '$deviceid'\n";
3781     }
3782
3783     return 1;
3784 }
3785
3786 # fixme: this should raise exceptions on error!
3787 sub vm_deviceunplug {
3788     my ($vmid, $conf, $deviceid) = @_;
3789
3790     my $devices_list = vm_devices_list($vmid);
3791     return 1 if !defined($devices_list->{$deviceid});
3792
3793     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3794
3795     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3796
3797         qemu_devicedel($vmid, $deviceid);
3798
3799     } elsif ($deviceid =~ m/^usb\d+$/) {
3800
3801         die "usb hotplug currently not reliable\n";
3802         # when unplugging usb devices this way,
3803         # there may be remaining usb controllers/hubs
3804         # so we disable it for now
3805         qemu_devicedel($vmid, $deviceid);
3806         qemu_devicedelverify($vmid, $deviceid);
3807
3808     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3809
3810         qemu_devicedel($vmid, $deviceid);
3811         qemu_devicedelverify($vmid, $deviceid);
3812         qemu_drivedel($vmid, $deviceid);
3813         qemu_iothread_del($conf, $vmid, $deviceid);
3814
3815     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3816
3817         qemu_devicedel($vmid, $deviceid);
3818         qemu_devicedelverify($vmid, $deviceid);
3819         qemu_iothread_del($conf, $vmid, $deviceid);
3820
3821     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3822
3823         qemu_devicedel($vmid, $deviceid);
3824         qemu_drivedel($vmid, $deviceid);
3825         qemu_deletescsihw($conf, $vmid, $deviceid);
3826
3827     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3828
3829         qemu_devicedel($vmid, $deviceid);
3830         qemu_devicedelverify($vmid, $deviceid);
3831         qemu_netdevdel($vmid, $deviceid);
3832
3833     } else {
3834         die "can't unplug device '$deviceid'\n";
3835     }
3836
3837     return 1;
3838 }
3839
3840 sub qemu_deviceadd {
3841     my ($vmid, $devicefull) = @_;
3842
3843     $devicefull = "driver=".$devicefull;
3844     my %options =  split(/[=,]/, $devicefull);
3845
3846     mon_cmd($vmid, "device_add" , %options);
3847 }
3848
3849 sub qemu_devicedel {
3850     my ($vmid, $deviceid) = @_;
3851
3852     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3853 }
3854
3855 sub qemu_iothread_add {
3856     my($vmid, $deviceid, $device) = @_;
3857
3858     if ($device->{iothread}) {
3859         my $iothreads = vm_iothreads_list($vmid);
3860         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3861     }
3862 }
3863
3864 sub qemu_iothread_del {
3865     my($conf, $vmid, $deviceid) = @_;
3866
3867     my $confid = $deviceid;
3868     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3869         $confid = 'scsi' . $1;
3870     }
3871     my $device = parse_drive($confid, $conf->{$confid});
3872     if ($device->{iothread}) {
3873         my $iothreads = vm_iothreads_list($vmid);
3874         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3875     }
3876 }
3877
3878 sub qemu_objectadd {
3879     my($vmid, $objectid, $qomtype) = @_;
3880
3881     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3882
3883     return 1;
3884 }
3885
3886 sub qemu_objectdel {
3887     my($vmid, $objectid) = @_;
3888
3889     mon_cmd($vmid, "object-del", id => $objectid);
3890
3891     return 1;
3892 }
3893
3894 sub qemu_driveadd {
3895     my ($storecfg, $vmid, $device) = @_;
3896
3897     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3898     $drive =~ s/\\/\\\\/g;
3899     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3900
3901     # If the command succeeds qemu prints: "OK"
3902     return 1 if $ret =~ m/OK/s;
3903
3904     die "adding drive failed: $ret\n";
3905 }
3906
3907 sub qemu_drivedel {
3908     my($vmid, $deviceid) = @_;
3909
3910     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3911     $ret =~ s/^\s+//;
3912
3913     return 1 if $ret eq "";
3914
3915     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3916     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3917
3918     die "deleting drive $deviceid failed : $ret\n";
3919 }
3920
3921 sub qemu_deviceaddverify {
3922     my ($vmid, $deviceid) = @_;
3923
3924     for (my $i = 0; $i <= 5; $i++) {
3925          my $devices_list = vm_devices_list($vmid);
3926          return 1 if defined($devices_list->{$deviceid});
3927          sleep 1;
3928     }
3929
3930     die "error on hotplug device '$deviceid'\n";
3931 }
3932
3933
3934 sub qemu_devicedelverify {
3935     my ($vmid, $deviceid) = @_;
3936
3937     # need to verify that the device is correctly removed as device_del
3938     # is async and empty return is not reliable
3939
3940     for (my $i = 0; $i <= 5; $i++) {
3941          my $devices_list = vm_devices_list($vmid);
3942          return 1 if !defined($devices_list->{$deviceid});
3943          sleep 1;
3944     }
3945
3946     die "error on hot-unplugging device '$deviceid'\n";
3947 }
3948
3949 sub qemu_findorcreatescsihw {
3950     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3951
3952     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3953
3954     my $scsihwid="$controller_prefix$controller";
3955     my $devices_list = vm_devices_list($vmid);
3956
3957     if(!defined($devices_list->{$scsihwid})) {
3958         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3959     }
3960
3961     return 1;
3962 }
3963
3964 sub qemu_deletescsihw {
3965     my ($conf, $vmid, $opt) = @_;
3966
3967     my $device = parse_drive($opt, $conf->{$opt});
3968
3969     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3970         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3971         return 1;
3972     }
3973
3974     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3975
3976     my $devices_list = vm_devices_list($vmid);
3977     foreach my $opt (keys %{$devices_list}) {
3978         if (is_valid_drivename($opt)) {
3979             my $drive = parse_drive($opt, $conf->{$opt});
3980             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3981                 return 1;
3982             }
3983         }
3984     }
3985
3986     my $scsihwid="scsihw$controller";
3987
3988     vm_deviceunplug($vmid, $conf, $scsihwid);
3989
3990     return 1;
3991 }
3992
3993 sub qemu_add_pci_bridge {
3994     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3995
3996     my $bridges = {};
3997
3998     my $bridgeid;
3999
4000     print_pci_addr($device, $bridges, $arch, $machine_type);
4001
4002     while (my ($k, $v) = each %$bridges) {
4003         $bridgeid = $k;
4004     }
4005     return 1 if !defined($bridgeid) || $bridgeid < 1;
4006
4007     my $bridge = "pci.$bridgeid";
4008     my $devices_list = vm_devices_list($vmid);
4009
4010     if (!defined($devices_list->{$bridge})) {
4011         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4012     }
4013
4014     return 1;
4015 }
4016
4017 sub qemu_set_link_status {
4018     my ($vmid, $device, $up) = @_;
4019
4020     mon_cmd($vmid, "set_link", name => $device,
4021                up => $up ? JSON::true : JSON::false);
4022 }
4023
4024 sub qemu_netdevadd {
4025     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4026
4027     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4028     my %options =  split(/[=,]/, $netdev);
4029
4030     mon_cmd($vmid, "netdev_add",  %options);
4031     return 1;
4032 }
4033
4034 sub qemu_netdevdel {
4035     my ($vmid, $deviceid) = @_;
4036
4037     mon_cmd($vmid, "netdev_del", id => $deviceid);
4038 }
4039
4040 sub qemu_usb_hotplug {
4041     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4042
4043     return if !$device;
4044
4045     # remove the old one first
4046     vm_deviceunplug($vmid, $conf, $deviceid);
4047
4048     # check if xhci controller is necessary and available
4049     if ($device->{usb3}) {
4050
4051         my $devicelist = vm_devices_list($vmid);
4052
4053         if (!$devicelist->{xhci}) {
4054             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4055             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4056         }
4057     }
4058     my $d = parse_usb_device($device->{host});
4059     $d->{usb3} = $device->{usb3};
4060
4061     # add the new one
4062     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4063 }
4064
4065 sub qemu_cpu_hotplug {
4066     my ($vmid, $conf, $vcpus) = @_;
4067
4068     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4069
4070     my $sockets = 1;
4071     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4072     $sockets = $conf->{sockets} if  $conf->{sockets};
4073     my $cores = $conf->{cores} || 1;
4074     my $maxcpus = $sockets * $cores;
4075
4076     $vcpus = $maxcpus if !$vcpus;
4077
4078     die "you can't add more vcpus than maxcpus\n"
4079         if $vcpus > $maxcpus;
4080
4081     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4082
4083     if ($vcpus < $currentvcpus) {
4084
4085         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4086
4087             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4088                 qemu_devicedel($vmid, "cpu$i");
4089                 my $retry = 0;
4090                 my $currentrunningvcpus = undef;
4091                 while (1) {
4092                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4093                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4094                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4095                     $retry++;
4096                     sleep 1;
4097                 }
4098                 #update conf after each succesfull cpu unplug
4099                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4100                 PVE::QemuConfig->write_config($vmid, $conf);
4101             }
4102         } else {
4103             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4104         }
4105
4106         return;
4107     }
4108
4109     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4110     die "vcpus in running vm does not match its configuration\n"
4111         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4112
4113     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4114
4115         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4116             my $cpustr = print_cpu_device($conf, $i);
4117             qemu_deviceadd($vmid, $cpustr);
4118
4119             my $retry = 0;
4120             my $currentrunningvcpus = undef;
4121             while (1) {
4122                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4123                 last if scalar(@{$currentrunningvcpus}) == $i;
4124                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4125                 sleep 1;
4126                 $retry++;
4127             }
4128             #update conf after each succesfull cpu hotplug
4129             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4130             PVE::QemuConfig->write_config($vmid, $conf);
4131         }
4132     } else {
4133
4134         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4135             mon_cmd($vmid, "cpu-add", id => int($i));
4136         }
4137     }
4138 }
4139
4140 sub qemu_block_set_io_throttle {
4141     my ($vmid, $deviceid,
4142         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4143         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4144         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4145         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4146
4147     return if !check_running($vmid) ;
4148
4149     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4150         bps => int($bps),
4151         bps_rd => int($bps_rd),
4152         bps_wr => int($bps_wr),
4153         iops => int($iops),
4154         iops_rd => int($iops_rd),
4155         iops_wr => int($iops_wr),
4156         bps_max => int($bps_max),
4157         bps_rd_max => int($bps_rd_max),
4158         bps_wr_max => int($bps_wr_max),
4159         iops_max => int($iops_max),
4160         iops_rd_max => int($iops_rd_max),
4161         iops_wr_max => int($iops_wr_max),
4162         bps_max_length => int($bps_max_length),
4163         bps_rd_max_length => int($bps_rd_max_length),
4164         bps_wr_max_length => int($bps_wr_max_length),
4165         iops_max_length => int($iops_max_length),
4166         iops_rd_max_length => int($iops_rd_max_length),
4167         iops_wr_max_length => int($iops_wr_max_length),
4168     );
4169
4170 }
4171
4172 # old code, only used to shutdown old VM after update
4173 sub __read_avail {
4174     my ($fh, $timeout) = @_;
4175
4176     my $sel = new IO::Select;
4177     $sel->add($fh);
4178
4179     my $res = '';
4180     my $buf;
4181
4182     my @ready;
4183     while (scalar (@ready = $sel->can_read($timeout))) {
4184         my $count;
4185         if ($count = $fh->sysread($buf, 8192)) {
4186             if ($buf =~ /^(.*)\(qemu\) $/s) {
4187                 $res .= $1;
4188                 last;
4189             } else {
4190                 $res .= $buf;
4191             }
4192         } else {
4193             if (!defined($count)) {
4194                 die "$!\n";
4195             }
4196             last;
4197         }
4198     }
4199
4200     die "monitor read timeout\n" if !scalar(@ready);
4201
4202     return $res;
4203 }
4204
4205 sub qemu_block_resize {
4206     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4207
4208     my $running = check_running($vmid);
4209
4210     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4211
4212     return if !$running;
4213
4214     my $padding = (1024 - $size % 1024) % 1024;
4215     $size = $size + $padding;
4216
4217     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4218
4219 }
4220
4221 sub qemu_volume_snapshot {
4222     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4223
4224     my $running = check_running($vmid);
4225
4226     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4227         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4228     } else {
4229         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4230     }
4231 }
4232
4233 sub qemu_volume_snapshot_delete {
4234     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4235
4236     my $running = check_running($vmid);
4237
4238     if($running) {
4239
4240         $running = undef;
4241         my $conf = PVE::QemuConfig->load_config($vmid);
4242         foreach_drive($conf, sub {
4243             my ($ds, $drive) = @_;
4244             $running = 1 if $drive->{file} eq $volid;
4245         });
4246     }
4247
4248     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4249         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4250     } else {
4251         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4252     }
4253 }
4254
4255 sub set_migration_caps {
4256     my ($vmid) = @_;
4257
4258     my $cap_ref = [];
4259
4260     my $enabled_cap = {
4261         "auto-converge" => 1,
4262         "xbzrle" => 1,
4263         "x-rdma-pin-all" => 0,
4264         "zero-blocks" => 0,
4265         "compress" => 0
4266     };
4267
4268     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4269
4270     for my $supported_capability (@$supported_capabilities) {
4271         push @$cap_ref, {
4272             capability => $supported_capability->{capability},
4273             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4274         };
4275     }
4276
4277     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4278 }
4279
4280 my $fast_plug_option = {
4281     'lock' => 1,
4282     'name' => 1,
4283     'onboot' => 1,
4284     'shares' => 1,
4285     'startup' => 1,
4286     'description' => 1,
4287     'protection' => 1,
4288     'vmstatestorage' => 1,
4289     'hookscript' => 1,
4290     'tags' => 1,
4291 };
4292
4293 # hotplug changes in [PENDING]
4294 # $selection hash can be used to only apply specified options, for
4295 # example: { cores => 1 } (only apply changed 'cores')
4296 # $errors ref is used to return error messages
4297 sub vmconfig_hotplug_pending {
4298     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4299
4300     my $defaults = load_defaults();
4301     my $arch = get_vm_arch($conf);
4302     my $machine_type = get_vm_machine($conf, undef, $arch);
4303
4304     # commit values which do not have any impact on running VM first
4305     # Note: those option cannot raise errors, we we do not care about
4306     # $selection and always apply them.
4307
4308     my $add_error = sub {
4309         my ($opt, $msg) = @_;
4310         $errors->{$opt} = "hotplug problem - $msg";
4311     };
4312
4313     my $changes = 0;
4314     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4315         if ($fast_plug_option->{$opt}) {
4316             $conf->{$opt} = $conf->{pending}->{$opt};
4317             delete $conf->{pending}->{$opt};
4318             $changes = 1;
4319         }
4320     }
4321
4322     if ($changes) {
4323         PVE::QemuConfig->write_config($vmid, $conf);
4324     }
4325
4326     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4327
4328     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4329     foreach my $opt (sort keys %$pending_delete_hash) {
4330         next if $selection && !$selection->{$opt};
4331         my $force = $pending_delete_hash->{$opt}->{force};
4332         eval {
4333             if ($opt eq 'hotplug') {
4334                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4335             } elsif ($opt eq 'tablet') {
4336                 die "skip\n" if !$hotplug_features->{usb};
4337                 if ($defaults->{tablet}) {
4338                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4339                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4340                         if $arch eq 'aarch64';
4341                 } else {
4342                     vm_deviceunplug($vmid, $conf, 'tablet');
4343                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4344                 }
4345             } elsif ($opt =~ m/^usb\d+/) {
4346                 die "skip\n";
4347                 # since we cannot reliably hot unplug usb devices
4348                 # we are disabling it
4349                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4350                 vm_deviceunplug($vmid, $conf, $opt);
4351             } elsif ($opt eq 'vcpus') {
4352                 die "skip\n" if !$hotplug_features->{cpu};
4353                 qemu_cpu_hotplug($vmid, $conf, undef);
4354             } elsif ($opt eq 'balloon') {
4355                 # enable balloon device is not hotpluggable
4356                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4357                 # here we reset the ballooning value to memory
4358                 my $balloon = $conf->{memory} || $defaults->{memory};
4359                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4360             } elsif ($fast_plug_option->{$opt}) {
4361                 # do nothing
4362             } elsif ($opt =~ m/^net(\d+)$/) {
4363                 die "skip\n" if !$hotplug_features->{network};
4364                 vm_deviceunplug($vmid, $conf, $opt);
4365             } elsif (is_valid_drivename($opt)) {
4366                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4367                 vm_deviceunplug($vmid, $conf, $opt);
4368                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4369             } elsif ($opt =~ m/^memory$/) {
4370                 die "skip\n" if !$hotplug_features->{memory};
4371                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4372             } elsif ($opt eq 'cpuunits') {
4373                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4374             } elsif ($opt eq 'cpulimit') {
4375                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4376             } else {
4377                 die "skip\n";
4378             }
4379         };
4380         if (my $err = $@) {
4381             &$add_error($opt, $err) if $err ne "skip\n";
4382         } else {
4383             delete $conf->{$opt};
4384             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4385         }
4386     }
4387
4388     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4389     $apply_pending_cloudinit = sub {
4390         return if $apply_pending_cloudinit_done; # once is enough
4391         $apply_pending_cloudinit_done = 1; # once is enough
4392
4393         my ($key, $value) = @_;
4394
4395         my @cloudinit_opts = keys %$confdesc_cloudinit;
4396         foreach my $opt (keys %{$conf->{pending}}) {
4397             next if !grep { $_ eq $opt } @cloudinit_opts;
4398             $conf->{$opt} = delete $conf->{pending}->{$opt};
4399         }
4400
4401         my $new_conf = { %$conf };
4402         $new_conf->{$key} = $value;
4403         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4404     };
4405
4406     foreach my $opt (keys %{$conf->{pending}}) {
4407         next if $selection && !$selection->{$opt};
4408         my $value = $conf->{pending}->{$opt};
4409         eval {
4410             if ($opt eq 'hotplug') {
4411                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4412             } elsif ($opt eq 'tablet') {
4413                 die "skip\n" if !$hotplug_features->{usb};
4414                 if ($value == 1) {
4415                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4416                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4417                         if $arch eq 'aarch64';
4418                 } elsif ($value == 0) {
4419                     vm_deviceunplug($vmid, $conf, 'tablet');
4420                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4421                 }
4422             } elsif ($opt =~ m/^usb\d+$/) {
4423                 die "skip\n";
4424                 # since we cannot reliably hot unplug usb devices
4425                 # we are disabling it
4426                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4427                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4428                 die "skip\n" if !$d;
4429                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4430             } elsif ($opt eq 'vcpus') {
4431                 die "skip\n" if !$hotplug_features->{cpu};
4432                 qemu_cpu_hotplug($vmid, $conf, $value);
4433             } elsif ($opt eq 'balloon') {
4434                 # enable/disable balloning device is not hotpluggable
4435                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4436                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4437                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4438
4439                 # allow manual ballooning if shares is set to zero
4440                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4441                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4442                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4443                 }
4444             } elsif ($opt =~ m/^net(\d+)$/) {
4445                 # some changes can be done without hotplug
4446                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4447                                     $vmid, $opt, $value, $arch, $machine_type);
4448             } elsif (is_valid_drivename($opt)) {
4449                 die "skip\n" if $opt eq 'efidisk0';
4450                 # some changes can be done without hotplug
4451                 my $drive = parse_drive($opt, $value);
4452                 if (drive_is_cloudinit($drive)) {
4453                     &$apply_pending_cloudinit($opt, $value);
4454                 }
4455                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4456                                      $vmid, $opt, $value, $arch, $machine_type);
4457             } elsif ($opt =~ m/^memory$/) { #dimms
4458                 die "skip\n" if !$hotplug_features->{memory};
4459                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4460             } elsif ($opt eq 'cpuunits') {
4461                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4462             } elsif ($opt eq 'cpulimit') {
4463                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4464                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4465             } else {
4466                 die "skip\n";  # skip non-hot-pluggable options
4467             }
4468         };
4469         if (my $err = $@) {
4470             &$add_error($opt, $err) if $err ne "skip\n";
4471         } else {
4472             $conf->{$opt} = $value;
4473             delete $conf->{pending}->{$opt};
4474         }
4475     }
4476
4477     PVE::QemuConfig->write_config($vmid, $conf);
4478 }
4479
4480 sub try_deallocate_drive {
4481     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4482
4483     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4484         my $volid = $drive->{file};
4485         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4486             my $sid = PVE::Storage::parse_volume_id($volid);
4487             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4488
4489             # check if the disk is really unused
4490             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4491                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4492             PVE::Storage::vdisk_free($storecfg, $volid);
4493             return 1;
4494         } else {
4495             # If vm is not owner of this disk remove from config
4496             return 1;
4497         }
4498     }
4499
4500     return undef;
4501 }
4502
4503 sub vmconfig_delete_or_detach_drive {
4504     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4505
4506     my $drive = parse_drive($opt, $conf->{$opt});
4507
4508     my $rpcenv = PVE::RPCEnvironment::get();
4509     my $authuser = $rpcenv->get_user();
4510
4511     if ($force) {
4512         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4513         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4514     } else {
4515         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4516     }
4517 }
4518
4519
4520
4521 sub vmconfig_apply_pending {
4522     my ($vmid, $conf, $storecfg, $errors) = @_;
4523
4524     my $add_apply_error = sub {
4525         my ($opt, $msg) = @_;
4526         my $err_msg = "unable to apply pending change $opt : $msg";
4527         $errors->{$opt} = $err_msg;
4528         warn $err_msg;
4529     };
4530
4531     # cold plug
4532
4533     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4534     foreach my $opt (sort keys %$pending_delete_hash) {
4535         my $force = $pending_delete_hash->{$opt}->{force};
4536         eval {
4537             if ($opt =~ m/^unused/) {
4538                 die "internal error";
4539             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4540                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4541             }
4542         };
4543         if (my $err = $@) {
4544             $add_apply_error->($opt, $err);
4545         } else {
4546             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4547             delete $conf->{$opt};
4548         }
4549     }
4550
4551     PVE::QemuConfig->cleanup_pending($conf);
4552
4553     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4554         next if $opt eq 'delete'; # just to be sure
4555         eval {
4556             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4557                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4558             }
4559         };
4560         if (my $err = $@) {
4561             $add_apply_error->($opt, $err);
4562         } else {
4563             $conf->{$opt} = delete $conf->{pending}->{$opt};
4564         }
4565     }
4566
4567     # write all changes at once to avoid unnecessary i/o
4568     PVE::QemuConfig->write_config($vmid, $conf);
4569 }
4570
4571 sub vmconfig_update_net {
4572     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4573
4574     my $newnet = parse_net($value);
4575
4576     if ($conf->{$opt}) {
4577         my $oldnet = parse_net($conf->{$opt});
4578
4579         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4580             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4581             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4582             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4583
4584             # for non online change, we try to hot-unplug
4585             die "skip\n" if !$hotplug;
4586             vm_deviceunplug($vmid, $conf, $opt);
4587         } else {
4588
4589             die "internal error" if $opt !~ m/net(\d+)/;
4590             my $iface = "tap${vmid}i$1";
4591
4592             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4593                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4594                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4595                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4596                 PVE::Network::tap_unplug($iface);
4597
4598                 if ($have_sdn) {
4599                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4600                 } else {
4601                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4602                 }
4603             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4604                 # Rate can be applied on its own but any change above needs to
4605                 # include the rate in tap_plug since OVS resets everything.
4606                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4607             }
4608
4609             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4610                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4611             }
4612
4613             return 1;
4614         }
4615     }
4616
4617     if ($hotplug) {
4618         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4619     } else {
4620         die "skip\n";
4621     }
4622 }
4623
4624 sub vmconfig_update_disk {
4625     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4626
4627     my $drive = parse_drive($opt, $value);
4628
4629     if ($conf->{$opt}) {
4630
4631         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4632
4633             my $media = $drive->{media} || 'disk';
4634             my $oldmedia = $old_drive->{media} || 'disk';
4635             die "unable to change media type\n" if $media ne $oldmedia;
4636
4637             if (!drive_is_cdrom($old_drive)) {
4638
4639                 if ($drive->{file} ne $old_drive->{file}) {
4640
4641                     die "skip\n" if !$hotplug;
4642
4643                     # unplug and register as unused
4644                     vm_deviceunplug($vmid, $conf, $opt);
4645                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4646
4647                 } else {
4648                     # update existing disk
4649
4650                     # skip non hotpluggable value
4651                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4652                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4653                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4654                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4655                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4656                         die "skip\n";
4657                     }
4658
4659                     # apply throttle
4660                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4661                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4662                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4663                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4664                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4665                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4666                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4667                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4668                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4669                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4670                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4671                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4672                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4673                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4674                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4675                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4676                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4677                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4678
4679                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4680                                                    ($drive->{mbps} || 0)*1024*1024,
4681                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4682                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4683                                                    $drive->{iops} || 0,
4684                                                    $drive->{iops_rd} || 0,
4685                                                    $drive->{iops_wr} || 0,
4686                                                    ($drive->{mbps_max} || 0)*1024*1024,
4687                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4688                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4689                                                    $drive->{iops_max} || 0,
4690                                                    $drive->{iops_rd_max} || 0,
4691                                                    $drive->{iops_wr_max} || 0,
4692                                                    $drive->{bps_max_length} || 1,
4693                                                    $drive->{bps_rd_max_length} || 1,
4694                                                    $drive->{bps_wr_max_length} || 1,
4695                                                    $drive->{iops_max_length} || 1,
4696                                                    $drive->{iops_rd_max_length} || 1,
4697                                                    $drive->{iops_wr_max_length} || 1);
4698
4699                     }
4700
4701                     return 1;
4702                 }
4703
4704             } else { # cdrom
4705
4706                 if ($drive->{file} eq 'none') {
4707                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4708                     if (drive_is_cloudinit($old_drive)) {
4709                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4710                     }
4711                 } else {
4712                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4713
4714                     # force eject if locked
4715                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4716
4717                     if ($path) {
4718                         mon_cmd($vmid, "blockdev-change-medium",
4719                             id => "$opt", filename => "$path");
4720                     }
4721                 }
4722
4723                 return 1;
4724             }
4725         }
4726     }
4727
4728     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4729     # hotplug new disks
4730     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4731     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4732 }
4733
4734 # called in locked context by incoming migration
4735 sub vm_migrate_get_nbd_disks {
4736     my ($storecfg, $conf, $replicated_volumes) = @_;
4737
4738     my $local_volumes = {};
4739     foreach_drive($conf, sub {
4740         my ($ds, $drive) = @_;
4741
4742         return if drive_is_cdrom($drive);
4743
4744         my $volid = $drive->{file};
4745
4746         return if !$volid;
4747
4748         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4749
4750         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4751         return if $scfg->{shared};
4752
4753         # replicated disks re-use existing state via bitmap
4754         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4755         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4756     });
4757     return $local_volumes;
4758 }
4759
4760 # called in locked context by incoming migration
4761 sub vm_migrate_alloc_nbd_disks {
4762     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4763
4764     my $format = undef;
4765
4766     my $nbd = {};
4767     foreach my $opt (sort keys %$source_volumes) {
4768         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4769
4770         if ($use_existing) {
4771             $nbd->{$opt}->{drivestr} = print_drive($drive);
4772             $nbd->{$opt}->{volid} = $volid;
4773             $nbd->{$opt}->{replicated} = 1;
4774             next;
4775         }
4776
4777         # If a remote storage is specified and the format of the original
4778         # volume is not available there, fall back to the default format.
4779         # Otherwise use the same format as the original.
4780         if (!$storagemap->{identity}) {
4781             $storeid = map_storage($storagemap, $storeid);
4782             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4783             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4784             my $fileFormat = qemu_img_format($scfg, $volname);
4785             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4786         } else {
4787             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4788             $format = qemu_img_format($scfg, $volname);
4789         }
4790
4791         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4792         my $newdrive = $drive;
4793         $newdrive->{format} = $format;
4794         $newdrive->{file} = $newvolid;
4795         my $drivestr = print_drive($newdrive);
4796         $nbd->{$opt}->{drivestr} = $drivestr;
4797         $nbd->{$opt}->{volid} = $newvolid;
4798     }
4799
4800     return $nbd;
4801 }
4802
4803 # see vm_start_nolock for parameters, additionally:
4804 # migrate_opts:
4805 #   storagemap = parsed storage map for allocating NBD disks
4806 sub vm_start {
4807     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4808
4809     return PVE::QemuConfig->lock_config($vmid, sub {
4810         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4811
4812         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4813
4814         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4815
4816         PVE::QemuConfig->check_lock($conf)
4817             if !($params->{skiplock} || $params->{resume});
4818
4819         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4820
4821         if (my $storagemap = $migrate_opts->{storagemap}) {
4822             my $replicated = $migrate_opts->{replicated_volumes};
4823             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4824             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4825
4826             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4827                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4828             }
4829         }
4830
4831         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4832     });
4833 }
4834
4835
4836 # params:
4837 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4838 #   skiplock => 0/1, skip checking for config lock
4839 #   forcemachine => to force Qemu machine (rollback/migration)
4840 #   timeout => in seconds
4841 #   paused => start VM in paused state (backup)
4842 #   resume => resume from hibernation
4843 # migrate_opts:
4844 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4845 #   migratedfrom => source node
4846 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4847 #   network => CIDR of migration network
4848 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4849 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4850 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4851 sub vm_start_nolock {
4852     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4853
4854     my $statefile = $params->{statefile};
4855     my $resume = $params->{resume};
4856
4857     my $migratedfrom = $migrate_opts->{migratedfrom};
4858     my $migration_type = $migrate_opts->{type};
4859
4860     my $res = {};
4861
4862     # clean up leftover reboot request files
4863     eval { clear_reboot_request($vmid); };
4864     warn $@ if $@;
4865
4866     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4867         vmconfig_apply_pending($vmid, $conf, $storecfg);
4868         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4869     }
4870
4871     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4872
4873     my $defaults = load_defaults();
4874
4875     # set environment variable useful inside network script
4876     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4877
4878     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4879
4880     my $forcemachine = $params->{forcemachine};
4881     if ($resume) {
4882         # enforce machine type on suspended vm to ensure HW compatibility
4883         $forcemachine = $conf->{runningmachine};
4884         print "Resuming suspended VM\n";
4885     }
4886
4887     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4888
4889     my $migration_ip;
4890     my $get_migration_ip = sub {
4891         my ($nodename) = @_;
4892
4893         return $migration_ip if defined($migration_ip);
4894
4895         my $cidr = $migrate_opts->{network};
4896
4897         if (!defined($cidr)) {
4898             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4899             $cidr = $dc_conf->{migration}->{network};
4900         }
4901
4902         if (defined($cidr)) {
4903             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4904
4905             die "could not get IP: no address configured on local " .
4906                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4907
4908             die "could not get IP: multiple addresses configured on local " .
4909                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4910
4911             $migration_ip = @$ips[0];
4912         }
4913
4914         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4915             if !defined($migration_ip);
4916
4917         return $migration_ip;
4918     };
4919
4920     my $migrate_uri;
4921     if ($statefile) {
4922         if ($statefile eq 'tcp') {
4923             my $localip = "localhost";
4924             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4925             my $nodename = nodename();
4926
4927             if (!defined($migration_type)) {
4928                 if (defined($datacenterconf->{migration}->{type})) {
4929                     $migration_type = $datacenterconf->{migration}->{type};
4930                 } else {
4931                     $migration_type = 'secure';
4932                 }
4933             }
4934
4935             if ($migration_type eq 'insecure') {
4936                 $localip = $get_migration_ip->($nodename);
4937                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4938             }
4939
4940             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4941             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4942             $migrate_uri = "tcp:${localip}:${migrate_port}";
4943             push @$cmd, '-incoming', $migrate_uri;
4944             push @$cmd, '-S';
4945
4946         } elsif ($statefile eq 'unix') {
4947             # should be default for secure migrations as a ssh TCP forward
4948             # tunnel is not deterministic reliable ready and fails regurarly
4949             # to set up in time, so use UNIX socket forwards
4950             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4951             unlink $socket_addr;
4952
4953             $migrate_uri = "unix:$socket_addr";
4954
4955             push @$cmd, '-incoming', $migrate_uri;
4956             push @$cmd, '-S';
4957
4958         } elsif (-e $statefile) {
4959             push @$cmd, '-loadstate', $statefile;
4960         } else {
4961             my $statepath = PVE::Storage::path($storecfg, $statefile);
4962             push @$vollist, $statefile;
4963             push @$cmd, '-loadstate', $statepath;
4964         }
4965     } elsif ($params->{paused}) {
4966         push @$cmd, '-S';
4967     }
4968
4969     # host pci devices
4970     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4971       my $d = parse_hostpci($conf->{"hostpci$i"});
4972       next if !$d;
4973       my $pcidevices = $d->{pciid};
4974       foreach my $pcidevice (@$pcidevices) {
4975             my $pciid = $pcidevice->{id};
4976
4977             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4978             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4979             die "no pci device info for device '$pciid'\n" if !$info;
4980
4981             if ($d->{mdev}) {
4982                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4983                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4984             } else {
4985                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4986                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4987                 die "can't reset pci device '$pciid'\n"
4988                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4989             }
4990       }
4991     }
4992
4993     PVE::Storage::activate_volumes($storecfg, $vollist);
4994
4995     eval {
4996         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4997             outfunc => sub {}, errfunc => sub {});
4998     };
4999     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5000     # timeout should be more than enough here...
5001     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5002
5003     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5004                                               : $defaults->{cpuunits};
5005
5006     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5007     my %run_params = (
5008         timeout => $statefile ? undef : $start_timeout,
5009         umask => 0077,
5010         noerr => 1,
5011     );
5012
5013     # when migrating, prefix QEMU output so other side can pick up any
5014     # errors that might occur and show the user
5015     if ($migratedfrom) {
5016         $run_params{quiet} = 1;
5017         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5018     }
5019
5020     my %properties = (
5021         Slice => 'qemu.slice',
5022         KillMode => 'none',
5023         CPUShares => $cpuunits
5024     );
5025
5026     if (my $cpulimit = $conf->{cpulimit}) {
5027         $properties{CPUQuota} = int($cpulimit * 100);
5028     }
5029     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5030
5031     my $run_qemu = sub {
5032         PVE::Tools::run_fork sub {
5033             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5034
5035             my $exitcode = run_command($cmd, %run_params);
5036             die "QEMU exited with code $exitcode\n" if $exitcode;
5037         };
5038     };
5039
5040     if ($conf->{hugepages}) {
5041
5042         my $code = sub {
5043             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5044             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5045
5046             PVE::QemuServer::Memory::hugepages_mount();
5047             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5048
5049             eval { $run_qemu->() };
5050             if (my $err = $@) {
5051                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5052                 die $err;
5053             }
5054
5055             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5056         };
5057         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5058
5059     } else {
5060         eval { $run_qemu->() };
5061     }
5062
5063     if (my $err = $@) {
5064         # deactivate volumes if start fails
5065         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5066         die "start failed: $err";
5067     }
5068
5069     print "migration listens on $migrate_uri\n" if $migrate_uri;
5070     $res->{migrate_uri} = $migrate_uri;
5071
5072     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5073         eval { mon_cmd($vmid, "cont"); };
5074         warn $@ if $@;
5075     }
5076
5077     #start nbd server for storage migration
5078     if (my $nbd = $migrate_opts->{nbd}) {
5079         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5080
5081         my $migrate_storage_uri;
5082         # nbd_protocol_version > 0 for unix socket support
5083         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5084             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5085             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5086             $migrate_storage_uri = "nbd:unix:$socket_path";
5087         } else {
5088             my $nodename = nodename();
5089             my $localip = $get_migration_ip->($nodename);
5090             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5091             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5092
5093             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5094             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5095             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5096         }
5097
5098         $res->{migrate_storage_uri} = $migrate_storage_uri;
5099
5100         foreach my $opt (sort keys %$nbd) {
5101             my $drivestr = $nbd->{$opt}->{drivestr};
5102             my $volid = $nbd->{$opt}->{volid};
5103             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5104             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5105             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5106             print "re-using replicated volume: $opt - $volid\n"
5107                 if $nbd->{$opt}->{replicated};
5108
5109             $res->{drives}->{$opt} = $nbd->{$opt};
5110             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5111         }
5112     }
5113
5114     if ($migratedfrom) {
5115         eval {
5116             set_migration_caps($vmid);
5117         };
5118         warn $@ if $@;
5119
5120         if ($spice_port) {
5121             print "spice listens on port $spice_port\n";
5122             $res->{spice_port} = $spice_port;
5123             if ($migrate_opts->{spice_ticket}) {
5124                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5125                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5126             }
5127         }
5128
5129     } else {
5130         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5131             if !$statefile && $conf->{balloon};
5132
5133         foreach my $opt (keys %$conf) {
5134             next if $opt !~  m/^net\d+$/;
5135             my $nicconf = parse_net($conf->{$opt});
5136             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5137         }
5138     }
5139
5140     mon_cmd($vmid, 'qom-set',
5141                 path => "machine/peripheral/balloon0",
5142                 property => "guest-stats-polling-interval",
5143                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5144
5145     if ($resume) {
5146         print "Resumed VM, removing state\n";
5147         if (my $vmstate = $conf->{vmstate}) {
5148             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5149             PVE::Storage::vdisk_free($storecfg, $vmstate);
5150         }
5151         delete $conf->@{qw(lock vmstate runningmachine)};
5152         PVE::QemuConfig->write_config($vmid, $conf);
5153     }
5154
5155     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5156
5157     return $res;
5158 }
5159
5160 sub vm_commandline {
5161     my ($storecfg, $vmid, $snapname) = @_;
5162
5163     my $conf = PVE::QemuConfig->load_config($vmid);
5164     my $forcemachine;
5165
5166     if ($snapname) {
5167         my $snapshot = $conf->{snapshots}->{$snapname};
5168         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5169
5170         # check for a 'runningmachine' in snapshot
5171         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5172
5173         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5174
5175         $conf = $snapshot;
5176     }
5177
5178     my $defaults = load_defaults();
5179
5180     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5181
5182     return PVE::Tools::cmd2string($cmd);
5183 }
5184
5185 sub vm_reset {
5186     my ($vmid, $skiplock) = @_;
5187
5188     PVE::QemuConfig->lock_config($vmid, sub {
5189
5190         my $conf = PVE::QemuConfig->load_config($vmid);
5191
5192         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5193
5194         mon_cmd($vmid, "system_reset");
5195     });
5196 }
5197
5198 sub get_vm_volumes {
5199     my ($conf) = @_;
5200
5201     my $vollist = [];
5202     foreach_volid($conf, sub {
5203         my ($volid, $attr) = @_;
5204
5205         return if $volid =~ m|^/|;
5206
5207         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5208         return if !$sid;
5209
5210         push @$vollist, $volid;
5211     });
5212
5213     return $vollist;
5214 }
5215
5216 sub vm_stop_cleanup {
5217     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5218
5219     eval {
5220
5221         if (!$keepActive) {
5222             my $vollist = get_vm_volumes($conf);
5223             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5224         }
5225
5226         foreach my $ext (qw(mon qmp pid vnc qga)) {
5227             unlink "/var/run/qemu-server/${vmid}.$ext";
5228         }
5229
5230         if ($conf->{ivshmem}) {
5231             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5232             # just delete it for now, VMs which have this already open do not
5233             # are affected, but new VMs will get a separated one. If this
5234             # becomes an issue we either add some sort of ref-counting or just
5235             # add a "don't delete on stop" flag to the ivshmem format.
5236             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5237         }
5238
5239         foreach my $key (keys %$conf) {
5240             next if $key !~ m/^hostpci(\d+)$/;
5241             my $hostpciindex = $1;
5242             my $d = parse_hostpci($conf->{$key});
5243             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5244
5245             foreach my $pci (@{$d->{pciid}}) {
5246                 my $pciid = $pci->{id};
5247                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5248             }
5249         }
5250
5251         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5252     };
5253     warn $@ if $@; # avoid errors - just warn
5254 }
5255
5256 # call only in locked context
5257 sub _do_vm_stop {
5258     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5259
5260     my $pid = check_running($vmid, $nocheck);
5261     return if !$pid;
5262
5263     my $conf;
5264     if (!$nocheck) {
5265         $conf = PVE::QemuConfig->load_config($vmid);
5266         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5267         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5268             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5269             $timeout = $opts->{down} if $opts->{down};
5270         }
5271         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5272     }
5273
5274     eval {
5275         if ($shutdown) {
5276             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5277                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5278             } else {
5279                 mon_cmd($vmid, "system_powerdown");
5280             }
5281         } else {
5282             mon_cmd($vmid, "quit");
5283         }
5284     };
5285     my $err = $@;
5286
5287     if (!$err) {
5288         $timeout = 60 if !defined($timeout);
5289
5290         my $count = 0;
5291         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5292             $count++;
5293             sleep 1;
5294         }
5295
5296         if ($count >= $timeout) {
5297             if ($force) {
5298                 warn "VM still running - terminating now with SIGTERM\n";
5299                 kill 15, $pid;
5300             } else {
5301                 die "VM quit/powerdown failed - got timeout\n";
5302             }
5303         } else {
5304             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5305             return;
5306         }
5307     } else {
5308         if ($force) {
5309             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5310             kill 15, $pid;
5311         } else {
5312             die "VM quit/powerdown failed\n";
5313         }
5314     }
5315
5316     # wait again
5317     $timeout = 10;
5318
5319     my $count = 0;
5320     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5321         $count++;
5322         sleep 1;
5323     }
5324
5325     if ($count >= $timeout) {
5326         warn "VM still running - terminating now with SIGKILL\n";
5327         kill 9, $pid;
5328         sleep 1;
5329     }
5330
5331     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5332 }
5333
5334 # Note: use $nocheck to skip tests if VM configuration file exists.
5335 # We need that when migration VMs to other nodes (files already moved)
5336 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5337 sub vm_stop {
5338     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5339
5340     $force = 1 if !defined($force) && !$shutdown;
5341
5342     if ($migratedfrom){
5343         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5344         kill 15, $pid if $pid;
5345         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5346         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5347         return;
5348     }
5349
5350     PVE::QemuConfig->lock_config($vmid, sub {
5351         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5352    });
5353 }
5354
5355 sub vm_reboot {
5356     my ($vmid, $timeout) = @_;
5357
5358     PVE::QemuConfig->lock_config($vmid, sub {
5359         eval {
5360
5361             # only reboot if running, as qmeventd starts it again on a stop event
5362             return if !check_running($vmid);
5363
5364             create_reboot_request($vmid);
5365
5366             my $storecfg = PVE::Storage::config();
5367             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5368
5369         };
5370         if (my $err = $@) {
5371             # avoid that the next normal shutdown will be confused for a reboot
5372             clear_reboot_request($vmid);
5373             die $err;
5374         }
5375    });
5376 }
5377
5378 # note: if using the statestorage parameter, the caller has to check privileges
5379 sub vm_suspend {
5380     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5381
5382     my $conf;
5383     my $path;
5384     my $storecfg;
5385     my $vmstate;
5386
5387     PVE::QemuConfig->lock_config($vmid, sub {
5388
5389         $conf = PVE::QemuConfig->load_config($vmid);
5390
5391         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5392         PVE::QemuConfig->check_lock($conf)
5393             if !($skiplock || $is_backing_up);
5394
5395         die "cannot suspend to disk during backup\n"
5396             if $is_backing_up && $includestate;
5397
5398         if ($includestate) {
5399             $conf->{lock} = 'suspending';
5400             my $date = strftime("%Y-%m-%d", localtime(time()));
5401             $storecfg = PVE::Storage::config();
5402             if (!$statestorage) {
5403                 $statestorage = find_vmstate_storage($conf, $storecfg);
5404                 # check permissions for the storage
5405                 my $rpcenv = PVE::RPCEnvironment::get();
5406                 if ($rpcenv->{type} ne 'cli') {
5407                     my $authuser = $rpcenv->get_user();
5408                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5409                 }
5410             }
5411
5412
5413             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5414             $path = PVE::Storage::path($storecfg, $vmstate);
5415             PVE::QemuConfig->write_config($vmid, $conf);
5416         } else {
5417             mon_cmd($vmid, "stop");
5418         }
5419     });
5420
5421     if ($includestate) {
5422         # save vm state
5423         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5424
5425         eval {
5426             mon_cmd($vmid, "savevm-start", statefile => $path);
5427             for(;;) {
5428                 my $state = mon_cmd($vmid, "query-savevm");
5429                 if (!$state->{status}) {
5430                     die "savevm not active\n";
5431                 } elsif ($state->{status} eq 'active') {
5432                     sleep(1);
5433                     next;
5434                 } elsif ($state->{status} eq 'completed') {
5435                     print "State saved, quitting\n";
5436                     last;
5437                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5438                     die "query-savevm failed with error '$state->{error}'\n"
5439                 } else {
5440                     die "query-savevm returned status '$state->{status}'\n";
5441                 }
5442             }
5443         };
5444         my $err = $@;
5445
5446         PVE::QemuConfig->lock_config($vmid, sub {
5447             $conf = PVE::QemuConfig->load_config($vmid);
5448             if ($err) {
5449                 # cleanup, but leave suspending lock, to indicate something went wrong
5450                 eval {
5451                     mon_cmd($vmid, "savevm-end");
5452                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5453                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5454                     delete $conf->@{qw(vmstate runningmachine)};
5455                     PVE::QemuConfig->write_config($vmid, $conf);
5456                 };
5457                 warn $@ if $@;
5458                 die $err;
5459             }
5460
5461             die "lock changed unexpectedly\n"
5462                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5463
5464             mon_cmd($vmid, "quit");
5465             $conf->{lock} = 'suspended';
5466             PVE::QemuConfig->write_config($vmid, $conf);
5467         });
5468     }
5469 }
5470
5471 sub vm_resume {
5472     my ($vmid, $skiplock, $nocheck) = @_;
5473
5474     PVE::QemuConfig->lock_config($vmid, sub {
5475         my $res = mon_cmd($vmid, 'query-status');
5476         my $resume_cmd = 'cont';
5477
5478         if ($res->{status} && $res->{status} eq 'suspended') {
5479             $resume_cmd = 'system_wakeup';
5480         }
5481
5482         if (!$nocheck) {
5483
5484             my $conf = PVE::QemuConfig->load_config($vmid);
5485
5486             PVE::QemuConfig->check_lock($conf)
5487                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5488         }
5489
5490         mon_cmd($vmid, $resume_cmd);
5491     });
5492 }
5493
5494 sub vm_sendkey {
5495     my ($vmid, $skiplock, $key) = @_;
5496
5497     PVE::QemuConfig->lock_config($vmid, sub {
5498
5499         my $conf = PVE::QemuConfig->load_config($vmid);
5500
5501         # there is no qmp command, so we use the human monitor command
5502         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5503         die $res if $res ne '';
5504     });
5505 }
5506
5507 # vzdump restore implementaion
5508
5509 sub tar_archive_read_firstfile {
5510     my $archive = shift;
5511
5512     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5513
5514     # try to detect archive type first
5515     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5516         die "unable to open file '$archive'\n";
5517     my $firstfile = <$fh>;
5518     kill 15, $pid;
5519     close $fh;
5520
5521     die "ERROR: archive contaions no data\n" if !$firstfile;
5522     chomp $firstfile;
5523
5524     return $firstfile;
5525 }
5526
5527 sub tar_restore_cleanup {
5528     my ($storecfg, $statfile) = @_;
5529
5530     print STDERR "starting cleanup\n";
5531
5532     if (my $fd = IO::File->new($statfile, "r")) {
5533         while (defined(my $line = <$fd>)) {
5534             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5535                 my $volid = $2;
5536                 eval {
5537                     if ($volid =~ m|^/|) {
5538                         unlink $volid || die 'unlink failed\n';
5539                     } else {
5540                         PVE::Storage::vdisk_free($storecfg, $volid);
5541                     }
5542                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5543                 };
5544                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5545             } else {
5546                 print STDERR "unable to parse line in statfile - $line";
5547             }
5548         }
5549         $fd->close();
5550     }
5551 }
5552
5553 sub restore_file_archive {
5554     my ($archive, $vmid, $user, $opts) = @_;
5555
5556     my $format = $opts->{format};
5557     my $comp;
5558
5559     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5560         $format = 'tar' if !$format;
5561         $comp = 'gzip';
5562     } elsif ($archive =~ m/\.tar$/) {
5563         $format = 'tar' if !$format;
5564     } elsif ($archive =~ m/.tar.lzo$/) {
5565         $format = 'tar' if !$format;
5566         $comp = 'lzop';
5567     } elsif ($archive =~ m/\.vma$/) {
5568         $format = 'vma' if !$format;
5569     } elsif ($archive =~ m/\.vma\.gz$/) {
5570         $format = 'vma' if !$format;
5571         $comp = 'gzip';
5572     } elsif ($archive =~ m/\.vma\.lzo$/) {
5573         $format = 'vma' if !$format;
5574         $comp = 'lzop';
5575     } else {
5576         $format = 'vma' if !$format; # default
5577     }
5578
5579     # try to detect archive format
5580     if ($format eq 'tar') {
5581         return restore_tar_archive($archive, $vmid, $user, $opts);
5582     } else {
5583         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5584     }
5585 }
5586
5587 # hepler to remove disks that will not be used after restore
5588 my $restore_cleanup_oldconf = sub {
5589     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5590
5591     foreach_drive($oldconf, sub {
5592         my ($ds, $drive) = @_;
5593
5594         return if drive_is_cdrom($drive, 1);
5595
5596         my $volid = $drive->{file};
5597         return if !$volid || $volid =~ m|^/|;
5598
5599         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5600         return if !$path || !$owner || ($owner != $vmid);
5601
5602         # Note: only delete disk we want to restore
5603         # other volumes will become unused
5604         if ($virtdev_hash->{$ds}) {
5605             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5606             if (my $err = $@) {
5607                 warn $err;
5608             }
5609         }
5610     });
5611
5612     # delete vmstate files, after the restore we have no snapshots anymore
5613     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5614         my $snap = $oldconf->{snapshots}->{$snapname};
5615         if ($snap->{vmstate}) {
5616             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5617             if (my $err = $@) {
5618                 warn $err;
5619             }
5620         }
5621     }
5622 };
5623
5624 # Helper to parse vzdump backup device hints
5625 #
5626 # $rpcenv: Environment, used to ckeck storage permissions
5627 # $user: User ID, to check storage permissions
5628 # $storecfg: Storage configuration
5629 # $fh: the file handle for reading the configuration
5630 # $devinfo: should contain device sizes for all backu-up'ed devices
5631 # $options: backup options (pool, default storage)
5632 #
5633 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5634 my $parse_backup_hints = sub {
5635     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5636
5637     my $virtdev_hash = {};
5638
5639     while (defined(my $line = <$fh>)) {
5640         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5641             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5642             die "archive does not contain data for drive '$virtdev'\n"
5643                 if !$devinfo->{$devname};
5644
5645             if (defined($options->{storage})) {
5646                 $storeid = $options->{storage} || 'local';
5647             } elsif (!$storeid) {
5648                 $storeid = 'local';
5649             }
5650             $format = 'raw' if !$format;
5651             $devinfo->{$devname}->{devname} = $devname;
5652             $devinfo->{$devname}->{virtdev} = $virtdev;
5653             $devinfo->{$devname}->{format} = $format;
5654             $devinfo->{$devname}->{storeid} = $storeid;
5655
5656             # check permission on storage
5657             my $pool = $options->{pool}; # todo: do we need that?
5658             if ($user ne 'root@pam') {
5659                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5660             }
5661
5662             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5663         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5664             my $virtdev = $1;
5665             my $drive = parse_drive($virtdev, $2);
5666             if (drive_is_cloudinit($drive)) {
5667                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5668                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5669                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5670
5671                 $virtdev_hash->{$virtdev} = {
5672                     format => $format,
5673                     storeid => $options->{storage} // $storeid,
5674                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5675                     is_cloudinit => 1,
5676                 };
5677             }
5678         }
5679     }
5680
5681     return $virtdev_hash;
5682 };
5683
5684 # Helper to allocate and activate all volumes required for a restore
5685 #
5686 # $storecfg: Storage configuration
5687 # $virtdev_hash: as returned by parse_backup_hints()
5688 #
5689 # Returns: { $virtdev => $volid }
5690 my $restore_allocate_devices = sub {
5691     my ($storecfg, $virtdev_hash, $vmid) = @_;
5692
5693     my $map = {};
5694     foreach my $virtdev (sort keys %$virtdev_hash) {
5695         my $d = $virtdev_hash->{$virtdev};
5696         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5697         my $storeid = $d->{storeid};
5698         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5699
5700         # test if requested format is supported
5701         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5702         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5703         $d->{format} = $defFormat if !$supported;
5704
5705         my $name;
5706         if ($d->{is_cloudinit}) {
5707             $name = "vm-$vmid-cloudinit";
5708             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5709         }
5710
5711         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5712
5713         print STDERR "new volume ID is '$volid'\n";
5714         $d->{volid} = $volid;
5715
5716         PVE::Storage::activate_volumes($storecfg, [$volid]);
5717
5718         $map->{$virtdev} = $volid;
5719     }
5720
5721     return $map;
5722 };
5723
5724 my $restore_update_config_line = sub {
5725     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5726
5727     return if $line =~ m/^\#qmdump\#/;
5728     return if $line =~ m/^\#vzdump\#/;
5729     return if $line =~ m/^lock:/;
5730     return if $line =~ m/^unused\d+:/;
5731     return if $line =~ m/^parent:/;
5732
5733     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5734     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5735         # try to convert old 1.X settings
5736         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5737         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5738             my ($model, $macaddr) = split(/\=/, $devconfig);
5739             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5740             my $net = {
5741                 model => $model,
5742                 bridge => "vmbr$ind",
5743                 macaddr => $macaddr,
5744             };
5745             my $netstr = print_net($net);
5746
5747             print $outfd "net$cookie->{netcount}: $netstr\n";
5748             $cookie->{netcount}++;
5749         }
5750     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5751         my ($id, $netstr) = ($1, $2);
5752         my $net = parse_net($netstr);
5753         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5754         $netstr = print_net($net);
5755         print $outfd "$id: $netstr\n";
5756     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5757         my $virtdev = $1;
5758         my $value = $3;
5759         my $di = parse_drive($virtdev, $value);
5760         if (defined($di->{backup}) && !$di->{backup}) {
5761             print $outfd "#$line";
5762         } elsif ($map->{$virtdev}) {
5763             delete $di->{format}; # format can change on restore
5764             $di->{file} = $map->{$virtdev};
5765             $value = print_drive($di);
5766             print $outfd "$virtdev: $value\n";
5767         } else {
5768             print $outfd $line;
5769         }
5770     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5771         my $vmgenid = $1;
5772         if ($vmgenid ne '0') {
5773             # always generate a new vmgenid if there was a valid one setup
5774             $vmgenid = generate_uuid();
5775         }
5776         print $outfd "vmgenid: $vmgenid\n";
5777     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5778         my ($uuid, $uuid_str);
5779         UUID::generate($uuid);
5780         UUID::unparse($uuid, $uuid_str);
5781         my $smbios1 = parse_smbios1($2);
5782         $smbios1->{uuid} = $uuid_str;
5783         print $outfd $1.print_smbios1($smbios1)."\n";
5784     } else {
5785         print $outfd $line;
5786     }
5787 };
5788
5789 my $restore_deactivate_volumes = sub {
5790     my ($storecfg, $devinfo) = @_;
5791
5792     my $vollist = [];
5793     foreach my $devname (keys %$devinfo) {
5794         my $volid = $devinfo->{$devname}->{volid};
5795         push @$vollist, $volid if $volid;
5796     }
5797
5798     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5799 };
5800
5801 my $restore_destroy_volumes = sub {
5802     my ($storecfg, $devinfo) = @_;
5803
5804     foreach my $devname (keys %$devinfo) {
5805         my $volid = $devinfo->{$devname}->{volid};
5806         next if !$volid;
5807         eval {
5808             if ($volid =~ m|^/|) {
5809                 unlink $volid || die 'unlink failed\n';
5810             } else {
5811                 PVE::Storage::vdisk_free($storecfg, $volid);
5812             }
5813             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5814         };
5815         print STDERR "unable to cleanup '$volid' - $@" if $@;
5816     }
5817 };
5818
5819 sub scan_volids {
5820     my ($cfg, $vmid) = @_;
5821
5822     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5823
5824     my $volid_hash = {};
5825     foreach my $storeid (keys %$info) {
5826         foreach my $item (@{$info->{$storeid}}) {
5827             next if !($item->{volid} && $item->{size});
5828             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5829             $volid_hash->{$item->{volid}} = $item;
5830         }
5831     }
5832
5833     return $volid_hash;
5834 }
5835
5836 sub update_disk_config {
5837     my ($vmid, $conf, $volid_hash) = @_;
5838
5839     my $changes;
5840     my $prefix = "VM $vmid:";
5841
5842     # used and unused disks
5843     my $referenced = {};
5844
5845     # Note: it is allowed to define multiple storages with same path (alias), so
5846     # we need to check both 'volid' and real 'path' (two different volid can point
5847     # to the same path).
5848
5849     my $referencedpath = {};
5850
5851     # update size info
5852     PVE::QemuConfig->foreach_volume($conf, sub {
5853         my ($opt, $drive) = @_;
5854
5855         my $volid = $drive->{file};
5856         return if !$volid;
5857
5858         # mark volid as "in-use" for next step
5859         $referenced->{$volid} = 1;
5860         if ($volid_hash->{$volid} &&
5861             (my $path = $volid_hash->{$volid}->{path})) {
5862             $referencedpath->{$path} = 1;
5863         }
5864
5865         return if drive_is_cdrom($drive);
5866         return if !$volid_hash->{$volid};
5867
5868         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5869         if (defined($updated)) {
5870             $changes = 1;
5871             $conf->{$opt} = print_drive($updated);
5872             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5873         }
5874     });
5875
5876     # remove 'unusedX' entry if volume is used
5877     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5878         my ($opt, $drive) = @_;
5879
5880         my $volid = $drive->{file};
5881         return if !$volid;
5882
5883         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5884         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5885             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5886             $changes = 1;
5887             delete $conf->{$opt};
5888         }
5889
5890         $referenced->{$volid} = 1;
5891         $referencedpath->{$path} = 1 if $path;
5892     });
5893
5894     foreach my $volid (sort keys %$volid_hash) {
5895         next if $volid =~ m/vm-$vmid-state-/;
5896         next if $referenced->{$volid};
5897         my $path = $volid_hash->{$volid}->{path};
5898         next if !$path; # just to be sure
5899         next if $referencedpath->{$path};
5900         $changes = 1;
5901         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5902         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5903         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5904     }
5905
5906     return $changes;
5907 }
5908
5909 sub rescan {
5910     my ($vmid, $nolock, $dryrun) = @_;
5911
5912     my $cfg = PVE::Storage::config();
5913
5914     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5915     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5916     foreach my $stor (keys %{$cfg->{ids}}) {
5917         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5918     }
5919
5920     print "rescan volumes...\n";
5921     my $volid_hash = scan_volids($cfg, $vmid);
5922
5923     my $updatefn =  sub {
5924         my ($vmid) = @_;
5925
5926         my $conf = PVE::QemuConfig->load_config($vmid);
5927
5928         PVE::QemuConfig->check_lock($conf);
5929
5930         my $vm_volids = {};
5931         foreach my $volid (keys %$volid_hash) {
5932             my $info = $volid_hash->{$volid};
5933             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5934         }
5935
5936         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5937
5938         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5939     };
5940
5941     if (defined($vmid)) {
5942         if ($nolock) {
5943             &$updatefn($vmid);
5944         } else {
5945             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5946         }
5947     } else {
5948         my $vmlist = config_list();
5949         foreach my $vmid (keys %$vmlist) {
5950             if ($nolock) {
5951                 &$updatefn($vmid);
5952             } else {
5953                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5954             }
5955         }
5956     }
5957 }
5958
5959 sub restore_proxmox_backup_archive {
5960     my ($archive, $vmid, $user, $options) = @_;
5961
5962     my $storecfg = PVE::Storage::config();
5963
5964     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5965     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5966
5967     my $server = $scfg->{server};
5968     my $datastore = $scfg->{datastore};
5969     my $username = $scfg->{username} // 'root@pam';
5970     my $fingerprint = $scfg->{fingerprint};
5971
5972     my $repo = "$username\@$server:$datastore";
5973     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5974     local $ENV{PBS_PASSWORD} = $password;
5975     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5976
5977     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5978         PVE::Storage::parse_volname($storecfg, $archive);
5979
5980     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5981
5982     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5983
5984     my $tmpdir = "/var/tmp/vzdumptmp$$";
5985     rmtree $tmpdir;
5986     mkpath $tmpdir;
5987
5988     my $conffile = PVE::QemuConfig->config_file($vmid);
5989     my $tmpfn = "$conffile.$$.tmp";
5990      # disable interrupts (always do cleanups)
5991     local $SIG{INT} =
5992         local $SIG{TERM} =
5993         local $SIG{QUIT} =
5994         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5995
5996     # Note: $oldconf is undef if VM does not exists
5997     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5998     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5999
6000     my $rpcenv = PVE::RPCEnvironment::get();
6001     my $devinfo = {};
6002
6003     eval {
6004         # enable interrupts
6005         local $SIG{INT} =
6006             local $SIG{TERM} =
6007             local $SIG{QUIT} =
6008             local $SIG{HUP} =
6009             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6010
6011         my $cfgfn = "$tmpdir/qemu-server.conf";
6012         my $firewall_config_fn = "$tmpdir/fw.conf";
6013         my $index_fn = "$tmpdir/index.json";
6014
6015         my $cmd = "restore";
6016
6017         my $param = [$pbs_backup_name, "index.json", $index_fn];
6018         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6019         my $index = PVE::Tools::file_get_contents($index_fn);
6020         $index = decode_json($index);
6021
6022         # print Dumper($index);
6023         foreach my $info (@{$index->{files}}) {
6024             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6025                 my $devname = $1;
6026                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6027                     $devinfo->{$devname}->{size} = $1;
6028                 } else {
6029                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6030                 }
6031             }
6032         }
6033
6034         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6035         if (!$is_qemu_server_backup) {
6036             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6037         }
6038         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6039
6040         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6041         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6042
6043         if ($has_firewall_config) {
6044             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6045             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6046
6047             my $pve_firewall_dir = '/etc/pve/firewall';
6048             mkdir $pve_firewall_dir; # make sure the dir exists
6049             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6050         }
6051
6052         my $fh = IO::File->new($cfgfn, "r") ||
6053             "unable to read qemu-server.conf - $!\n";
6054
6055         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6056
6057         # fixme: rate limit?
6058
6059         # create empty/temp config
6060         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6061
6062         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6063
6064         # allocate volumes
6065         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6066
6067         foreach my $virtdev (sort keys %$virtdev_hash) {
6068             my $d = $virtdev_hash->{$virtdev};
6069             next if $d->{is_cloudinit}; # no need to restore cloudinit
6070
6071             my $volid = $d->{volid};
6072
6073             my $path = PVE::Storage::path($storecfg, $volid);
6074
6075             my $pbs_restore_cmd = [
6076                 '/usr/bin/pbs-restore',
6077                 '--repository', $repo,
6078                 $pbs_backup_name,
6079                 "$d->{devname}.img.fidx",
6080                 $path,
6081                 '--verbose',
6082                 ];
6083
6084             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6085                 push @$pbs_restore_cmd, '--skip-zero';
6086             }
6087
6088             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6089             print "restore proxmox backup image: $dbg_cmdstring\n";
6090             run_command($pbs_restore_cmd);
6091         }
6092
6093         $fh->seek(0, 0) || die "seek failed - $!\n";
6094
6095         my $outfd = new IO::File ($tmpfn, "w") ||
6096             die "unable to write config for VM $vmid\n";
6097
6098         my $cookie = { netcount => 0 };
6099         while (defined(my $line = <$fh>)) {
6100             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6101         }
6102
6103         $fh->close();
6104         $outfd->close();
6105     };
6106     my $err = $@;
6107
6108     $restore_deactivate_volumes->($storecfg, $devinfo);
6109
6110     rmtree $tmpdir;
6111
6112     if ($err) {
6113         unlink $tmpfn;
6114         $restore_destroy_volumes->($storecfg, $devinfo);
6115         die $err;
6116     }
6117
6118     rename($tmpfn, $conffile) ||
6119         die "unable to commit configuration file '$conffile'\n";
6120
6121     PVE::Cluster::cfs_update(); # make sure we read new file
6122
6123     eval { rescan($vmid, 1); };
6124     warn $@ if $@;
6125 }
6126
6127 sub restore_vma_archive {
6128     my ($archive, $vmid, $user, $opts, $comp) = @_;
6129
6130     my $readfrom = $archive;
6131
6132     my $cfg = PVE::Storage::config();
6133     my $commands = [];
6134     my $bwlimit = $opts->{bwlimit};
6135
6136     my $dbg_cmdstring = '';
6137     my $add_pipe = sub {
6138         my ($cmd) = @_;
6139         push @$commands, $cmd;
6140         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6141         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6142         $readfrom = '-';
6143     };
6144
6145     my $input = undef;
6146     if ($archive eq '-') {
6147         $input = '<&STDIN';
6148     } else {
6149         # If we use a backup from a PVE defined storage we also consider that
6150         # storage's rate limit:
6151         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6152         if (defined($volid)) {
6153             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6154             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6155             if ($readlimit) {
6156                 print STDERR "applying read rate limit: $readlimit\n";
6157                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6158                 $add_pipe->($cstream);
6159             }
6160         }
6161     }
6162
6163     if ($comp) {
6164         my $cmd;
6165         if ($comp eq 'gzip') {
6166             $cmd = ['zcat', $readfrom];
6167         } elsif ($comp eq 'lzop') {
6168             $cmd = ['lzop', '-d', '-c', $readfrom];
6169         } else {
6170             die "unknown compression method '$comp'\n";
6171         }
6172         $add_pipe->($cmd);
6173     }
6174
6175     my $tmpdir = "/var/tmp/vzdumptmp$$";
6176     rmtree $tmpdir;
6177
6178     # disable interrupts (always do cleanups)
6179     local $SIG{INT} =
6180         local $SIG{TERM} =
6181         local $SIG{QUIT} =
6182         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6183
6184     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6185     POSIX::mkfifo($mapfifo, 0600);
6186     my $fifofh;
6187
6188     my $openfifo = sub {
6189         open($fifofh, '>', $mapfifo) || die $!;
6190     };
6191
6192     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6193
6194     my $oldtimeout;
6195     my $timeout = 5;
6196
6197     my $devinfo = {};
6198
6199     my $rpcenv = PVE::RPCEnvironment::get();
6200
6201     my $conffile = PVE::QemuConfig->config_file($vmid);
6202     my $tmpfn = "$conffile.$$.tmp";
6203
6204     # Note: $oldconf is undef if VM does not exist
6205     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6206     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6207
6208     my %storage_limits;
6209
6210     my $print_devmap = sub {
6211         my $cfgfn = "$tmpdir/qemu-server.conf";
6212
6213         # we can read the config - that is already extracted
6214         my $fh = IO::File->new($cfgfn, "r") ||
6215             "unable to read qemu-server.conf - $!\n";
6216
6217         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6218         if (-f $fwcfgfn) {
6219             my $pve_firewall_dir = '/etc/pve/firewall';
6220             mkdir $pve_firewall_dir; # make sure the dir exists
6221             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6222         }
6223
6224         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6225
6226         foreach my $key (keys %storage_limits) {
6227             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6228             next if !$limit;
6229             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6230             $storage_limits{$key} = $limit * 1024;
6231         }
6232
6233         foreach my $devname (keys %$devinfo) {
6234             die "found no device mapping information for device '$devname'\n"
6235                 if !$devinfo->{$devname}->{virtdev};
6236         }
6237
6238         # create empty/temp config
6239         if ($oldconf) {
6240             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6241             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6242         }
6243
6244         # allocate volumes
6245         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6246
6247         # print restore information to $fifofh
6248         foreach my $virtdev (sort keys %$virtdev_hash) {
6249             my $d = $virtdev_hash->{$virtdev};
6250             next if $d->{is_cloudinit}; # no need to restore cloudinit
6251
6252             my $storeid = $d->{storeid};
6253             my $volid = $d->{volid};
6254
6255             my $map_opts = '';
6256             if (my $limit = $storage_limits{$storeid}) {
6257                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6258             }
6259
6260             my $write_zeros = 1;
6261             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6262                 $write_zeros = 0;
6263             }
6264
6265             my $path = PVE::Storage::path($cfg, $volid);
6266
6267             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6268
6269             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6270         }
6271
6272         $fh->seek(0, 0) || die "seek failed - $!\n";
6273
6274         my $outfd = new IO::File ($tmpfn, "w") ||
6275             die "unable to write config for VM $vmid\n";
6276
6277         my $cookie = { netcount => 0 };
6278         while (defined(my $line = <$fh>)) {
6279             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6280         }
6281
6282         $fh->close();
6283         $outfd->close();
6284     };
6285
6286     eval {
6287         # enable interrupts
6288         local $SIG{INT} =
6289             local $SIG{TERM} =
6290             local $SIG{QUIT} =
6291             local $SIG{HUP} =
6292             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6293         local $SIG{ALRM} = sub { die "got timeout\n"; };
6294
6295         $oldtimeout = alarm($timeout);
6296
6297         my $parser = sub {
6298             my $line = shift;
6299
6300             print "$line\n";
6301
6302             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6303                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6304                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6305             } elsif ($line =~ m/^CTIME: /) {
6306                 # we correctly received the vma config, so we can disable
6307                 # the timeout now for disk allocation (set to 10 minutes, so
6308                 # that we always timeout if something goes wrong)
6309                 alarm(600);
6310                 &$print_devmap();
6311                 print $fifofh "done\n";
6312                 my $tmp = $oldtimeout || 0;
6313                 $oldtimeout = undef;
6314                 alarm($tmp);
6315                 close($fifofh);
6316             }
6317         };
6318
6319         print "restore vma archive: $dbg_cmdstring\n";
6320         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6321     };
6322     my $err = $@;
6323
6324     alarm($oldtimeout) if $oldtimeout;
6325
6326     $restore_deactivate_volumes->($cfg, $devinfo);
6327
6328     unlink $mapfifo;
6329     rmtree $tmpdir;
6330
6331     if ($err) {
6332         unlink $tmpfn;
6333         $restore_destroy_volumes->($cfg, $devinfo);
6334         die $err;
6335     }
6336
6337     rename($tmpfn, $conffile) ||
6338         die "unable to commit configuration file '$conffile'\n";
6339
6340     PVE::Cluster::cfs_update(); # make sure we read new file
6341
6342     eval { rescan($vmid, 1); };
6343     warn $@ if $@;
6344 }
6345
6346 sub restore_tar_archive {
6347     my ($archive, $vmid, $user, $opts) = @_;
6348
6349     if ($archive ne '-') {
6350         my $firstfile = tar_archive_read_firstfile($archive);
6351         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6352             if $firstfile ne 'qemu-server.conf';
6353     }
6354
6355     my $storecfg = PVE::Storage::config();
6356
6357     # avoid zombie disks when restoring over an existing VM -> cleanup first
6358     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6359     # skiplock=1 because qmrestore has set the 'create' lock itself already
6360     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6361     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6362
6363     my $tocmd = "/usr/lib/qemu-server/qmextract";
6364
6365     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6366     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6367     $tocmd .= ' --prealloc' if $opts->{prealloc};
6368     $tocmd .= ' --info' if $opts->{info};
6369
6370     # tar option "xf" does not autodetect compression when read from STDIN,
6371     # so we pipe to zcat
6372     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6373         PVE::Tools::shellquote("--to-command=$tocmd");
6374
6375     my $tmpdir = "/var/tmp/vzdumptmp$$";
6376     mkpath $tmpdir;
6377
6378     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6379     local $ENV{VZDUMP_VMID} = $vmid;
6380     local $ENV{VZDUMP_USER} = $user;
6381
6382     my $conffile = PVE::QemuConfig->config_file($vmid);
6383     my $tmpfn = "$conffile.$$.tmp";
6384
6385     # disable interrupts (always do cleanups)
6386     local $SIG{INT} =
6387         local $SIG{TERM} =
6388         local $SIG{QUIT} =
6389         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6390
6391     eval {
6392         # enable interrupts
6393         local $SIG{INT} =
6394             local $SIG{TERM} =
6395             local $SIG{QUIT} =
6396             local $SIG{HUP} =
6397             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6398
6399         if ($archive eq '-') {
6400             print "extracting archive from STDIN\n";
6401             run_command($cmd, input => "<&STDIN");
6402         } else {
6403             print "extracting archive '$archive'\n";
6404             run_command($cmd);
6405         }
6406
6407         return if $opts->{info};
6408
6409         # read new mapping
6410         my $map = {};
6411         my $statfile = "$tmpdir/qmrestore.stat";
6412         if (my $fd = IO::File->new($statfile, "r")) {
6413             while (defined (my $line = <$fd>)) {
6414                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6415                     $map->{$1} = $2 if $1;
6416                 } else {
6417                     print STDERR "unable to parse line in statfile - $line\n";
6418                 }
6419             }
6420             $fd->close();
6421         }
6422
6423         my $confsrc = "$tmpdir/qemu-server.conf";
6424
6425         my $srcfd = new IO::File($confsrc, "r") ||
6426             die "unable to open file '$confsrc'\n";
6427
6428         my $outfd = new IO::File ($tmpfn, "w") ||
6429             die "unable to write config for VM $vmid\n";
6430
6431         my $cookie = { netcount => 0 };
6432         while (defined (my $line = <$srcfd>)) {
6433             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6434         }
6435
6436         $srcfd->close();
6437         $outfd->close();
6438     };
6439     if (my $err = $@) {
6440         unlink $tmpfn;
6441         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6442         die $err;
6443     }
6444
6445     rmtree $tmpdir;
6446
6447     rename $tmpfn, $conffile ||
6448         die "unable to commit configuration file '$conffile'\n";
6449
6450     PVE::Cluster::cfs_update(); # make sure we read new file
6451
6452     eval { rescan($vmid, 1); };
6453     warn $@ if $@;
6454 };
6455
6456 sub foreach_storage_used_by_vm {
6457     my ($conf, $func) = @_;
6458
6459     my $sidhash = {};
6460
6461     foreach_drive($conf, sub {
6462         my ($ds, $drive) = @_;
6463         return if drive_is_cdrom($drive);
6464
6465         my $volid = $drive->{file};
6466
6467         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6468         $sidhash->{$sid} = $sid if $sid;
6469     });
6470
6471     foreach my $sid (sort keys %$sidhash) {
6472         &$func($sid);
6473     }
6474 }
6475
6476 my $qemu_snap_storage = {
6477     rbd => 1,
6478 };
6479 sub do_snapshots_with_qemu {
6480     my ($storecfg, $volid) = @_;
6481
6482     my $storage_name = PVE::Storage::parse_volume_id($volid);
6483     my $scfg = $storecfg->{ids}->{$storage_name};
6484
6485     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6486         return 1;
6487     }
6488
6489     if ($volid =~ m/\.(qcow2|qed)$/){
6490         return 1;
6491     }
6492
6493     return undef;
6494 }
6495
6496 sub qga_check_running {
6497     my ($vmid, $nowarn) = @_;
6498
6499     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6500     if ($@) {
6501         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6502         return 0;
6503     }
6504     return 1;
6505 }
6506
6507 sub template_create {
6508     my ($vmid, $conf, $disk) = @_;
6509
6510     my $storecfg = PVE::Storage::config();
6511
6512     foreach_drive($conf, sub {
6513         my ($ds, $drive) = @_;
6514
6515         return if drive_is_cdrom($drive);
6516         return if $disk && $ds ne $disk;
6517
6518         my $volid = $drive->{file};
6519         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6520
6521         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6522         $drive->{file} = $voliddst;
6523         $conf->{$ds} = print_drive($drive);
6524         PVE::QemuConfig->write_config($vmid, $conf);
6525     });
6526 }
6527
6528 sub convert_iscsi_path {
6529     my ($path) = @_;
6530
6531     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6532         my $portal = $1;
6533         my $target = $2;
6534         my $lun = $3;
6535
6536         my $initiator_name = get_initiator_name();
6537
6538         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6539                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6540     }
6541
6542     die "cannot convert iscsi path '$path', unkown format\n";
6543 }
6544
6545 sub qemu_img_convert {
6546     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6547
6548     my $storecfg = PVE::Storage::config();
6549     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6550     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6551
6552     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6553
6554     my $cachemode;
6555     my $src_path;
6556     my $src_is_iscsi = 0;
6557     my $src_format;
6558
6559     if ($src_storeid) {
6560         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6561         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6562         $src_format = qemu_img_format($src_scfg, $src_volname);
6563         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6564         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6565         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6566     } elsif (-f $src_volid) {
6567         $src_path = $src_volid;
6568         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6569             $src_format = $1;
6570         }
6571     }
6572
6573     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6574
6575     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6576     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6577     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6578     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6579
6580     my $cmd = [];
6581     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6582     push @$cmd, '-l', "snapshot.name=$snapname"
6583         if $snapname && $src_format && $src_format eq "qcow2";
6584     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6585     push @$cmd, '-T', $cachemode if defined($cachemode);
6586
6587     if ($src_is_iscsi) {
6588         push @$cmd, '--image-opts';
6589         $src_path = convert_iscsi_path($src_path);
6590     } elsif ($src_format) {
6591         push @$cmd, '-f', $src_format;
6592     }
6593
6594     if ($dst_is_iscsi) {
6595         push @$cmd, '--target-image-opts';
6596         $dst_path = convert_iscsi_path($dst_path);
6597     } else {
6598         push @$cmd, '-O', $dst_format;
6599     }
6600
6601     push @$cmd, $src_path;
6602
6603     if (!$dst_is_iscsi && $is_zero_initialized) {
6604         push @$cmd, "zeroinit:$dst_path";
6605     } else {
6606         push @$cmd, $dst_path;
6607     }
6608
6609     my $parser = sub {
6610         my $line = shift;
6611         if($line =~ m/\((\S+)\/100\%\)/){
6612             my $percent = $1;
6613             my $transferred = int($size * $percent / 100);
6614             my $remaining = $size - $transferred;
6615
6616             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6617         }
6618
6619     };
6620
6621     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6622     my $err = $@;
6623     die "copy failed: $err" if $err;
6624 }
6625
6626 sub qemu_img_format {
6627     my ($scfg, $volname) = @_;
6628
6629     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6630         return $1;
6631     } else {
6632         return "raw";
6633     }
6634 }
6635
6636 sub qemu_drive_mirror {
6637     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6638
6639     $jobs = {} if !$jobs;
6640
6641     my $qemu_target;
6642     my $format;
6643     $jobs->{"drive-$drive"} = {};
6644
6645     if ($dst_volid =~ /^nbd:/) {
6646         $qemu_target = $dst_volid;
6647         $format = "nbd";
6648     } else {
6649         my $storecfg = PVE::Storage::config();
6650         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6651
6652         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6653
6654         $format = qemu_img_format($dst_scfg, $dst_volname);
6655
6656         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6657
6658         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6659     }
6660
6661     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6662     $opts->{format} = $format if $format;
6663
6664     if (defined($src_bitmap)) {
6665         $opts->{sync} = 'incremental';
6666         $opts->{bitmap} = $src_bitmap;
6667         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6668     }
6669
6670     if (defined($bwlimit)) {
6671         $opts->{speed} = $bwlimit * 1024;
6672         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6673     } else {
6674         print "drive mirror is starting for drive-$drive\n";
6675     }
6676
6677     # if a job already runs for this device we get an error, catch it for cleanup
6678     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6679     if (my $err = $@) {
6680         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6681         warn "$@\n" if $@;
6682         die "mirroring error: $err\n";
6683     }
6684
6685     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6686 }
6687
6688 # $completion can be either
6689 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6690 # 'cancel': wait until all jobs are ready, block-job-cancel them
6691 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6692 sub qemu_drive_mirror_monitor {
6693     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6694
6695     $completion //= 'complete';
6696
6697     eval {
6698         my $err_complete = 0;
6699
6700         while (1) {
6701             die "storage migration timed out\n" if $err_complete > 300;
6702
6703             my $stats = mon_cmd($vmid, "query-block-jobs");
6704
6705             my $running_mirror_jobs = {};
6706             foreach my $stat (@$stats) {
6707                 next if $stat->{type} ne 'mirror';
6708                 $running_mirror_jobs->{$stat->{device}} = $stat;
6709             }
6710
6711             my $readycounter = 0;
6712
6713             foreach my $job (keys %$jobs) {
6714
6715                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6716                     print "$job : finished\n";
6717                     delete $jobs->{$job};
6718                     next;
6719                 }
6720
6721                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6722
6723                 my $busy = $running_mirror_jobs->{$job}->{busy};
6724                 my $ready = $running_mirror_jobs->{$job}->{ready};
6725                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6726                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6727                     my $remaining = $total - $transferred;
6728                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6729
6730                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6731                 }
6732
6733                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6734             }
6735
6736             last if scalar(keys %$jobs) == 0;
6737
6738             if ($readycounter == scalar(keys %$jobs)) {
6739                 print "all mirroring jobs are ready \n";
6740                 last if $completion eq 'skip'; #do the complete later
6741
6742                 if ($vmiddst && $vmiddst != $vmid) {
6743                     my $agent_running = $qga && qga_check_running($vmid);
6744                     if ($agent_running) {
6745                         print "freeze filesystem\n";
6746                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6747                     } else {
6748                         print "suspend vm\n";
6749                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6750                     }
6751
6752                     # if we clone a disk for a new target vm, we don't switch the disk
6753                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6754
6755                     if ($agent_running) {
6756                         print "unfreeze filesystem\n";
6757                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6758                     } else {
6759                         print "resume vm\n";
6760                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6761                     }
6762
6763                     last;
6764                 } else {
6765
6766                     foreach my $job (keys %$jobs) {
6767                         # try to switch the disk if source and destination are on the same guest
6768                         print "$job: Completing block job...\n";
6769
6770                         my $op;
6771                         if ($completion eq 'complete') {
6772                             $op = 'block-job-complete';
6773                         } elsif ($completion eq 'cancel') {
6774                             $op = 'block-job-cancel';
6775                         } else {
6776                             die "invalid completion value: $completion\n";
6777                         }
6778                         eval { mon_cmd($vmid, $op, device => $job) };
6779                         if ($@ =~ m/cannot be completed/) {
6780                             print "$job: Block job cannot be completed, try again.\n";
6781                             $err_complete++;
6782                         }else {
6783                             print "$job: Completed successfully.\n";
6784                             $jobs->{$job}->{complete} = 1;
6785                         }
6786                     }
6787                 }
6788             }
6789             sleep 1;
6790         }
6791     };
6792     my $err = $@;
6793
6794     if ($err) {
6795         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6796         die "mirroring error: $err";
6797     }
6798
6799 }
6800
6801 sub qemu_blockjobs_cancel {
6802     my ($vmid, $jobs) = @_;
6803
6804     foreach my $job (keys %$jobs) {
6805         print "$job: Cancelling block job\n";
6806         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6807         $jobs->{$job}->{cancel} = 1;
6808     }
6809
6810     while (1) {
6811         my $stats = mon_cmd($vmid, "query-block-jobs");
6812
6813         my $running_jobs = {};
6814         foreach my $stat (@$stats) {
6815             $running_jobs->{$stat->{device}} = $stat;
6816         }
6817
6818         foreach my $job (keys %$jobs) {
6819
6820             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6821                 print "$job: Done.\n";
6822                 delete $jobs->{$job};
6823             }
6824         }
6825
6826         last if scalar(keys %$jobs) == 0;
6827
6828         sleep 1;
6829     }
6830 }
6831
6832 sub clone_disk {
6833     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6834         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6835
6836     my $newvolid;
6837
6838     if (!$full) {
6839         print "create linked clone of drive $drivename ($drive->{file})\n";
6840         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6841         push @$newvollist, $newvolid;
6842     } else {
6843
6844         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6845         $storeid = $storage if $storage;
6846
6847         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6848         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6849
6850         print "create full clone of drive $drivename ($drive->{file})\n";
6851         my $name = undef;
6852         if (drive_is_cloudinit($drive)) {
6853             $name = "vm-$newvmid-cloudinit";
6854             $name .= ".$dst_format" if $dst_format ne 'raw';
6855             $snapname = undef;
6856             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6857         } elsif ($drivename eq 'efidisk0') {
6858             $size = get_efivars_size($conf);
6859         }
6860         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6861         push @$newvollist, $newvolid;
6862
6863         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6864
6865         if (drive_is_cloudinit($drive)) {
6866             goto no_data_clone;
6867         }
6868
6869         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6870         if (!$running || $snapname) {
6871             # TODO: handle bwlimits
6872             if ($drivename eq 'efidisk0') {
6873                 # the relevant data on the efidisk may be smaller than the source
6874                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6875                 # that is given by the OVMF_VARS.fd
6876                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6877                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6878                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6879             } else {
6880                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6881             }
6882         } else {
6883
6884             my $kvmver = get_running_qemu_version ($vmid);
6885             if (!min_version($kvmver, 2, 7)) {
6886                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6887                     if $drive->{iothread};
6888             }
6889
6890             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6891         }
6892     }
6893
6894 no_data_clone:
6895     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6896
6897     my $disk = $drive;
6898     $disk->{format} = undef;
6899     $disk->{file} = $newvolid;
6900     $disk->{size} = $size;
6901
6902     return $disk;
6903 }
6904
6905 sub get_running_qemu_version {
6906     my ($vmid) = @_;
6907     my $res = mon_cmd($vmid, "query-version");
6908     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6909 }
6910
6911 sub qemu_use_old_bios_files {
6912     my ($machine_type) = @_;
6913
6914     return if !$machine_type;
6915
6916     my $use_old_bios_files = undef;
6917
6918     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6919         $machine_type = $1;
6920         $use_old_bios_files = 1;
6921     } else {
6922         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6923         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6924         # load new efi bios files on migration. So this hack is required to allow
6925         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6926         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6927         $use_old_bios_files = !min_version($version, 2, 4);
6928     }
6929
6930     return ($use_old_bios_files, $machine_type);
6931 }
6932
6933 sub get_efivars_size {
6934     my ($conf) = @_;
6935     my $arch = get_vm_arch($conf);
6936     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6937     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6938     return -s $ovmf_vars;
6939 }
6940
6941 sub update_efidisk_size {
6942     my ($conf) = @_;
6943
6944     return if !defined($conf->{efidisk0});
6945
6946     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6947     $disk->{size} = get_efivars_size($conf);
6948     $conf->{efidisk0} = print_drive($disk);
6949
6950     return;
6951 }
6952
6953 sub create_efidisk($$$$$) {
6954     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6955
6956     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6957     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6958
6959     my $vars_size_b = -s $ovmf_vars;
6960     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6961     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6962     PVE::Storage::activate_volumes($storecfg, [$volid]);
6963
6964     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6965     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6966
6967     return ($volid, $size/1024);
6968 }
6969
6970 sub vm_iothreads_list {
6971     my ($vmid) = @_;
6972
6973     my $res = mon_cmd($vmid, 'query-iothreads');
6974
6975     my $iothreads = {};
6976     foreach my $iothread (@$res) {
6977         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6978     }
6979
6980     return $iothreads;
6981 }
6982
6983 sub scsihw_infos {
6984     my ($conf, $drive) = @_;
6985
6986     my $maxdev = 0;
6987
6988     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6989         $maxdev = 7;
6990     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6991         $maxdev = 1;
6992     } else {
6993         $maxdev = 256;
6994     }
6995
6996     my $controller = int($drive->{index} / $maxdev);
6997     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6998
6999     return ($maxdev, $controller, $controller_prefix);
7000 }
7001
7002 sub windows_version {
7003     my ($ostype) = @_;
7004
7005     return 0 if !$ostype;
7006
7007     my $winversion = 0;
7008
7009     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7010         $winversion = 5;
7011     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7012         $winversion = 6;
7013     } elsif ($ostype =~ m/^win(\d+)$/) {
7014         $winversion = $1;
7015     }
7016
7017     return $winversion;
7018 }
7019
7020 sub resolve_dst_disk_format {
7021         my ($storecfg, $storeid, $src_volname, $format) = @_;
7022         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7023
7024         if (!$format) {
7025             # if no target format is specified, use the source disk format as hint
7026             if ($src_volname) {
7027                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7028                 $format = qemu_img_format($scfg, $src_volname);
7029             } else {
7030                 return $defFormat;
7031             }
7032         }
7033
7034         # test if requested format is supported - else use default
7035         my $supported = grep { $_ eq $format } @$validFormats;
7036         $format = $defFormat if !$supported;
7037         return $format;
7038 }
7039
7040 # NOTE: if this logic changes, please update docs & possibly gui logic
7041 sub find_vmstate_storage {
7042     my ($conf, $storecfg) = @_;
7043
7044     # first, return storage from conf if set
7045     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7046
7047     my ($target, $shared, $local);
7048
7049     foreach_storage_used_by_vm($conf, sub {
7050         my ($sid) = @_;
7051         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7052         my $dst = $scfg->{shared} ? \$shared : \$local;
7053         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7054     });
7055
7056     # second, use shared storage where VM has at least one disk
7057     # third, use local storage where VM has at least one disk
7058     # fall back to local storage
7059     $target = $shared // $local // 'local';
7060
7061     return $target;
7062 }
7063
7064 sub generate_uuid {
7065     my ($uuid, $uuid_str);
7066     UUID::generate($uuid);
7067     UUID::unparse($uuid, $uuid_str);
7068     return $uuid_str;
7069 }
7070
7071 sub generate_smbios1_uuid {
7072     return "uuid=".generate_uuid();
7073 }
7074
7075 sub nbd_stop {
7076     my ($vmid) = @_;
7077
7078     mon_cmd($vmid, 'nbd-server-stop');
7079 }
7080
7081 sub create_reboot_request {
7082     my ($vmid) = @_;
7083     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7084         or die "failed to create reboot trigger file: $!\n";
7085     close($fh);
7086 }
7087
7088 sub clear_reboot_request {
7089     my ($vmid) = @_;
7090     my $path = "/run/qemu-server/$vmid.reboot";
7091     my $res = 0;
7092
7093     $res = unlink($path);
7094     die "could not remove reboot request for $vmid: $!"
7095         if !$res && $! != POSIX::ENOENT;
7096
7097     return $res;
7098 }
7099
7100 # bash completion helper
7101
7102 sub complete_backup_archives {
7103     my ($cmdname, $pname, $cvalue) = @_;
7104
7105     my $cfg = PVE::Storage::config();
7106
7107     my $storeid;
7108
7109     if ($cvalue =~ m/^([^:]+):/) {
7110         $storeid = $1;
7111     }
7112
7113     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7114
7115     my $res = [];
7116     foreach my $id (keys %$data) {
7117         foreach my $item (@{$data->{$id}}) {
7118             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7119             push @$res, $item->{volid} if defined($item->{volid});
7120         }
7121     }
7122
7123     return $res;
7124 }
7125
7126 my $complete_vmid_full = sub {
7127     my ($running) = @_;
7128
7129     my $idlist = vmstatus();
7130
7131     my $res = [];
7132
7133     foreach my $id (keys %$idlist) {
7134         my $d = $idlist->{$id};
7135         if (defined($running)) {
7136             next if $d->{template};
7137             next if $running && $d->{status} ne 'running';
7138             next if !$running && $d->{status} eq 'running';
7139         }
7140         push @$res, $id;
7141
7142     }
7143     return $res;
7144 };
7145
7146 sub complete_vmid {
7147     return &$complete_vmid_full();
7148 }
7149
7150 sub complete_vmid_stopped {
7151     return &$complete_vmid_full(0);
7152 }
7153
7154 sub complete_vmid_running {
7155     return &$complete_vmid_full(1);
7156 }
7157
7158 sub complete_storage {
7159
7160     my $cfg = PVE::Storage::config();
7161     my $ids = $cfg->{ids};
7162
7163     my $res = [];
7164     foreach my $sid (keys %$ids) {
7165         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7166         next if !$ids->{$sid}->{content}->{images};
7167         push @$res, $sid;
7168     }
7169
7170     return $res;
7171 }
7172
7173 sub complete_migration_storage {
7174     my ($cmd, $param, $current_value, $all_args) = @_;
7175
7176     my $targetnode = @$all_args[1];
7177
7178     my $cfg = PVE::Storage::config();
7179     my $ids = $cfg->{ids};
7180
7181     my $res = [];
7182     foreach my $sid (keys %$ids) {
7183         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7184         next if !$ids->{$sid}->{content}->{images};
7185         push @$res, $sid;
7186     }
7187
7188     return $res;
7189 }
7190
7191 1;