PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_HOSTPCI_DEVICES = 16;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         pattern => '[-_.\w\d]+',
 850         optional => 1,
 851     },
 852     queues => {
 853         type => 'integer',
 854         minimum => 0, maximum => 16,
 855         description => 'Number of packet queues to be used on the device.',
 856         optional => 1,
 857     },
 858     rate => {
 859         type => 'number',
 860         minimum => 0,
 861         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 862         optional => 1,
 863     },
 864     tag => {
 865         type => 'integer',
 866         minimum => 1, maximum => 4094,
 867         description => 'VLAN tag to apply to packets on this interface.',
 868         optional => 1,
 869     },
 870     trunks => {
 871         type => 'string',
 872         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 873         description => 'VLAN trunks to pass through this interface.',
 874         format_description => 'vlanid[;vlanid...]',
 875         optional => 1,
 876     },
 877     firewall => {
 878         type => 'boolean',
 879         description => 'Whether this interface should be protected by the firewall.',
 880         optional => 1,
 881     },
 882     link_down => {
 883         type => 'boolean',
 884         description => 'Whether this interface should be disconnected (like pulling the plug).',
 885         optional => 1,
 886     },
 887 };
 888
 889 my $netdesc = {
 890     optional => 1,
 891     type => 'string', format => $net_fmt,
 892     description => "Specify network devices.",
 893 };
 894
 895 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 896
 897 my $ipconfig_fmt = {
 898     ip => {
 899         type => 'string',
 900         format => 'pve-ipv4-config',
 901         format_description => 'IPv4Format/CIDR',
 902         description => 'IPv4 address in CIDR format.',
 903         optional => 1,
 904         default => 'dhcp',
 905     },
 906     gw => {
 907         type => 'string',
 908         format => 'ipv4',
 909         format_description => 'GatewayIPv4',
 910         description => 'Default gateway for IPv4 traffic.',
 911         optional => 1,
 912         requires => 'ip',
 913     },
 914     ip6 => {
 915         type => 'string',
 916         format => 'pve-ipv6-config',
 917         format_description => 'IPv6Format/CIDR',
 918         description => 'IPv6 address in CIDR format.',
 919         optional => 1,
 920         default => 'dhcp',
 921     },
 922     gw6 => {
 923         type => 'string',
 924         format => 'ipv6',
 925         format_description => 'GatewayIPv6',
 926         description => 'Default gateway for IPv6 traffic.',
 927         optional => 1,
 928         requires => 'ip6',
 929     },
 930 };
 931 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 932 my $ipconfigdesc = {
 933     optional => 1,
 934     type => 'string', format => 'pve-qm-ipconfig',
 935     description => <<'EODESCR',
 936 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 937
 938 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 939
 940 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 941 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 942
 943 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 944 EODESCR
 945 };
 946 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 947
 948 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 949     $confdesc->{"net$i"} = $netdesc;
 950     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 951 }
 952
 953 foreach my $key (keys %$confdesc_cloudinit) {
 954     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 955 }
 956
 957 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 958 sub verify_volume_id_or_qm_path {
 959     my ($volid, $noerr) = @_;
 960
 961     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 962         return $volid;
 963     }
 964
 965     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 966     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 967     if ($@) {
 968         return undef if $noerr;
 969         die $@;
 970     }
 971     return $volid;
 972 }
 973
 974 my $usb_fmt = {
 975     host => {
 976         default_key => 1,
 977         type => 'string', format => 'pve-qm-usb-device',
 978         format_description => 'HOSTUSBDEVICE|spice',
 979         description => <<EODESCR,
 980 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 981
 982  'bus-port(.port)*' (decimal numbers) or
 983  'vendor_id:product_id' (hexadeciaml numbers) or
 984  'spice'
 985
 986 You can use the 'lsusb -t' command to list existing usb devices.
 987
 988 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 989
 990 The value 'spice' can be used to add a usb redirection devices for spice.
 991 EODESCR
 992     },
 993     usb3 => {
 994         optional => 1,
 995         type => 'boolean',
 996         description => "Specifies whether if given host option is a USB3 device or port.",
 997         default => 0,
 998     },
 999 };
1000
1001 my $usbdesc = {
1002     optional => 1,
1003     type => 'string', format => $usb_fmt,
1004     description => "Configure an USB device (n is 0 to 4).",
1005 };
1006 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1007
1008 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1009 my $hostpci_fmt = {
1010     host => {
1011         default_key => 1,
1012         type => 'string',
1013         pattern => qr/$PCIRE(;$PCIRE)*/,
1014         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1015         description => <<EODESCR,
1016 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1017 of PCI virtual functions of the host. HOSTPCIID syntax is:
1018
1019 'bus:dev.func' (hexadecimal numbers)
1020
1021 You can us the 'lspci' command to list existing PCI devices.
1022 EODESCR
1023     },
1024     rombar => {
1025         type => 'boolean',
1026         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1027         optional => 1,
1028         default => 1,
1029     },
1030     romfile => {
1031         type => 'string',
1032         pattern => '[^,;]+',
1033         format_description => 'string',
1034         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1035         optional => 1,
1036     },
1037     pcie => {
1038         type => 'boolean',
1039         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1040         optional => 1,
1041         default => 0,
1042     },
1043     'x-vga' => {
1044         type => 'boolean',
1045         description =>  "Enable vfio-vga device support.",
1046         optional => 1,
1047         default => 0,
1048     },
1049     'mdev' => {
1050         type => 'string',
1051         format_description => 'string',
1052         pattern => '[^/\.:]+',
1053         optional => 1,
1054         description => <<EODESCR
1055 The type of mediated device to use.
1056 An instance of this type will be created on startup of the VM and
1057 will be cleaned up when the VM stops.
1058 EODESCR
1059     }
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1121     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1122 }
1123
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1125     $confdesc->{"usb$i"} = $usbdesc;
1126 }
1127
1128 my $kvm_api_version = 0;
1129
1130 sub kvm_version {
1131     return $kvm_api_version if $kvm_api_version;
1132
1133     open my $fh, '<', '/dev/kvm'
1134         or return undef;
1135
1136     # 0xae00 => KVM_GET_API_VERSION
1137     $kvm_api_version = ioctl($fh, 0xae00, 0);
1138
1139     return $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version = {};
1143 my $kvm_mtime = {};
1144
1145 sub kvm_user_version {
1146     my ($binary) = @_;
1147
1148     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1149     my $st = stat($binary);
1150
1151     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1152     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1153         $cachedmtime == $st->mtime;
1154
1155     $kvm_user_version->{$binary} = 'unknown';
1156     $kvm_mtime->{$binary} = $st->mtime;
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version->{$binary} = $2;
1162         }
1163     };
1164
1165     eval { run_command([$binary, '--version'], outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version->{$binary};
1169
1170 }
1171
1172 sub kernel_has_vhost_net {
1173     return -c '/dev/vhost-net';
1174 }
1175
1176 sub option_exists {
1177     my $key = shift;
1178     return defined($confdesc->{$key});
1179 }
1180
1181 my $cdrom_path;
1182 sub get_cdrom_path {
1183
1184     return  $cdrom_path if $cdrom_path;
1185
1186     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1187     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1188     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1189 }
1190
1191 sub get_iso_path {
1192     my ($storecfg, $vmid, $cdrom) = @_;
1193
1194     if ($cdrom eq 'cdrom') {
1195         return get_cdrom_path();
1196     } elsif ($cdrom eq 'none') {
1197         return '';
1198     } elsif ($cdrom =~ m|^/|) {
1199         return $cdrom;
1200     } else {
1201         return PVE::Storage::path($storecfg, $cdrom);
1202     }
1203 }
1204
1205 # try to convert old style file names to volume IDs
1206 sub filename_to_volume_id {
1207     my ($vmid, $file, $media) = @_;
1208
1209      if (!($file eq 'none' || $file eq 'cdrom' ||
1210           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1211
1212         return undef if $file =~ m|/|;
1213
1214         if ($media && $media eq 'cdrom') {
1215             $file = "local:iso/$file";
1216         } else {
1217             $file = "local:$vmid/$file";
1218         }
1219     }
1220
1221     return $file;
1222 }
1223
1224 sub verify_media_type {
1225     my ($opt, $vtype, $media) = @_;
1226
1227     return if !$media;
1228
1229     my $etype;
1230     if ($media eq 'disk') {
1231         $etype = 'images';
1232     } elsif ($media eq 'cdrom') {
1233         $etype = 'iso';
1234     } else {
1235         die "internal error";
1236     }
1237
1238     return if ($vtype eq $etype);
1239
1240     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1241 }
1242
1243 sub cleanup_drive_path {
1244     my ($opt, $storecfg, $drive) = @_;
1245
1246     # try to convert filesystem paths to volume IDs
1247
1248     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1249         ($drive->{file} !~ m|^/dev/.+|) &&
1250         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1251         ($drive->{file} !~ m/^\d+$/)) {
1252         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1253         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1254         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1255         verify_media_type($opt, $vtype, $drive->{media});
1256         $drive->{file} = $volid;
1257     }
1258
1259     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1260 }
1261
1262 sub parse_hotplug_features {
1263     my ($data) = @_;
1264
1265     my $res = {};
1266
1267     return $res if $data eq '0';
1268
1269     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1270
1271     foreach my $feature (PVE::Tools::split_list($data)) {
1272         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1273             $res->{$1} = 1;
1274         } else {
1275             die "invalid hotplug feature '$feature'\n";
1276         }
1277     }
1278     return $res;
1279 }
1280
1281 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1282 sub pve_verify_hotplug_features {
1283     my ($value, $noerr) = @_;
1284
1285     return $value if parse_hotplug_features($value);
1286
1287     return undef if $noerr;
1288
1289     die "unable to parse hotplug option\n";
1290 }
1291
1292 sub scsi_inquiry {
1293     my($fh, $noerr) = @_;
1294
1295     my $SG_IO = 0x2285;
1296     my $SG_GET_VERSION_NUM = 0x2282;
1297
1298     my $versionbuf = "\x00" x 8;
1299     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1300     if (!$ret) {
1301         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1302         return undef;
1303     }
1304     my $version = unpack("I", $versionbuf);
1305     if ($version < 30000) {
1306         die "scsi generic interface too old\n"  if !$noerr;
1307         return undef;
1308     }
1309
1310     my $buf = "\x00" x 36;
1311     my $sensebuf = "\x00" x 8;
1312     my $cmd = pack("C x3 C x1", 0x12, 36);
1313
1314     # see /usr/include/scsi/sg.h
1315     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1316
1317     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1318                       length($sensebuf), 0, length($buf), $buf,
1319                       $cmd, $sensebuf, 6000);
1320
1321     $ret = ioctl($fh, $SG_IO, $packet);
1322     if (!$ret) {
1323         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1324         return undef;
1325     }
1326
1327     my @res = unpack($sg_io_hdr_t, $packet);
1328     if ($res[17] || $res[18]) {
1329         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1330         return undef;
1331     }
1332
1333     my $res = {};
1334     (my $byte0, my $byte1, $res->{vendor},
1335      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1336
1337     $res->{removable} = $byte1 & 128 ? 1 : 0;
1338     $res->{type} = $byte0 & 31;
1339
1340     return $res;
1341 }
1342
1343 sub path_is_scsi {
1344     my ($path) = @_;
1345
1346     my $fh = IO::File->new("+<$path") || return undef;
1347     my $res = scsi_inquiry($fh, 1);
1348     close($fh);
1349
1350     return $res;
1351 }
1352
1353 sub print_tabletdevice_full {
1354     my ($conf, $arch) = @_;
1355
1356     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1357
1358     # we use uhci for old VMs because tablet driver was buggy in older qemu
1359     my $usbbus;
1360     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1361         $usbbus = 'ehci';
1362     } else {
1363         $usbbus = 'uhci';
1364     }
1365
1366     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1367 }
1368
1369 sub print_keyboarddevice_full {
1370     my ($conf, $arch, $machine) = @_;
1371
1372     return undef if $arch ne 'aarch64';
1373
1374     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1375 }
1376
1377 sub print_drivedevice_full {
1378     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1379
1380     my $device = '';
1381     my $maxdev = 0;
1382
1383     if ($drive->{interface} eq 'virtio') {
1384         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1385         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1386         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1387     } elsif ($drive->{interface} eq 'scsi') {
1388
1389         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1390         my $unit = $drive->{index} % $maxdev;
1391         my $devicetype = 'hd';
1392         my $path = '';
1393         if (drive_is_cdrom($drive)) {
1394             $devicetype = 'cd';
1395         } else {
1396             if ($drive->{file} =~ m|^/|) {
1397                 $path = $drive->{file};
1398                 if (my $info = path_is_scsi($path)) {
1399                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1400                         $devicetype = 'block';
1401                     } elsif ($info->{type} == 1) { # tape
1402                         $devicetype = 'generic';
1403                     }
1404                 }
1405             } else {
1406                  $path = PVE::Storage::path($storecfg, $drive->{file});
1407             }
1408
1409             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1410             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1411             if ($path =~ m/^iscsi\:\/\// &&
1412                !min_version($version, 4, 1)) {
1413                 $devicetype = 'generic';
1414             }
1415         }
1416
1417         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1421         }
1422
1423         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1424             $device .= ",rotation_rate=1";
1425         }
1426         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1427
1428     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1429         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1430         my $controller = int($drive->{index} / $maxdev);
1431         my $unit = $drive->{index} % $maxdev;
1432         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1433
1434         $device = "ide-$devicetype";
1435         if ($drive->{interface} eq 'ide') {
1436             $device .= ",bus=ide.$controller,unit=$unit";
1437         } else {
1438             $device .= ",bus=ahci$controller.$unit";
1439         }
1440         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1441
1442         if ($devicetype eq 'hd') {
1443             if (my $model = $drive->{model}) {
1444                 $model = URI::Escape::uri_unescape($model);
1445                 $device .= ",model=$model";
1446             }
1447             if ($drive->{ssd}) {
1448                 $device .= ",rotation_rate=1";
1449             }
1450         }
1451         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1452     } elsif ($drive->{interface} eq 'usb') {
1453         die "implement me";
1454         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1455     } else {
1456         die "unsupported interface type";
1457     }
1458
1459     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1460
1461     if (my $serial = $drive->{serial}) {
1462         $serial = URI::Escape::uri_unescape($serial);
1463         $device .= ",serial=$serial";
1464     }
1465
1466
1467     return $device;
1468 }
1469
1470 sub get_initiator_name {
1471     my $initiator;
1472
1473     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1474     while (defined(my $line = <$fh>)) {
1475         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1476         $initiator = $1;
1477         last;
1478     }
1479     $fh->close();
1480
1481     return $initiator;
1482 }
1483
1484 sub print_drive_commandline_full {
1485     my ($storecfg, $vmid, $drive) = @_;
1486
1487     my $path;
1488     my $volid = $drive->{file};
1489     my $format;
1490
1491     if (drive_is_cdrom($drive)) {
1492         $path = get_iso_path($storecfg, $vmid, $volid);
1493     } else {
1494         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1495         if ($storeid) {
1496             $path = PVE::Storage::path($storecfg, $volid);
1497             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1498             $format = qemu_img_format($scfg, $volname);
1499         } else {
1500             $path = $volid;
1501             $format = "raw";
1502         }
1503    }
1504
1505     my $opts = '';
1506     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1507     foreach my $o (@qemu_drive_options) {
1508         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1509     }
1510
1511     # snapshot only accepts on|off
1512     if (defined($drive->{snapshot})) {
1513         my $v = $drive->{snapshot} ? 'on' : 'off';
1514         $opts .= ",snapshot=$v";
1515     }
1516
1517     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1518         my ($dir, $qmpname) = @$type;
1519         if (my $v = $drive->{"mbps$dir"}) {
1520             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1521         }
1522         if (my $v = $drive->{"mbps${dir}_max"}) {
1523             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1524         }
1525         if (my $v = $drive->{"bps${dir}_max_length"}) {
1526             $opts .= ",throttling.bps$qmpname-max-length=$v";
1527         }
1528         if (my $v = $drive->{"iops${dir}"}) {
1529             $opts .= ",throttling.iops$qmpname=$v";
1530         }
1531         if (my $v = $drive->{"iops${dir}_max"}) {
1532             $opts .= ",throttling.iops$qmpname-max=$v";
1533         }
1534         if (my $v = $drive->{"iops${dir}_max_length"}) {
1535             $opts .= ",throttling.iops$qmpname-max-length=$v";
1536         }
1537     }
1538
1539     $opts .= ",format=$format" if $format && !$drive->{format};
1540
1541     my $cache_direct = 0;
1542
1543     if (my $cache = $drive->{cache}) {
1544         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1545     } elsif (!drive_is_cdrom($drive)) {
1546         $opts .= ",cache=none";
1547         $cache_direct = 1;
1548     }
1549
1550     # aio native works only with O_DIRECT
1551     if (!$drive->{aio}) {
1552         if($cache_direct) {
1553             $opts .= ",aio=native";
1554         } else {
1555             $opts .= ",aio=threads";
1556         }
1557     }
1558
1559     if (!drive_is_cdrom($drive)) {
1560         my $detectzeroes;
1561         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1562             $detectzeroes = 'off';
1563         } elsif ($drive->{discard}) {
1564             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1565         } else {
1566             # This used to be our default with discard not being specified:
1567             $detectzeroes = 'on';
1568         }
1569         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1570     }
1571
1572     my $pathinfo = $path ? "file=$path," : '';
1573
1574     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1575 }
1576
1577 sub print_netdevice_full {
1578     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1579
1580     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1581
1582     my $device = $net->{model};
1583     if ($net->{model} eq 'virtio') {
1584          $device = 'virtio-net-pci';
1585      };
1586
1587     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1588     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1589     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1590         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1591         my $vectors = $net->{queues} * 2 + 2;
1592         $tmpstr .= ",vectors=$vectors,mq=on";
1593     }
1594     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1595
1596     if ($use_old_bios_files) {
1597         my $romfile;
1598         if ($device eq 'virtio-net-pci') {
1599             $romfile = 'pxe-virtio.rom';
1600         } elsif ($device eq 'e1000') {
1601             $romfile = 'pxe-e1000.rom';
1602         } elsif ($device eq 'ne2k') {
1603             $romfile = 'pxe-ne2k_pci.rom';
1604         } elsif ($device eq 'pcnet') {
1605             $romfile = 'pxe-pcnet.rom';
1606         } elsif ($device eq 'rtl8139') {
1607             $romfile = 'pxe-rtl8139.rom';
1608         }
1609         $tmpstr .= ",romfile=$romfile" if $romfile;
1610     }
1611
1612     return $tmpstr;
1613 }
1614
1615 sub print_netdev_full {
1616     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1617
1618     my $i = '';
1619     if ($netid =~ m/^net(\d+)$/) {
1620         $i = int($1);
1621     }
1622
1623     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1624
1625     my $ifname = "tap${vmid}i$i";
1626
1627     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1628     die "interface name '$ifname' is too long (max 15 character)\n"
1629         if length($ifname) >= 16;
1630
1631     my $vhostparam = '';
1632     if (is_native($arch)) {
1633         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1634     }
1635
1636     my $vmname = $conf->{name} || "vm$vmid";
1637
1638     my $netdev = "";
1639     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1640
1641     if ($net->{bridge}) {
1642         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1643     } else {
1644         $netdev = "type=user,id=$netid,hostname=$vmname";
1645     }
1646
1647     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1648
1649     return $netdev;
1650 }
1651
1652 my $vga_map = {
1653     'cirrus' => 'cirrus-vga',
1654     'std' => 'VGA',
1655     'vmware' => 'vmware-svga',
1656     'virtio' => 'virtio-vga',
1657 };
1658
1659 sub print_vga_device {
1660     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1661
1662     my $type = $vga_map->{$vga->{type}};
1663     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1664         $type = 'virtio-gpu';
1665     }
1666     my $vgamem_mb = $vga->{memory};
1667
1668     my $max_outputs = '';
1669     if ($qxlnum) {
1670         $type = $id ? 'qxl' : 'qxl-vga';
1671
1672         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1673             # set max outputs so linux can have up to 4 qxl displays with one device
1674             if (min_version($machine_version, 4, 1)) {
1675                 $max_outputs = ",max_outputs=4";
1676             }
1677         }
1678     }
1679
1680     die "no devicetype for $vga->{type}\n" if !$type;
1681
1682     my $memory = "";
1683     if ($vgamem_mb) {
1684         if ($vga->{type} eq 'virtio') {
1685             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1686             $memory = ",max_hostmem=$bytes";
1687         } elsif ($qxlnum) {
1688             # from https://www.spice-space.org/multiple-monitors.html
1689             $memory = ",vgamem_mb=$vga->{memory}";
1690             my $ram = $vgamem_mb * 4;
1691             my $vram = $vgamem_mb * 2;
1692             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1693         } else {
1694             $memory = ",vgamem_mb=$vga->{memory}";
1695         }
1696     } elsif ($qxlnum && $id) {
1697         $memory = ",ram_size=67108864,vram_size=33554432";
1698     }
1699
1700     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1701     my $vgaid = "vga" . ($id // '');
1702     my $pciaddr;
1703
1704     if ($q35 && $vgaid eq 'vga') {
1705         # the first display uses pcie.0 bus on q35 machines
1706         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1707     } else {
1708         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1709     }
1710
1711     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1712 }
1713
1714 sub parse_number_sets {
1715     my ($set) = @_;
1716     my $res = [];
1717     foreach my $part (split(/;/, $set)) {
1718         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1719             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1720             push @$res, [ $1, $2 ];
1721         } else {
1722             die "invalid range: $part\n";
1723         }
1724     }
1725     return $res;
1726 }
1727
1728 sub parse_numa {
1729     my ($data) = @_;
1730
1731     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1732     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1733     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1734     return $res;
1735 }
1736
1737 sub parse_hostpci {
1738     my ($value) = @_;
1739
1740     return undef if !$value;
1741
1742     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1743
1744     my @idlist = split(/;/, $res->{host});
1745     delete $res->{host};
1746     foreach my $id (@idlist) {
1747         my $devs = PVE::SysFSTools::lspci($id);
1748         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1749         push @{$res->{pciid}}, @$devs;
1750     }
1751     return $res;
1752 }
1753
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1755 sub parse_net {
1756     my ($data) = @_;
1757
1758     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1759     if ($@) {
1760         warn $@;
1761         return undef;
1762     }
1763     if (!defined($res->{macaddr})) {
1764         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1765         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1766     }
1767     return $res;
1768 }
1769
1770 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1771 sub parse_ipconfig {
1772     my ($data) = @_;
1773
1774     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1775     if ($@) {
1776         warn $@;
1777         return undef;
1778     }
1779
1780     if ($res->{gw} && !$res->{ip}) {
1781         warn 'gateway specified without specifying an IP address';
1782         return undef;
1783     }
1784     if ($res->{gw6} && !$res->{ip6}) {
1785         warn 'IPv6 gateway specified without specifying an IPv6 address';
1786         return undef;
1787     }
1788     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1789         warn 'gateway specified together with DHCP';
1790         return undef;
1791     }
1792     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1793         # gw6 + auto/dhcp
1794         warn "IPv6 gateway specified together with $res->{ip6} address";
1795         return undef;
1796     }
1797
1798     if (!$res->{ip} && !$res->{ip6}) {
1799         return { ip => 'dhcp', ip6 => 'dhcp' };
1800     }
1801
1802     return $res;
1803 }
1804
1805 sub print_net {
1806     my $net = shift;
1807
1808     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1809 }
1810
1811 sub add_random_macs {
1812     my ($settings) = @_;
1813
1814     foreach my $opt (keys %$settings) {
1815         next if $opt !~ m/^net(\d+)$/;
1816         my $net = parse_net($settings->{$opt});
1817         next if !$net;
1818         $settings->{$opt} = print_net($net);
1819     }
1820 }
1821
1822 sub vm_is_volid_owner {
1823     my ($storecfg, $vmid, $volid) = @_;
1824
1825     if ($volid !~  m|^/|) {
1826         my ($path, $owner);
1827         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1828         if ($owner && ($owner == $vmid)) {
1829             return 1;
1830         }
1831     }
1832
1833     return undef;
1834 }
1835
1836 sub vmconfig_register_unused_drive {
1837     my ($storecfg, $vmid, $conf, $drive) = @_;
1838
1839     if (drive_is_cloudinit($drive)) {
1840         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1841         warn $@ if $@;
1842     } elsif (!drive_is_cdrom($drive)) {
1843         my $volid = $drive->{file};
1844         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1845             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1846         }
1847     }
1848 }
1849
1850 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1851 my $smbios1_fmt = {
1852     uuid => {
1853         type => 'string',
1854         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1855         format_description => 'UUID',
1856         description => "Set SMBIOS1 UUID.",
1857         optional => 1,
1858     },
1859     version => {
1860         type => 'string',
1861         pattern => '[A-Za-z0-9+\/]+={0,2}',
1862         format_description => 'Base64 encoded string',
1863         description => "Set SMBIOS1 version.",
1864         optional => 1,
1865     },
1866     serial => {
1867         type => 'string',
1868         pattern => '[A-Za-z0-9+\/]+={0,2}',
1869         format_description => 'Base64 encoded string',
1870         description => "Set SMBIOS1 serial number.",
1871         optional => 1,
1872     },
1873     manufacturer => {
1874         type => 'string',
1875         pattern => '[A-Za-z0-9+\/]+={0,2}',
1876         format_description => 'Base64 encoded string',
1877         description => "Set SMBIOS1 manufacturer.",
1878         optional => 1,
1879     },
1880     product => {
1881         type => 'string',
1882         pattern => '[A-Za-z0-9+\/]+={0,2}',
1883         format_description => 'Base64 encoded string',
1884         description => "Set SMBIOS1 product ID.",
1885         optional => 1,
1886     },
1887     sku => {
1888         type => 'string',
1889         pattern => '[A-Za-z0-9+\/]+={0,2}',
1890         format_description => 'Base64 encoded string',
1891         description => "Set SMBIOS1 SKU string.",
1892         optional => 1,
1893     },
1894     family => {
1895         type => 'string',
1896         pattern => '[A-Za-z0-9+\/]+={0,2}',
1897         format_description => 'Base64 encoded string',
1898         description => "Set SMBIOS1 family string.",
1899         optional => 1,
1900     },
1901     base64 => {
1902         type => 'boolean',
1903         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1904         optional => 1,
1905     },
1906 };
1907
1908 sub parse_smbios1 {
1909     my ($data) = @_;
1910
1911     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1912     warn $@ if $@;
1913     return $res;
1914 }
1915
1916 sub print_smbios1 {
1917     my ($smbios1) = @_;
1918     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1919 }
1920
1921 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1922
1923 sub parse_watchdog {
1924     my ($value) = @_;
1925
1926     return undef if !$value;
1927
1928     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1929     warn $@ if $@;
1930     return $res;
1931 }
1932
1933 sub parse_guest_agent {
1934     my ($value) = @_;
1935
1936     return {} if !defined($value->{agent});
1937
1938     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1939     warn $@ if $@;
1940
1941     # if the agent is disabled ignore the other potentially set properties
1942     return {} if !$res->{enabled};
1943     return $res;
1944 }
1945
1946 sub parse_vga {
1947     my ($value) = @_;
1948
1949     return {} if !$value;
1950     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1951     warn $@ if $@;
1952     return $res;
1953 }
1954
1955 sub parse_rng {
1956     my ($value) = @_;
1957
1958     return undef if !$value;
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1961     warn $@ if $@;
1962     return $res;
1963 }
1964
1965 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1966 sub verify_usb_device {
1967     my ($value, $noerr) = @_;
1968
1969     return $value if parse_usb_device($value);
1970
1971     return undef if $noerr;
1972
1973     die "unable to parse usb device\n";
1974 }
1975
1976 # add JSON properties for create and set function
1977 sub json_config_properties {
1978     my $prop = shift;
1979
1980     foreach my $opt (keys %$confdesc) {
1981         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1982             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1983         $prop->{$opt} = $confdesc->{$opt};
1984     }
1985
1986     return $prop;
1987 }
1988
1989 # return copy of $confdesc_cloudinit to generate documentation
1990 sub cloudinit_config_properties {
1991
1992     return dclone($confdesc_cloudinit);
1993 }
1994
1995 sub check_type {
1996     my ($key, $value) = @_;
1997
1998     die "unknown setting '$key'\n" if !$confdesc->{$key};
1999
2000     my $type = $confdesc->{$key}->{type};
2001
2002     if (!defined($value)) {
2003         die "got undefined value\n";
2004     }
2005
2006     if ($value =~ m/[\n\r]/) {
2007         die "property contains a line feed\n";
2008     }
2009
2010     if ($type eq 'boolean') {
2011         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2012         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2013         die "type check ('boolean') failed - got '$value'\n";
2014     } elsif ($type eq 'integer') {
2015         return int($1) if $value =~ m/^(\d+)$/;
2016         die "type check ('integer') failed - got '$value'\n";
2017     } elsif ($type eq 'number') {
2018         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2019         die "type check ('number') failed - got '$value'\n";
2020     } elsif ($type eq 'string') {
2021         if (my $fmt = $confdesc->{$key}->{format}) {
2022             PVE::JSONSchema::check_format($fmt, $value);
2023             return $value;
2024         }
2025         $value =~ s/^\"(.*)\"$/$1/;
2026         return $value;
2027     } else {
2028         die "internal error"
2029     }
2030 }
2031
2032 sub destroy_vm {
2033     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2034
2035     my $conf = PVE::QemuConfig->load_config($vmid);
2036
2037     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2038
2039     if ($conf->{template}) {
2040         # check if any base image is still used by a linked clone
2041         PVE::QemuConfig->foreach_volume($conf, sub {
2042                 my ($ds, $drive) = @_;
2043                 return if drive_is_cdrom($drive);
2044
2045                 my $volid = $drive->{file};
2046                 return if !$volid || $volid =~ m|^/|;
2047
2048                 die "base volume '$volid' is still in use by linked cloned\n"
2049                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2050
2051         });
2052     }
2053
2054     # only remove disks owned by this VM
2055     PVE::QemuConfig->foreach_volume($conf, sub {
2056         my ($ds, $drive) = @_;
2057         return if drive_is_cdrom($drive, 1);
2058
2059         my $volid = $drive->{file};
2060         return if !$volid || $volid =~ m|^/|;
2061
2062         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2063         return if !$path || !$owner || ($owner != $vmid);
2064
2065         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2066         warn "Could not remove disk '$volid', check manually: $@" if $@;
2067     });
2068
2069     # also remove unused disk
2070     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2071     PVE::Storage::foreach_volid($vmdisks, sub {
2072         my ($volid, $sid, $volname, $d) = @_;
2073         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2074         warn $@ if $@;
2075     });
2076
2077     if (defined $replacement_conf) {
2078         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2079     } else {
2080         PVE::QemuConfig->destroy_config($vmid);
2081     }
2082 }
2083
2084 sub parse_vm_config {
2085     my ($filename, $raw) = @_;
2086
2087     return undef if !defined($raw);
2088
2089     my $res = {
2090         digest => Digest::SHA::sha1_hex($raw),
2091         snapshots => {},
2092         pending => {},
2093     };
2094
2095     $filename =~ m|/qemu-server/(\d+)\.conf$|
2096         || die "got strange filename '$filename'";
2097
2098     my $vmid = $1;
2099
2100     my $conf = $res;
2101     my $descr;
2102     my $section = '';
2103
2104     my @lines = split(/\n/, $raw);
2105     foreach my $line (@lines) {
2106         next if $line =~ m/^\s*$/;
2107
2108         if ($line =~ m/^\[PENDING\]\s*$/i) {
2109             $section = 'pending';
2110             if (defined($descr)) {
2111                 $descr =~ s/\s+$//;
2112                 $conf->{description} = $descr;
2113             }
2114             $descr = undef;
2115             $conf = $res->{$section} = {};
2116             next;
2117
2118         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2119             $section = $1;
2120             if (defined($descr)) {
2121                 $descr =~ s/\s+$//;
2122                 $conf->{description} = $descr;
2123             }
2124             $descr = undef;
2125             $conf = $res->{snapshots}->{$section} = {};
2126             next;
2127         }
2128
2129         if ($line =~ m/^\#(.*)\s*$/) {
2130             $descr = '' if !defined($descr);
2131             $descr .= PVE::Tools::decode_text($1) . "\n";
2132             next;
2133         }
2134
2135         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2136             $descr = '' if !defined($descr);
2137             $descr .= PVE::Tools::decode_text($2);
2138         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2139             $conf->{snapstate} = $1;
2140         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2141             my $key = $1;
2142             my $value = $2;
2143             $conf->{$key} = $value;
2144         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2145             my $value = $1;
2146             if ($section eq 'pending') {
2147                 $conf->{delete} = $value; # we parse this later
2148             } else {
2149                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2150             }
2151         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2152             my $key = $1;
2153             my $value = $2;
2154             eval { $value = check_type($key, $value); };
2155             if ($@) {
2156                 warn "vm $vmid - unable to parse value of '$key' - $@";
2157             } else {
2158                 $key = 'ide2' if $key eq 'cdrom';
2159                 my $fmt = $confdesc->{$key}->{format};
2160                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2161                     my $v = parse_drive($key, $value);
2162                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2163                         $v->{file} = $volid;
2164                         $value = print_drive($v);
2165                     } else {
2166                         warn "vm $vmid - unable to parse value of '$key'\n";
2167                         next;
2168                     }
2169                 }
2170
2171                 $conf->{$key} = $value;
2172             }
2173         }
2174     }
2175
2176     if (defined($descr)) {
2177         $descr =~ s/\s+$//;
2178         $conf->{description} = $descr;
2179     }
2180     delete $res->{snapstate}; # just to be sure
2181
2182     return $res;
2183 }
2184
2185 sub write_vm_config {
2186     my ($filename, $conf) = @_;
2187
2188     delete $conf->{snapstate}; # just to be sure
2189
2190     if ($conf->{cdrom}) {
2191         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2192         $conf->{ide2} = $conf->{cdrom};
2193         delete $conf->{cdrom};
2194     }
2195
2196     # we do not use 'smp' any longer
2197     if ($conf->{sockets}) {
2198         delete $conf->{smp};
2199     } elsif ($conf->{smp}) {
2200         $conf->{sockets} = $conf->{smp};
2201         delete $conf->{cores};
2202         delete $conf->{smp};
2203     }
2204
2205     my $used_volids = {};
2206
2207     my $cleanup_config = sub {
2208         my ($cref, $pending, $snapname) = @_;
2209
2210         foreach my $key (keys %$cref) {
2211             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2212                 $key eq 'snapstate' || $key eq 'pending';
2213             my $value = $cref->{$key};
2214             if ($key eq 'delete') {
2215                 die "propertry 'delete' is only allowed in [PENDING]\n"
2216                     if !$pending;
2217                 # fixme: check syntax?
2218                 next;
2219             }
2220             eval { $value = check_type($key, $value); };
2221             die "unable to parse value of '$key' - $@" if $@;
2222
2223             $cref->{$key} = $value;
2224
2225             if (!$snapname && is_valid_drivename($key)) {
2226                 my $drive = parse_drive($key, $value);
2227                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2228             }
2229         }
2230     };
2231
2232     &$cleanup_config($conf);
2233
2234     &$cleanup_config($conf->{pending}, 1);
2235
2236     foreach my $snapname (keys %{$conf->{snapshots}}) {
2237         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2238         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2239     }
2240
2241     # remove 'unusedX' settings if we re-add a volume
2242     foreach my $key (keys %$conf) {
2243         my $value = $conf->{$key};
2244         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2245             delete $conf->{$key};
2246         }
2247     }
2248
2249     my $generate_raw_config = sub {
2250         my ($conf, $pending) = @_;
2251
2252         my $raw = '';
2253
2254         # add description as comment to top of file
2255         if (defined(my $descr = $conf->{description})) {
2256             if ($descr) {
2257                 foreach my $cl (split(/\n/, $descr)) {
2258                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2259                 }
2260             } else {
2261                 $raw .= "#\n" if $pending;
2262             }
2263         }
2264
2265         foreach my $key (sort keys %$conf) {
2266             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2267             $raw .= "$key: $conf->{$key}\n";
2268         }
2269         return $raw;
2270     };
2271
2272     my $raw = &$generate_raw_config($conf);
2273
2274     if (scalar(keys %{$conf->{pending}})){
2275         $raw .= "\n[PENDING]\n";
2276         $raw .= &$generate_raw_config($conf->{pending}, 1);
2277     }
2278
2279     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2280         $raw .= "\n[$snapname]\n";
2281         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2282     }
2283
2284     return $raw;
2285 }
2286
2287 sub load_defaults {
2288
2289     my $res = {};
2290
2291     # we use static defaults from our JSON schema configuration
2292     foreach my $key (keys %$confdesc) {
2293         if (defined(my $default = $confdesc->{$key}->{default})) {
2294             $res->{$key} = $default;
2295         }
2296     }
2297
2298     return $res;
2299 }
2300
2301 sub config_list {
2302     my $vmlist = PVE::Cluster::get_vmlist();
2303     my $res = {};
2304     return $res if !$vmlist || !$vmlist->{ids};
2305     my $ids = $vmlist->{ids};
2306     my $nodename = nodename();
2307
2308     foreach my $vmid (keys %$ids) {
2309         my $d = $ids->{$vmid};
2310         next if !$d->{node} || $d->{node} ne $nodename;
2311         next if !$d->{type} || $d->{type} ne 'qemu';
2312         $res->{$vmid}->{exists} = 1;
2313     }
2314     return $res;
2315 }
2316
2317 # test if VM uses local resources (to prevent migration)
2318 sub check_local_resources {
2319     my ($conf, $noerr) = @_;
2320
2321     my @loc_res = ();
2322
2323     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2324     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2325
2326     push @loc_res, "ivshmem" if $conf->{ivshmem};
2327
2328     foreach my $k (keys %$conf) {
2329         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2330         # sockets are safe: they will recreated be on the target side post-migrate
2331         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2332         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2333     }
2334
2335     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2336
2337     return \@loc_res;
2338 }
2339
2340 # check if used storages are available on all nodes (use by migrate)
2341 sub check_storage_availability {
2342     my ($storecfg, $conf, $node) = @_;
2343
2344     PVE::QemuConfig->foreach_volume($conf, sub {
2345         my ($ds, $drive) = @_;
2346
2347         my $volid = $drive->{file};
2348         return if !$volid;
2349
2350         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2351         return if !$sid;
2352
2353         # check if storage is available on both nodes
2354         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2355         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2356    });
2357 }
2358
2359 # list nodes where all VM images are available (used by has_feature API)
2360 sub shared_nodes {
2361     my ($conf, $storecfg) = @_;
2362
2363     my $nodelist = PVE::Cluster::get_nodelist();
2364     my $nodehash = { map { $_ => 1 } @$nodelist };
2365     my $nodename = nodename();
2366
2367     PVE::QemuConfig->foreach_volume($conf, sub {
2368         my ($ds, $drive) = @_;
2369
2370         my $volid = $drive->{file};
2371         return if !$volid;
2372
2373         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2374         if ($storeid) {
2375             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2376             if ($scfg->{disable}) {
2377                 $nodehash = {};
2378             } elsif (my $avail = $scfg->{nodes}) {
2379                 foreach my $node (keys %$nodehash) {
2380                     delete $nodehash->{$node} if !$avail->{$node};
2381                 }
2382             } elsif (!$scfg->{shared}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     delete $nodehash->{$node} if $node ne $nodename
2385                 }
2386             }
2387         }
2388     });
2389
2390     return $nodehash
2391 }
2392
2393 sub check_local_storage_availability {
2394     my ($conf, $storecfg) = @_;
2395
2396     my $nodelist = PVE::Cluster::get_nodelist();
2397     my $nodehash = { map { $_ => {} } @$nodelist };
2398
2399     PVE::QemuConfig->foreach_volume($conf, sub {
2400         my ($ds, $drive) = @_;
2401
2402         my $volid = $drive->{file};
2403         return if !$volid;
2404
2405         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2406         if ($storeid) {
2407             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2408
2409             if ($scfg->{disable}) {
2410                 foreach my $node (keys %$nodehash) {
2411                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2412                 }
2413             } elsif (my $avail = $scfg->{nodes}) {
2414                 foreach my $node (keys %$nodehash) {
2415                     if (!$avail->{$node}) {
2416                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2417                     }
2418                 }
2419             }
2420         }
2421     });
2422
2423     foreach my $node (values %$nodehash) {
2424         if (my $unavail = $node->{unavailable_storages}) {
2425             $node->{unavailable_storages} = [ sort keys %$unavail ];
2426         }
2427     }
2428
2429     return $nodehash
2430 }
2431
2432 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2433 sub check_running {
2434     my ($vmid, $nocheck, $node) = @_;
2435
2436     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2437     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2438 }
2439
2440 sub vzlist {
2441
2442     my $vzlist = config_list();
2443
2444     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2445
2446     while (defined(my $de = $fd->read)) {
2447         next if $de !~ m/^(\d+)\.pid$/;
2448         my $vmid = $1;
2449         next if !defined($vzlist->{$vmid});
2450         if (my $pid = check_running($vmid)) {
2451             $vzlist->{$vmid}->{pid} = $pid;
2452         }
2453     }
2454
2455     return $vzlist;
2456 }
2457
2458 our $vmstatus_return_properties = {
2459     vmid => get_standard_option('pve-vmid'),
2460     status => {
2461         description => "Qemu process status.",
2462         type => 'string',
2463         enum => ['stopped', 'running'],
2464     },
2465     maxmem => {
2466         description => "Maximum memory in bytes.",
2467         type => 'integer',
2468         optional => 1,
2469         renderer => 'bytes',
2470     },
2471     maxdisk => {
2472         description => "Root disk size in bytes.",
2473         type => 'integer',
2474         optional => 1,
2475         renderer => 'bytes',
2476     },
2477     name => {
2478         description => "VM name.",
2479         type => 'string',
2480         optional => 1,
2481     },
2482     qmpstatus => {
2483         description => "Qemu QMP agent status.",
2484         type => 'string',
2485         optional => 1,
2486     },
2487     pid => {
2488         description => "PID of running qemu process.",
2489         type => 'integer',
2490         optional => 1,
2491     },
2492     uptime => {
2493         description => "Uptime.",
2494         type => 'integer',
2495         optional => 1,
2496         renderer => 'duration',
2497     },
2498     cpus => {
2499         description => "Maximum usable CPUs.",
2500         type => 'number',
2501         optional => 1,
2502     },
2503     lock => {
2504         description => "The current config lock, if any.",
2505         type => 'string',
2506         optional => 1,
2507     },
2508     tags => {
2509         description  => "The current configured tags, if any",
2510         type => 'string',
2511         optional => 1,
2512     },
2513 };
2514
2515 my $last_proc_pid_stat;
2516
2517 # get VM status information
2518 # This must be fast and should not block ($full == false)
2519 # We only query KVM using QMP if $full == true (this can be slow)
2520 sub vmstatus {
2521     my ($opt_vmid, $full) = @_;
2522
2523     my $res = {};
2524
2525     my $storecfg = PVE::Storage::config();
2526
2527     my $list = vzlist();
2528     my $defaults = load_defaults();
2529
2530     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2531
2532     my $cpucount = $cpuinfo->{cpus} || 1;
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536
2537         my $conf = PVE::QemuConfig->load_config($vmid);
2538
2539         my $d = { vmid => $vmid };
2540         $d->{pid} = $list->{$vmid}->{pid};
2541
2542         # fixme: better status?
2543         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2544
2545         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2546         if (defined($size)) {
2547             $d->{disk} = 0; # no info available
2548             $d->{maxdisk} = $size;
2549         } else {
2550             $d->{disk} = 0;
2551             $d->{maxdisk} = 0;
2552         }
2553
2554         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2555             * ($conf->{cores} || $defaults->{cores});
2556         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2557         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2558
2559         $d->{name} = $conf->{name} || "VM $vmid";
2560         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2561             : $defaults->{memory}*(1024*1024);
2562
2563         if ($conf->{balloon}) {
2564             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2565             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2566                 : $defaults->{shares};
2567         }
2568
2569         $d->{uptime} = 0;
2570         $d->{cpu} = 0;
2571         $d->{mem} = 0;
2572
2573         $d->{netout} = 0;
2574         $d->{netin} = 0;
2575
2576         $d->{diskread} = 0;
2577         $d->{diskwrite} = 0;
2578
2579         $d->{template} = PVE::QemuConfig->is_template($conf);
2580
2581         $d->{serial} = 1 if conf_has_serial($conf);
2582         $d->{lock} = $conf->{lock} if $conf->{lock};
2583         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2584
2585         $res->{$vmid} = $d;
2586     }
2587
2588     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2589     foreach my $dev (keys %$netdev) {
2590         next if $dev !~ m/^tap([1-9]\d*)i/;
2591         my $vmid = $1;
2592         my $d = $res->{$vmid};
2593         next if !$d;
2594
2595         $d->{netout} += $netdev->{$dev}->{receive};
2596         $d->{netin} += $netdev->{$dev}->{transmit};
2597
2598         if ($full) {
2599             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2600             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2601         }
2602
2603     }
2604
2605     my $ctime = gettimeofday;
2606
2607     foreach my $vmid (keys %$list) {
2608
2609         my $d = $res->{$vmid};
2610         my $pid = $d->{pid};
2611         next if !$pid;
2612
2613         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2614         next if !$pstat; # not running
2615
2616         my $used = $pstat->{utime} + $pstat->{stime};
2617
2618         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2619
2620         if ($pstat->{vsize}) {
2621             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2622         }
2623
2624         my $old = $last_proc_pid_stat->{$pid};
2625         if (!$old) {
2626             $last_proc_pid_stat->{$pid} = {
2627                 time => $ctime,
2628                 used => $used,
2629                 cpu => 0,
2630             };
2631             next;
2632         }
2633
2634         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2635
2636         if ($dtime > 1000) {
2637             my $dutime = $used -  $old->{used};
2638
2639             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2640             $last_proc_pid_stat->{$pid} = {
2641                 time => $ctime,
2642                 used => $used,
2643                 cpu => $d->{cpu},
2644             };
2645         } else {
2646             $d->{cpu} = $old->{cpu};
2647         }
2648     }
2649
2650     return $res if !$full;
2651
2652     my $qmpclient = PVE::QMPClient->new();
2653
2654     my $ballooncb = sub {
2655         my ($vmid, $resp) = @_;
2656
2657         my $info = $resp->{'return'};
2658         return if !$info->{max_mem};
2659
2660         my $d = $res->{$vmid};
2661
2662         # use memory assigned to VM
2663         $d->{maxmem} = $info->{max_mem};
2664         $d->{balloon} = $info->{actual};
2665
2666         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2667             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2668             $d->{freemem} = $info->{free_mem};
2669         }
2670
2671         $d->{ballooninfo} = $info;
2672     };
2673
2674     my $blockstatscb = sub {
2675         my ($vmid, $resp) = @_;
2676         my $data = $resp->{'return'} || [];
2677         my $totalrdbytes = 0;
2678         my $totalwrbytes = 0;
2679
2680         for my $blockstat (@$data) {
2681             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2682             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2683
2684             $blockstat->{device} =~ s/drive-//;
2685             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2686         }
2687         $res->{$vmid}->{diskread} = $totalrdbytes;
2688         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2689     };
2690
2691     my $statuscb = sub {
2692         my ($vmid, $resp) = @_;
2693
2694         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2695         # this fails if ballon driver is not loaded, so this must be
2696         # the last commnand (following command are aborted if this fails).
2697         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2698
2699         my $status = 'unknown';
2700         if (!defined($status = $resp->{'return'}->{status})) {
2701             warn "unable to get VM status\n";
2702             return;
2703         }
2704
2705         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2706     };
2707
2708     foreach my $vmid (keys %$list) {
2709         next if $opt_vmid && ($vmid ne $opt_vmid);
2710         next if !$res->{$vmid}->{pid}; # not running
2711         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2712     }
2713
2714     $qmpclient->queue_execute(undef, 2);
2715
2716     foreach my $vmid (keys %$list) {
2717         next if $opt_vmid && ($vmid ne $opt_vmid);
2718         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2719     }
2720
2721     return $res;
2722 }
2723
2724 sub conf_has_serial {
2725     my ($conf) = @_;
2726
2727     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2728         if ($conf->{"serial$i"}) {
2729             return 1;
2730         }
2731     }
2732
2733     return 0;
2734 }
2735
2736 sub conf_has_audio {
2737     my ($conf, $id) = @_;
2738
2739     $id //= 0;
2740     my $audio = $conf->{"audio$id"};
2741     return undef if !defined($audio);
2742
2743     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2744     my $audiodriver = $audioproperties->{driver} // 'spice';
2745
2746     return {
2747         dev => $audioproperties->{device},
2748         dev_id => "audiodev$id",
2749         backend => $audiodriver,
2750         backend_id => "$audiodriver-backend${id}",
2751     };
2752 }
2753
2754 sub audio_devs {
2755     my ($audio, $audiopciaddr) = @_;
2756
2757     my $devs = [];
2758
2759     my $id = $audio->{dev_id};
2760     my $audiodev = "audiodev=$audio->{backend_id}";
2761
2762     if ($audio->{dev} eq 'AC97') {
2763         push @$devs, '-device', "AC97,id=${id}${audiopciaddr},$audiodev";
2764     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2765         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2766         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0,$audiodev";
2767         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1,$audiodev";
2768     } else {
2769         die "unkown audio device '$audio->{dev}', implement me!";
2770     }
2771
2772     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2773
2774     return $devs;
2775 }
2776
2777 sub vga_conf_has_spice {
2778     my ($vga) = @_;
2779
2780     my $vgaconf = parse_vga($vga);
2781     my $vgatype = $vgaconf->{type};
2782     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2783
2784     return $1 || 1;
2785 }
2786
2787 sub is_native($) {
2788     my ($arch) = @_;
2789     return get_host_arch() eq $arch;
2790 }
2791
2792 sub get_vm_arch {
2793     my ($conf) = @_;
2794     return $conf->{arch} // get_host_arch();
2795 }
2796
2797 my $default_machines = {
2798     x86_64 => 'pc',
2799     aarch64 => 'virt',
2800 };
2801
2802 sub get_vm_machine {
2803     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2804
2805     my $machine = $forcemachine || $conf->{machine};
2806
2807     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2808         $arch //= 'x86_64';
2809         $machine ||= $default_machines->{$arch};
2810         if ($add_pve_version) {
2811             $kvmversion //= kvm_user_version();
2812             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2813             $machine .= "+pve$pvever";
2814         }
2815     }
2816
2817     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2818         # for version-pinned machines that do not include a pve-version (e.g.
2819         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2820         $machine .= '+pve0';
2821     }
2822
2823     return $machine;
2824 }
2825
2826 sub get_ovmf_files($) {
2827     my ($arch) = @_;
2828
2829     my $ovmf = $OVMF->{$arch}
2830         or die "no OVMF images known for architecture '$arch'\n";
2831
2832     return @$ovmf;
2833 }
2834
2835 my $Arch2Qemu = {
2836     aarch64 => '/usr/bin/qemu-system-aarch64',
2837     x86_64 => '/usr/bin/qemu-system-x86_64',
2838 };
2839 sub get_command_for_arch($) {
2840     my ($arch) = @_;
2841     return '/usr/bin/kvm' if is_native($arch);
2842
2843     my $cmd = $Arch2Qemu->{$arch}
2844         or die "don't know how to emulate architecture '$arch'\n";
2845     return $cmd;
2846 }
2847
2848 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2849 # to use in a QEMU command line (-cpu element), first array_intersect the result
2850 # of query_supported_ with query_understood_. This is necessary because:
2851 #
2852 # a) query_understood_ returns flags the host cannot use and
2853 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2854 #    flags, but CPU settings - with most of them being flags. Those settings
2855 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2856 #
2857 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2858 # expensive. If you need the value returned from this, you can get it much
2859 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2860 # $accel being 'kvm' or 'tcg'.
2861 #
2862 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2863 # changes, automatically populating pmxcfs.
2864 #
2865 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2866 # since kvm and tcg machines support different flags
2867 #
2868 sub query_supported_cpu_flags {
2869     my ($arch) = @_;
2870
2871     $arch //= get_host_arch();
2872     my $default_machine = $default_machines->{$arch};
2873
2874     my $flags = {};
2875
2876     # FIXME: Once this is merged, the code below should work for ARM as well:
2877     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2878     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2879         $arch eq "aarch64";
2880
2881     my $kvm_supported = defined(kvm_version());
2882     my $qemu_cmd = get_command_for_arch($arch);
2883     my $fakevmid = -1;
2884     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2885
2886     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2887     my $query_supported_run_qemu = sub {
2888         my ($kvm) = @_;
2889
2890         my $flags = {};
2891         my $cmd = [
2892             $qemu_cmd,
2893             '-machine', $default_machine,
2894             '-display', 'none',
2895             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2896             '-mon', 'chardev=qmp,mode=control',
2897             '-pidfile', $pidfile,
2898             '-S', '-daemonize'
2899         ];
2900
2901         if (!$kvm) {
2902             push @$cmd, '-accel', 'tcg';
2903         }
2904
2905         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2906         die "QEMU flag querying VM exited with code " . $rc if $rc;
2907
2908         eval {
2909             my $cmd_result = mon_cmd(
2910                 $fakevmid,
2911                 'query-cpu-model-expansion',
2912                 type => 'full',
2913                 model => { name => 'host' }
2914             );
2915
2916             my $props = $cmd_result->{model}->{props};
2917             foreach my $prop (keys %$props) {
2918                 next if $props->{$prop} ne '1';
2919                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2920                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2921                 # We only keep those with underscores, to match /proc/cpuinfo
2922                 $prop =~ s/\.|-/_/g;
2923                 $flags->{$prop} = 1;
2924             }
2925         };
2926         my $err = $@;
2927
2928         # force stop with 10 sec timeout and 'nocheck'
2929         # always stop, even if QMP failed
2930         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2931
2932         die $err if $err;
2933
2934         return [ sort keys %$flags ];
2935     };
2936
2937     # We need to query QEMU twice, since KVM and TCG have different supported flags
2938     PVE::QemuConfig->lock_config($fakevmid, sub {
2939         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2940         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2941
2942         if ($kvm_supported) {
2943             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2944             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2945         }
2946     });
2947
2948     return $flags;
2949 }
2950
2951 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2952 my $understood_cpu_flag_dir = "/usr/share/kvm";
2953 sub query_understood_cpu_flags {
2954     my $arch = get_host_arch();
2955     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2956
2957     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2958         if ! -e $filepath;
2959
2960     my $raw = file_get_contents($filepath);
2961     $raw =~ s/^\s+|\s+$//g;
2962     my @flags = split(/\s+/, $raw);
2963
2964     return \@flags;
2965 }
2966
2967 sub config_to_command {
2968     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2969
2970     my $cmd = [];
2971     my $globalFlags = [];
2972     my $machineFlags = [];
2973     my $rtcFlags = [];
2974     my $devices = [];
2975     my $pciaddr = '';
2976     my $bridges = {};
2977     my $ostype = $conf->{ostype};
2978     my $winversion = windows_version($ostype);
2979     my $kvm = $conf->{kvm};
2980     my $nodename = nodename();
2981
2982     my $arch = get_vm_arch($conf);
2983     my $kvm_binary = get_command_for_arch($arch);
2984     my $kvmver = kvm_user_version($kvm_binary);
2985
2986     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2987         $kvmver //= "undefined";
2988         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2989     }
2990
2991     my $add_pve_version = min_version($kvmver, 4, 1);
2992
2993     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2994     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2995     $kvm //= 1 if is_native($arch);
2996
2997     $machine_version =~ m/(\d+)\.(\d+)/;
2998     my ($machine_major, $machine_minor) = ($1, $2);
2999
3000     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3001         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3002     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3003         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3004     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3005         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3006         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3007             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3008     }
3009
3010     # if a specific +pve version is required for a feature, use $version_guard
3011     # instead of min_version to allow machines to be run with the minimum
3012     # required version
3013     my $required_pve_version = 0;
3014     my $version_guard = sub {
3015         my ($major, $minor, $pve) = @_;
3016         return 0 if !min_version($machine_version, $major, $minor, $pve);
3017         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3018         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3019         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3020         return 1;
3021     };
3022
3023     if ($kvm) {
3024         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3025             if !defined kvm_version();
3026     }
3027
3028     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3029     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3030     my $use_old_bios_files = undef;
3031     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3032
3033     my $cpuunits = defined($conf->{cpuunits}) ?
3034             $conf->{cpuunits} : $defaults->{cpuunits};
3035
3036     push @$cmd, $kvm_binary;
3037
3038     push @$cmd, '-id', $vmid;
3039
3040     my $vmname = $conf->{name} || "vm$vmid";
3041
3042     push @$cmd, '-name', $vmname;
3043
3044     my $use_virtio = 0;
3045
3046     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3047     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3048     push @$cmd, '-mon', "chardev=qmp,mode=control";
3049
3050     if (min_version($machine_version, 2, 12)) {
3051         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3052         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3053     }
3054
3055     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3056
3057     push @$cmd, '-daemonize';
3058
3059     if ($conf->{smbios1}) {
3060         my $smbios_conf = parse_smbios1($conf->{smbios1});
3061         if ($smbios_conf->{base64}) {
3062             # Do not pass base64 flag to qemu
3063             delete $smbios_conf->{base64};
3064             my $smbios_string = "";
3065             foreach my $key (keys %$smbios_conf) {
3066                 my $value;
3067                 if ($key eq "uuid") {
3068                     $value = $smbios_conf->{uuid}
3069                 } else {
3070                     $value = decode_base64($smbios_conf->{$key});
3071                 }
3072                 # qemu accepts any binary data, only commas need escaping by double comma
3073                 $value =~ s/,/,,/g;
3074                 $smbios_string .= "," . $key . "=" . $value if $value;
3075             }
3076             push @$cmd, '-smbios', "type=1" . $smbios_string;
3077         } else {
3078             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3079         }
3080     }
3081
3082     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3083         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3084         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3085
3086         my ($path, $format);
3087         if (my $d = parse_drive('efidisk0', $conf->{efidisk0})) {
3088             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3089             $format = $d->{format};
3090             if ($storeid) {
3091                 $path = PVE::Storage::path($storecfg, $d->{file});
3092                 if (!defined($format)) {
3093                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3094                     $format = qemu_img_format($scfg, $volname);
3095                 }
3096             } else {
3097                 $path = $d->{file};
3098                 die "efidisk format must be specified\n"
3099                     if !defined($format);
3100             }
3101         } else {
3102             warn "no efidisk configured! Using temporary efivars disk.\n";
3103             $path = "/tmp/$vmid-ovmf.fd";
3104             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3105             $format = 'raw';
3106         }
3107
3108         my $size_str = "";
3109
3110         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3111             $size_str = ",size=" . (-s $ovmf_vars);
3112         }
3113
3114         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3115         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3116     }
3117
3118     # load q35 config
3119     if ($q35) {
3120         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3121         if (min_version($machine_version, 4, 0)) {
3122             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3123         } else {
3124             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3125         }
3126     }
3127
3128     if ($conf->{vmgenid}) {
3129         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3130     }
3131
3132     # add usb controllers
3133     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3134     push @$devices, @usbcontrollers if @usbcontrollers;
3135     my $vga = parse_vga($conf->{vga});
3136
3137     my $qxlnum = vga_conf_has_spice($conf->{vga});
3138     $vga->{type} = 'qxl' if $qxlnum;
3139
3140     if (!$vga->{type}) {
3141         if ($arch eq 'aarch64') {
3142             $vga->{type} = 'virtio';
3143         } elsif (min_version($machine_version, 2, 9)) {
3144             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3145         } else {
3146             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3147         }
3148     }
3149
3150     # enable absolute mouse coordinates (needed by vnc)
3151     my $tablet;
3152     if (defined($conf->{tablet})) {
3153         $tablet = $conf->{tablet};
3154     } else {
3155         $tablet = $defaults->{tablet};
3156         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3157         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3158     }
3159
3160     if ($tablet) {
3161         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3162         my $kbd = print_keyboarddevice_full($conf, $arch);
3163         push @$devices, '-device', $kbd if defined($kbd);
3164     }
3165
3166     my $kvm_off = 0;
3167     my $gpu_passthrough;
3168
3169     # host pci devices
3170     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3171         my $id = "hostpci$i";
3172         my $d = parse_hostpci($conf->{$id});
3173         next if !$d;
3174
3175         if (my $pcie = $d->{pcie}) {
3176             die "q35 machine model is not enabled" if !$q35;
3177             # win7 wants to have the pcie devices directly on the pcie bus
3178             # instead of in the root port
3179             if ($winversion == 7) {
3180                 $pciaddr = print_pcie_addr("${id}bus0");
3181             } else {
3182                 # add more root ports if needed, 4 are present by default
3183                 # by pve-q35 cfgs, rest added here on demand.
3184                 if ($i > 3) {
3185                     push @$devices, '-device', print_pcie_root_port($i);
3186                 }
3187                 $pciaddr = print_pcie_addr($id);
3188             }
3189         } else {
3190             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3191         }
3192
3193         my $xvga = '';
3194         if ($d->{'x-vga'}) {
3195             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3196             $kvm_off = 1;
3197             $vga->{type} = 'none' if !defined($conf->{vga});
3198             $gpu_passthrough = 1;
3199         }
3200
3201         my $pcidevices = $d->{pciid};
3202         my $multifunction = 1 if @$pcidevices > 1;
3203
3204         my $sysfspath;
3205         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3206             my $pci_id = $pcidevices->[0]->{id};
3207             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3208             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3209         } elsif ($d->{mdev}) {
3210             warn "ignoring mediated device '$id' with multifunction device\n";
3211         }
3212
3213         my $j=0;
3214         foreach my $pcidevice (@$pcidevices) {
3215             my $devicestr = "vfio-pci";
3216
3217             if ($sysfspath) {
3218                 $devicestr .= ",sysfsdev=$sysfspath";
3219             } else {
3220                 $devicestr .= ",host=$pcidevice->{id}";
3221             }
3222
3223             my $mf_addr = $multifunction ? ".$j" : '';
3224             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3225
3226             if ($j == 0) {
3227                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3228                 $devicestr .= "$xvga";
3229                 $devicestr .= ",multifunction=on" if $multifunction;
3230                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3231             }
3232
3233             push @$devices, '-device', $devicestr;
3234             $j++;
3235         }
3236     }
3237
3238     # usb devices
3239     my $usb_dev_features = {};
3240     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3241
3242     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3243     push @$devices, @usbdevices if @usbdevices;
3244     # serial devices
3245     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3246         if (my $path = $conf->{"serial$i"}) {
3247             if ($path eq 'socket') {
3248                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3249                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3250                 # On aarch64, serial0 is the UART device. Qemu only allows
3251                 # connecting UART devices via the '-serial' command line, as
3252                 # the device has a fixed slot on the hardware...
3253                 if ($arch eq 'aarch64' && $i == 0) {
3254                     push @$devices, '-serial', "chardev:serial$i";
3255                 } else {
3256                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3257                 }
3258             } else {
3259                 die "no such serial device\n" if ! -c $path;
3260                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3261                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3262             }
3263         }
3264     }
3265
3266     # parallel devices
3267     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3268         if (my $path = $conf->{"parallel$i"}) {
3269             die "no such parallel device\n" if ! -c $path;
3270             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3271             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3272             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3273         }
3274     }
3275
3276     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3277         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3278         my $audio_devs = audio_devs($audio, $audiopciaddr);
3279         push @$devices, @$audio_devs;
3280     }
3281
3282     my $sockets = 1;
3283     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3284     $sockets = $conf->{sockets} if  $conf->{sockets};
3285
3286     my $cores = $conf->{cores} || 1;
3287
3288     my $maxcpus = $sockets * $cores;
3289
3290     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3291
3292     my $allowed_vcpus = $cpuinfo->{cpus};
3293
3294     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3295         if ($allowed_vcpus < $maxcpus);
3296
3297     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3298
3299         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3300         for (my $i = 2; $i <= $vcpus; $i++)  {
3301             my $cpustr = print_cpu_device($conf,$i);
3302             push @$cmd, '-device', $cpustr;
3303         }
3304
3305     } else {
3306
3307         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3308     }
3309     push @$cmd, '-nodefaults';
3310
3311     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3312
3313     my $bootindex_hash = {};
3314     my $i = 1;
3315     foreach my $o (split(//, $bootorder)) {
3316         $bootindex_hash->{$o} = $i*100;
3317         $i++;
3318     }
3319
3320     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3321
3322     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3323
3324     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3325
3326     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3327         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3328         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3329         push @$cmd,  '-vnc', "unix:$socket,password";
3330     } else {
3331         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3332         push @$cmd, '-nographic';
3333     }
3334
3335     # time drift fix
3336     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3337     my $useLocaltime = $conf->{localtime};
3338
3339     if ($winversion >= 5) { # windows
3340         $useLocaltime = 1 if !defined($conf->{localtime});
3341
3342         # use time drift fix when acpi is enabled
3343         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3344             $tdf = 1 if !defined($conf->{tdf});
3345         }
3346     }
3347
3348     if ($winversion >= 6) {
3349         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3350         push @$cmd, '-no-hpet';
3351     }
3352
3353     push @$rtcFlags, 'driftfix=slew' if $tdf;
3354
3355     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3356         push @$rtcFlags, "base=$conf->{startdate}";
3357     } elsif ($useLocaltime) {
3358         push @$rtcFlags, 'base=localtime';
3359     }
3360
3361     if ($forcecpu) {
3362         push @$cmd, '-cpu', $forcecpu;
3363     } else {
3364         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3365     }
3366
3367     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3368
3369     push @$cmd, '-S' if $conf->{freeze};
3370
3371     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3372
3373     my $guest_agent = parse_guest_agent($conf);
3374
3375     if ($guest_agent->{enabled}) {
3376         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3377         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3378
3379         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3380             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3381             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3382             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3383         } elsif ($guest_agent->{type} eq 'isa') {
3384             push @$devices, '-device', "isa-serial,chardev=qga0";
3385         }
3386     }
3387
3388     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3389     if ($rng && &$version_guard(4, 1, 2)) {
3390         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3391         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3392
3393         my $limiter_str = "";
3394         if ($max_bytes) {
3395             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3396         }
3397
3398         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3399         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3400             if ! -e $rng->{source};
3401
3402         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3403
3404         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3405         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3406     }
3407
3408     my $spice_port;
3409
3410     if ($qxlnum) {
3411         if ($qxlnum > 1) {
3412             if ($winversion){
3413                 for (my $i = 1; $i < $qxlnum; $i++){
3414                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3415                 }
3416             } else {
3417                 # assume other OS works like Linux
3418                 my ($ram, $vram) = ("134217728", "67108864");
3419                 if ($vga->{memory}) {
3420                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3421                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3422                 }
3423                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3424                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3425             }
3426         }
3427
3428         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3429
3430         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3431         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3432         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3433
3434         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3435         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3436         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3437
3438         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3439         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3440
3441         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3442         if ($spice_enhancement->{foldersharing}) {
3443             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3444             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3445         }
3446
3447         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3448         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3449         push @$devices, '-spice', "$spice_opts";
3450     }
3451
3452     # enable balloon by default, unless explicitly disabled
3453     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3454         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3455         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3456     }
3457
3458     if ($conf->{watchdog}) {
3459         my $wdopts = parse_watchdog($conf->{watchdog});
3460         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3461         my $watchdog = $wdopts->{model} || 'i6300esb';
3462         push @$devices, '-device', "$watchdog$pciaddr";
3463         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3464     }
3465
3466     my $vollist = [];
3467     my $scsicontroller = {};
3468     my $ahcicontroller = {};
3469     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3470
3471     # Add iscsi initiator name if available
3472     if (my $initiator = get_initiator_name()) {
3473         push @$devices, '-iscsi', "initiator-name=$initiator";
3474     }
3475
3476     PVE::QemuConfig->foreach_volume($conf, sub {
3477         my ($ds, $drive) = @_;
3478
3479         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3480             push @$vollist, $drive->{file};
3481         }
3482
3483         # ignore efidisk here, already added in bios/fw handling code above
3484         return if $drive->{interface} eq 'efidisk';
3485
3486         $use_virtio = 1 if $ds =~ m/^virtio/;
3487
3488         if (drive_is_cdrom ($drive)) {
3489             if ($bootindex_hash->{d}) {
3490                 $drive->{bootindex} = $bootindex_hash->{d};
3491                 $bootindex_hash->{d} += 1;
3492             }
3493         } else {
3494             if ($bootindex_hash->{c}) {
3495                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3496                 $bootindex_hash->{c} += 1;
3497             }
3498         }
3499
3500         if ($drive->{interface} eq 'virtio'){
3501            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3502         }
3503
3504         if ($drive->{interface} eq 'scsi') {
3505
3506             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3507
3508             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3509                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3510
3511             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3512             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3513
3514             my $iothread = '';
3515             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3516                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3517                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3518             } elsif ($drive->{iothread}) {
3519                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3520             }
3521
3522             my $queues = '';
3523             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3524                 $queues = ",num_queues=$drive->{queues}";
3525             }
3526
3527             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3528             $scsicontroller->{$controller}=1;
3529         }
3530
3531         if ($drive->{interface} eq 'sata') {
3532             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3533             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3534             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3535             $ahcicontroller->{$controller}=1;
3536         }
3537
3538         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3539         push @$devices, '-drive',$drive_cmd;
3540         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3541     });
3542
3543     for (my $i = 0; $i < $MAX_NETS; $i++) {
3544          next if !$conf->{"net$i"};
3545          my $d = parse_net($conf->{"net$i"});
3546          next if !$d;
3547
3548          $use_virtio = 1 if $d->{model} eq 'virtio';
3549
3550          if ($bootindex_hash->{n}) {
3551             $d->{bootindex} = $bootindex_hash->{n};
3552             $bootindex_hash->{n} += 1;
3553          }
3554
3555          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3556          push @$devices, '-netdev', $netdevfull;
3557
3558          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3559          push @$devices, '-device', $netdevicefull;
3560     }
3561
3562     if ($conf->{ivshmem}) {
3563         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3564
3565         my $bus;
3566         if ($q35) {
3567             $bus = print_pcie_addr("ivshmem");
3568         } else {
3569             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3570         }
3571
3572         my $ivshmem_name = $ivshmem->{name} // $vmid;
3573         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3574
3575         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3576         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3577     }
3578
3579     # pci.4 is nested in pci.1
3580     $bridges->{1} = 1 if $bridges->{4};
3581
3582     if (!$q35) {
3583         # add pci bridges
3584         if (min_version($machine_version, 2, 3)) {
3585            $bridges->{1} = 1;
3586            $bridges->{2} = 1;
3587         }
3588
3589         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3590
3591     }
3592
3593     for my $k (sort {$b cmp $a} keys %$bridges) {
3594         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3595         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3596         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3597         if ($q35) {
3598             # add after -readconfig pve-q35.cfg
3599             splice @$devices, 2, 0, '-device', $devstr;
3600         } else {
3601             unshift @$devices, '-device', $devstr if $k > 0;
3602         }
3603     }
3604
3605     if (!$kvm) {
3606         push @$machineFlags, 'accel=tcg';
3607     }
3608
3609     my $machine_type_min = $machine_type;
3610     if ($add_pve_version) {
3611         $machine_type_min =~ s/\+pve\d+$//;
3612         $machine_type_min .= "+pve$required_pve_version";
3613     }
3614     push @$machineFlags, "type=${machine_type_min}";
3615
3616     push @$cmd, @$devices;
3617     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3618     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3619     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3620
3621     if (my $vmstate = $conf->{vmstate}) {
3622         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3623         push @$vollist, $vmstate;
3624         push @$cmd, '-loadstate', $statepath;
3625         print "activating and using '$vmstate' as vmstate\n";
3626     }
3627
3628     # add custom args
3629     if ($conf->{args}) {
3630         my $aa = PVE::Tools::split_args($conf->{args});
3631         push @$cmd, @$aa;
3632     }
3633
3634     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3635 }
3636
3637 sub spice_port {
3638     my ($vmid) = @_;
3639
3640     my $res = mon_cmd($vmid, 'query-spice');
3641
3642     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3643 }
3644
3645 sub vm_devices_list {
3646     my ($vmid) = @_;
3647
3648     my $res = mon_cmd($vmid, 'query-pci');
3649     my $devices_to_check = [];
3650     my $devices = {};
3651     foreach my $pcibus (@$res) {
3652         push @$devices_to_check, @{$pcibus->{devices}},
3653     }
3654
3655     while (@$devices_to_check) {
3656         my $to_check = [];
3657         for my $d (@$devices_to_check) {
3658             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3659             next if !$d->{'pci_bridge'};
3660
3661             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3662             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3663         }
3664         $devices_to_check = $to_check;
3665     }
3666
3667     my $resblock = mon_cmd($vmid, 'query-block');
3668     foreach my $block (@$resblock) {
3669         if($block->{device} =~ m/^drive-(\S+)/){
3670                 $devices->{$1} = 1;
3671         }
3672     }
3673
3674     my $resmice = mon_cmd($vmid, 'query-mice');
3675     foreach my $mice (@$resmice) {
3676         if ($mice->{name} eq 'QEMU HID Tablet') {
3677             $devices->{tablet} = 1;
3678             last;
3679         }
3680     }
3681
3682     # for usb devices there is no query-usb
3683     # but we can iterate over the entries in
3684     # qom-list path=/machine/peripheral
3685     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3686     foreach my $per (@$resperipheral) {
3687         if ($per->{name} =~ m/^usb\d+$/) {
3688             $devices->{$per->{name}} = 1;
3689         }
3690     }
3691
3692     return $devices;
3693 }
3694
3695 sub vm_deviceplug {
3696     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3697
3698     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3699
3700     my $devices_list = vm_devices_list($vmid);
3701     return 1 if defined($devices_list->{$deviceid});
3702
3703     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3704
3705     if ($deviceid eq 'tablet') {
3706
3707         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3708
3709     } elsif ($deviceid eq 'keyboard') {
3710
3711         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3712
3713     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3714
3715         die "usb hotplug currently not reliable\n";
3716         # since we can't reliably hot unplug all added usb devices
3717         # and usb passthrough disables live migration
3718         # we disable usb hotplugging for now
3719         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3720
3721     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3722
3723         qemu_iothread_add($vmid, $deviceid, $device);
3724
3725         qemu_driveadd($storecfg, $vmid, $device);
3726         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3727
3728         qemu_deviceadd($vmid, $devicefull);
3729         eval { qemu_deviceaddverify($vmid, $deviceid); };
3730         if (my $err = $@) {
3731             eval { qemu_drivedel($vmid, $deviceid); };
3732             warn $@ if $@;
3733             die $err;
3734         }
3735
3736     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3737
3738
3739         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3740         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3741         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3742
3743         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3744
3745         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3746             qemu_iothread_add($vmid, $deviceid, $device);
3747             $devicefull .= ",iothread=iothread-$deviceid";
3748         }
3749
3750         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3751             $devicefull .= ",num_queues=$device->{queues}";
3752         }
3753
3754         qemu_deviceadd($vmid, $devicefull);
3755         qemu_deviceaddverify($vmid, $deviceid);
3756
3757     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3758
3759         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3760         qemu_driveadd($storecfg, $vmid, $device);
3761
3762         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3763         eval { qemu_deviceadd($vmid, $devicefull); };
3764         if (my $err = $@) {
3765             eval { qemu_drivedel($vmid, $deviceid); };
3766             warn $@ if $@;
3767             die $err;
3768         }
3769
3770     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3771
3772         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3773
3774         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3775         my $use_old_bios_files = undef;
3776         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3777
3778         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3779         qemu_deviceadd($vmid, $netdevicefull);
3780         eval {
3781             qemu_deviceaddverify($vmid, $deviceid);
3782             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3783         };
3784         if (my $err = $@) {
3785             eval { qemu_netdevdel($vmid, $deviceid); };
3786             warn $@ if $@;
3787             die $err;
3788         }
3789
3790     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3791
3792         my $bridgeid = $2;
3793         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3794         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3795
3796         qemu_deviceadd($vmid, $devicefull);
3797         qemu_deviceaddverify($vmid, $deviceid);
3798
3799     } else {
3800         die "can't hotplug device '$deviceid'\n";
3801     }
3802
3803     return 1;
3804 }
3805
3806 # fixme: this should raise exceptions on error!
3807 sub vm_deviceunplug {
3808     my ($vmid, $conf, $deviceid) = @_;
3809
3810     my $devices_list = vm_devices_list($vmid);
3811     return 1 if !defined($devices_list->{$deviceid});
3812
3813     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3814
3815     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3816
3817         qemu_devicedel($vmid, $deviceid);
3818
3819     } elsif ($deviceid =~ m/^usb\d+$/) {
3820
3821         die "usb hotplug currently not reliable\n";
3822         # when unplugging usb devices this way,
3823         # there may be remaining usb controllers/hubs
3824         # so we disable it for now
3825         qemu_devicedel($vmid, $deviceid);
3826         qemu_devicedelverify($vmid, $deviceid);
3827
3828     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3829
3830         qemu_devicedel($vmid, $deviceid);
3831         qemu_devicedelverify($vmid, $deviceid);
3832         qemu_drivedel($vmid, $deviceid);
3833         qemu_iothread_del($conf, $vmid, $deviceid);
3834
3835     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3836
3837         qemu_devicedel($vmid, $deviceid);
3838         qemu_devicedelverify($vmid, $deviceid);
3839         qemu_iothread_del($conf, $vmid, $deviceid);
3840
3841     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3842
3843         qemu_devicedel($vmid, $deviceid);
3844         qemu_drivedel($vmid, $deviceid);
3845         qemu_deletescsihw($conf, $vmid, $deviceid);
3846
3847     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3848
3849         qemu_devicedel($vmid, $deviceid);
3850         qemu_devicedelverify($vmid, $deviceid);
3851         qemu_netdevdel($vmid, $deviceid);
3852
3853     } else {
3854         die "can't unplug device '$deviceid'\n";
3855     }
3856
3857     return 1;
3858 }
3859
3860 sub qemu_deviceadd {
3861     my ($vmid, $devicefull) = @_;
3862
3863     $devicefull = "driver=".$devicefull;
3864     my %options =  split(/[=,]/, $devicefull);
3865
3866     mon_cmd($vmid, "device_add" , %options);
3867 }
3868
3869 sub qemu_devicedel {
3870     my ($vmid, $deviceid) = @_;
3871
3872     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3873 }
3874
3875 sub qemu_iothread_add {
3876     my($vmid, $deviceid, $device) = @_;
3877
3878     if ($device->{iothread}) {
3879         my $iothreads = vm_iothreads_list($vmid);
3880         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3881     }
3882 }
3883
3884 sub qemu_iothread_del {
3885     my($conf, $vmid, $deviceid) = @_;
3886
3887     my $confid = $deviceid;
3888     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3889         $confid = 'scsi' . $1;
3890     }
3891     my $device = parse_drive($confid, $conf->{$confid});
3892     if ($device->{iothread}) {
3893         my $iothreads = vm_iothreads_list($vmid);
3894         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3895     }
3896 }
3897
3898 sub qemu_objectadd {
3899     my($vmid, $objectid, $qomtype) = @_;
3900
3901     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3902
3903     return 1;
3904 }
3905
3906 sub qemu_objectdel {
3907     my($vmid, $objectid) = @_;
3908
3909     mon_cmd($vmid, "object-del", id => $objectid);
3910
3911     return 1;
3912 }
3913
3914 sub qemu_driveadd {
3915     my ($storecfg, $vmid, $device) = @_;
3916
3917     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3918     $drive =~ s/\\/\\\\/g;
3919     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3920
3921     # If the command succeeds qemu prints: "OK"
3922     return 1 if $ret =~ m/OK/s;
3923
3924     die "adding drive failed: $ret\n";
3925 }
3926
3927 sub qemu_drivedel {
3928     my($vmid, $deviceid) = @_;
3929
3930     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3931     $ret =~ s/^\s+//;
3932
3933     return 1 if $ret eq "";
3934
3935     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3936     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3937
3938     die "deleting drive $deviceid failed : $ret\n";
3939 }
3940
3941 sub qemu_deviceaddverify {
3942     my ($vmid, $deviceid) = @_;
3943
3944     for (my $i = 0; $i <= 5; $i++) {
3945          my $devices_list = vm_devices_list($vmid);
3946          return 1 if defined($devices_list->{$deviceid});
3947          sleep 1;
3948     }
3949
3950     die "error on hotplug device '$deviceid'\n";
3951 }
3952
3953
3954 sub qemu_devicedelverify {
3955     my ($vmid, $deviceid) = @_;
3956
3957     # need to verify that the device is correctly removed as device_del
3958     # is async and empty return is not reliable
3959
3960     for (my $i = 0; $i <= 5; $i++) {
3961          my $devices_list = vm_devices_list($vmid);
3962          return 1 if !defined($devices_list->{$deviceid});
3963          sleep 1;
3964     }
3965
3966     die "error on hot-unplugging device '$deviceid'\n";
3967 }
3968
3969 sub qemu_findorcreatescsihw {
3970     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3971
3972     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3973
3974     my $scsihwid="$controller_prefix$controller";
3975     my $devices_list = vm_devices_list($vmid);
3976
3977     if(!defined($devices_list->{$scsihwid})) {
3978         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3979     }
3980
3981     return 1;
3982 }
3983
3984 sub qemu_deletescsihw {
3985     my ($conf, $vmid, $opt) = @_;
3986
3987     my $device = parse_drive($opt, $conf->{$opt});
3988
3989     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3990         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3991         return 1;
3992     }
3993
3994     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3995
3996     my $devices_list = vm_devices_list($vmid);
3997     foreach my $opt (keys %{$devices_list}) {
3998         if (is_valid_drivename($opt)) {
3999             my $drive = parse_drive($opt, $conf->{$opt});
4000             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4001                 return 1;
4002             }
4003         }
4004     }
4005
4006     my $scsihwid="scsihw$controller";
4007
4008     vm_deviceunplug($vmid, $conf, $scsihwid);
4009
4010     return 1;
4011 }
4012
4013 sub qemu_add_pci_bridge {
4014     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4015
4016     my $bridges = {};
4017
4018     my $bridgeid;
4019
4020     print_pci_addr($device, $bridges, $arch, $machine_type);
4021
4022     while (my ($k, $v) = each %$bridges) {
4023         $bridgeid = $k;
4024     }
4025     return 1 if !defined($bridgeid) || $bridgeid < 1;
4026
4027     my $bridge = "pci.$bridgeid";
4028     my $devices_list = vm_devices_list($vmid);
4029
4030     if (!defined($devices_list->{$bridge})) {
4031         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4032     }
4033
4034     return 1;
4035 }
4036
4037 sub qemu_set_link_status {
4038     my ($vmid, $device, $up) = @_;
4039
4040     mon_cmd($vmid, "set_link", name => $device,
4041                up => $up ? JSON::true : JSON::false);
4042 }
4043
4044 sub qemu_netdevadd {
4045     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4046
4047     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4048     my %options =  split(/[=,]/, $netdev);
4049
4050     mon_cmd($vmid, "netdev_add",  %options);
4051     return 1;
4052 }
4053
4054 sub qemu_netdevdel {
4055     my ($vmid, $deviceid) = @_;
4056
4057     mon_cmd($vmid, "netdev_del", id => $deviceid);
4058 }
4059
4060 sub qemu_usb_hotplug {
4061     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4062
4063     return if !$device;
4064
4065     # remove the old one first
4066     vm_deviceunplug($vmid, $conf, $deviceid);
4067
4068     # check if xhci controller is necessary and available
4069     if ($device->{usb3}) {
4070
4071         my $devicelist = vm_devices_list($vmid);
4072
4073         if (!$devicelist->{xhci}) {
4074             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4075             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4076         }
4077     }
4078     my $d = parse_usb_device($device->{host});
4079     $d->{usb3} = $device->{usb3};
4080
4081     # add the new one
4082     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4083 }
4084
4085 sub qemu_cpu_hotplug {
4086     my ($vmid, $conf, $vcpus) = @_;
4087
4088     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4089
4090     my $sockets = 1;
4091     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4092     $sockets = $conf->{sockets} if  $conf->{sockets};
4093     my $cores = $conf->{cores} || 1;
4094     my $maxcpus = $sockets * $cores;
4095
4096     $vcpus = $maxcpus if !$vcpus;
4097
4098     die "you can't add more vcpus than maxcpus\n"
4099         if $vcpus > $maxcpus;
4100
4101     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4102
4103     if ($vcpus < $currentvcpus) {
4104
4105         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4106
4107             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4108                 qemu_devicedel($vmid, "cpu$i");
4109                 my $retry = 0;
4110                 my $currentrunningvcpus = undef;
4111                 while (1) {
4112                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4113                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4114                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4115                     $retry++;
4116                     sleep 1;
4117                 }
4118                 #update conf after each succesfull cpu unplug
4119                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4120                 PVE::QemuConfig->write_config($vmid, $conf);
4121             }
4122         } else {
4123             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4124         }
4125
4126         return;
4127     }
4128
4129     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4130     die "vcpus in running vm does not match its configuration\n"
4131         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4132
4133     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4134
4135         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4136             my $cpustr = print_cpu_device($conf, $i);
4137             qemu_deviceadd($vmid, $cpustr);
4138
4139             my $retry = 0;
4140             my $currentrunningvcpus = undef;
4141             while (1) {
4142                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4143                 last if scalar(@{$currentrunningvcpus}) == $i;
4144                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4145                 sleep 1;
4146                 $retry++;
4147             }
4148             #update conf after each succesfull cpu hotplug
4149             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4150             PVE::QemuConfig->write_config($vmid, $conf);
4151         }
4152     } else {
4153
4154         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4155             mon_cmd($vmid, "cpu-add", id => int($i));
4156         }
4157     }
4158 }
4159
4160 sub qemu_block_set_io_throttle {
4161     my ($vmid, $deviceid,
4162         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4163         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4164         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4165         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4166
4167     return if !check_running($vmid) ;
4168
4169     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4170         bps => int($bps),
4171         bps_rd => int($bps_rd),
4172         bps_wr => int($bps_wr),
4173         iops => int($iops),
4174         iops_rd => int($iops_rd),
4175         iops_wr => int($iops_wr),
4176         bps_max => int($bps_max),
4177         bps_rd_max => int($bps_rd_max),
4178         bps_wr_max => int($bps_wr_max),
4179         iops_max => int($iops_max),
4180         iops_rd_max => int($iops_rd_max),
4181         iops_wr_max => int($iops_wr_max),
4182         bps_max_length => int($bps_max_length),
4183         bps_rd_max_length => int($bps_rd_max_length),
4184         bps_wr_max_length => int($bps_wr_max_length),
4185         iops_max_length => int($iops_max_length),
4186         iops_rd_max_length => int($iops_rd_max_length),
4187         iops_wr_max_length => int($iops_wr_max_length),
4188     );
4189
4190 }
4191
4192 # old code, only used to shutdown old VM after update
4193 sub __read_avail {
4194     my ($fh, $timeout) = @_;
4195
4196     my $sel = new IO::Select;
4197     $sel->add($fh);
4198
4199     my $res = '';
4200     my $buf;
4201
4202     my @ready;
4203     while (scalar (@ready = $sel->can_read($timeout))) {
4204         my $count;
4205         if ($count = $fh->sysread($buf, 8192)) {
4206             if ($buf =~ /^(.*)\(qemu\) $/s) {
4207                 $res .= $1;
4208                 last;
4209             } else {
4210                 $res .= $buf;
4211             }
4212         } else {
4213             if (!defined($count)) {
4214                 die "$!\n";
4215             }
4216             last;
4217         }
4218     }
4219
4220     die "monitor read timeout\n" if !scalar(@ready);
4221
4222     return $res;
4223 }
4224
4225 sub qemu_block_resize {
4226     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4227
4228     my $running = check_running($vmid);
4229
4230     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4231
4232     return if !$running;
4233
4234     my $padding = (1024 - $size % 1024) % 1024;
4235     $size = $size + $padding;
4236
4237     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4238
4239 }
4240
4241 sub qemu_volume_snapshot {
4242     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4243
4244     my $running = check_running($vmid);
4245
4246     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4247         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4248     } else {
4249         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4250     }
4251 }
4252
4253 sub qemu_volume_snapshot_delete {
4254     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4255
4256     my $running = check_running($vmid);
4257
4258     if($running) {
4259
4260         $running = undef;
4261         my $conf = PVE::QemuConfig->load_config($vmid);
4262         PVE::QemuConfig->foreach_volume($conf, sub {
4263             my ($ds, $drive) = @_;
4264             $running = 1 if $drive->{file} eq $volid;
4265         });
4266     }
4267
4268     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4269         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4270     } else {
4271         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4272     }
4273 }
4274
4275 sub set_migration_caps {
4276     my ($vmid) = @_;
4277
4278     my $cap_ref = [];
4279
4280     my $enabled_cap = {
4281         "auto-converge" => 1,
4282         "xbzrle" => 1,
4283         "x-rdma-pin-all" => 0,
4284         "zero-blocks" => 0,
4285         "compress" => 0
4286     };
4287
4288     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4289
4290     for my $supported_capability (@$supported_capabilities) {
4291         push @$cap_ref, {
4292             capability => $supported_capability->{capability},
4293             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4294         };
4295     }
4296
4297     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4298 }
4299
4300 sub foreach_volid {
4301     my ($conf, $func, @param) = @_;
4302
4303     my $volhash = {};
4304
4305     my $test_volid = sub {
4306         my ($key, $drive, $snapname) = @_;
4307
4308         my $volid = $drive->{file};
4309         return if !$volid;
4310
4311         $volhash->{$volid}->{cdrom} //= 1;
4312         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4313
4314         my $replicate = $drive->{replicate} // 1;
4315         $volhash->{$volid}->{replicate} //= 0;
4316         $volhash->{$volid}->{replicate} = 1 if $replicate;
4317
4318         $volhash->{$volid}->{shared} //= 0;
4319         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4320
4321         $volhash->{$volid}->{referenced_in_config} //= 0;
4322         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4323
4324         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4325             if defined($snapname);
4326
4327         my $size = $drive->{size};
4328         $volhash->{$volid}->{size} //= $size if $size;
4329
4330         $volhash->{$volid}->{is_vmstate} //= 0;
4331         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4332
4333         $volhash->{$volid}->{is_unused} //= 0;
4334         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4335     };
4336
4337     my $include_opts = {
4338         extra_keys => ['vmstate'],
4339         include_unused => 1,
4340     };
4341
4342     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, sub {
4343         my ($ds, $drive) = @_;
4344         $test_volid->($ds, $drive);
4345     });
4346
4347     foreach my $snapname (keys %{$conf->{snapshots}}) {
4348         my $snap = $conf->{snapshots}->{$snapname};
4349         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, sub {
4350             my ($ds, $drive) = @_;
4351             $test_volid->($ds, $drive, $snapname);
4352         });
4353     }
4354
4355     foreach my $volid (keys %$volhash) {
4356         &$func($volid, $volhash->{$volid}, @param);
4357     }
4358 }
4359
4360 my $fast_plug_option = {
4361     'lock' => 1,
4362     'name' => 1,
4363     'onboot' => 1,
4364     'shares' => 1,
4365     'startup' => 1,
4366     'description' => 1,
4367     'protection' => 1,
4368     'vmstatestorage' => 1,
4369     'hookscript' => 1,
4370     'tags' => 1,
4371 };
4372
4373 # hotplug changes in [PENDING]
4374 # $selection hash can be used to only apply specified options, for
4375 # example: { cores => 1 } (only apply changed 'cores')
4376 # $errors ref is used to return error messages
4377 sub vmconfig_hotplug_pending {
4378     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4379
4380     my $defaults = load_defaults();
4381     my $arch = get_vm_arch($conf);
4382     my $machine_type = get_vm_machine($conf, undef, $arch);
4383
4384     # commit values which do not have any impact on running VM first
4385     # Note: those option cannot raise errors, we we do not care about
4386     # $selection and always apply them.
4387
4388     my $add_error = sub {
4389         my ($opt, $msg) = @_;
4390         $errors->{$opt} = "hotplug problem - $msg";
4391     };
4392
4393     my $changes = 0;
4394     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4395         if ($fast_plug_option->{$opt}) {
4396             $conf->{$opt} = $conf->{pending}->{$opt};
4397             delete $conf->{pending}->{$opt};
4398             $changes = 1;
4399         }
4400     }
4401
4402     if ($changes) {
4403         PVE::QemuConfig->write_config($vmid, $conf);
4404     }
4405
4406     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4407
4408     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4409     foreach my $opt (sort keys %$pending_delete_hash) {
4410         next if $selection && !$selection->{$opt};
4411         my $force = $pending_delete_hash->{$opt}->{force};
4412         eval {
4413             if ($opt eq 'hotplug') {
4414                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4415             } elsif ($opt eq 'tablet') {
4416                 die "skip\n" if !$hotplug_features->{usb};
4417                 if ($defaults->{tablet}) {
4418                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4419                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4420                         if $arch eq 'aarch64';
4421                 } else {
4422                     vm_deviceunplug($vmid, $conf, 'tablet');
4423                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4424                 }
4425             } elsif ($opt =~ m/^usb\d+/) {
4426                 die "skip\n";
4427                 # since we cannot reliably hot unplug usb devices
4428                 # we are disabling it
4429                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4430                 vm_deviceunplug($vmid, $conf, $opt);
4431             } elsif ($opt eq 'vcpus') {
4432                 die "skip\n" if !$hotplug_features->{cpu};
4433                 qemu_cpu_hotplug($vmid, $conf, undef);
4434             } elsif ($opt eq 'balloon') {
4435                 # enable balloon device is not hotpluggable
4436                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4437                 # here we reset the ballooning value to memory
4438                 my $balloon = $conf->{memory} || $defaults->{memory};
4439                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4440             } elsif ($fast_plug_option->{$opt}) {
4441                 # do nothing
4442             } elsif ($opt =~ m/^net(\d+)$/) {
4443                 die "skip\n" if !$hotplug_features->{network};
4444                 vm_deviceunplug($vmid, $conf, $opt);
4445             } elsif (is_valid_drivename($opt)) {
4446                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4447                 vm_deviceunplug($vmid, $conf, $opt);
4448                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4449             } elsif ($opt =~ m/^memory$/) {
4450                 die "skip\n" if !$hotplug_features->{memory};
4451                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4452             } elsif ($opt eq 'cpuunits') {
4453                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4454             } elsif ($opt eq 'cpulimit') {
4455                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4456             } else {
4457                 die "skip\n";
4458             }
4459         };
4460         if (my $err = $@) {
4461             &$add_error($opt, $err) if $err ne "skip\n";
4462         } else {
4463             delete $conf->{$opt};
4464             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4465         }
4466     }
4467
4468     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4469     $apply_pending_cloudinit = sub {
4470         return if $apply_pending_cloudinit_done; # once is enough
4471         $apply_pending_cloudinit_done = 1; # once is enough
4472
4473         my ($key, $value) = @_;
4474
4475         my @cloudinit_opts = keys %$confdesc_cloudinit;
4476         foreach my $opt (keys %{$conf->{pending}}) {
4477             next if !grep { $_ eq $opt } @cloudinit_opts;
4478             $conf->{$opt} = delete $conf->{pending}->{$opt};
4479         }
4480
4481         my $new_conf = { %$conf };
4482         $new_conf->{$key} = $value;
4483         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4484     };
4485
4486     foreach my $opt (keys %{$conf->{pending}}) {
4487         next if $selection && !$selection->{$opt};
4488         my $value = $conf->{pending}->{$opt};
4489         eval {
4490             if ($opt eq 'hotplug') {
4491                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4492             } elsif ($opt eq 'tablet') {
4493                 die "skip\n" if !$hotplug_features->{usb};
4494                 if ($value == 1) {
4495                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4496                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4497                         if $arch eq 'aarch64';
4498                 } elsif ($value == 0) {
4499                     vm_deviceunplug($vmid, $conf, 'tablet');
4500                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4501                 }
4502             } elsif ($opt =~ m/^usb\d+$/) {
4503                 die "skip\n";
4504                 # since we cannot reliably hot unplug usb devices
4505                 # we are disabling it
4506                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4507                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4508                 die "skip\n" if !$d;
4509                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4510             } elsif ($opt eq 'vcpus') {
4511                 die "skip\n" if !$hotplug_features->{cpu};
4512                 qemu_cpu_hotplug($vmid, $conf, $value);
4513             } elsif ($opt eq 'balloon') {
4514                 # enable/disable balloning device is not hotpluggable
4515                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4516                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4517                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4518
4519                 # allow manual ballooning if shares is set to zero
4520                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4521                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4522                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4523                 }
4524             } elsif ($opt =~ m/^net(\d+)$/) {
4525                 # some changes can be done without hotplug
4526                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4527                                     $vmid, $opt, $value, $arch, $machine_type);
4528             } elsif (is_valid_drivename($opt)) {
4529                 die "skip\n" if $opt eq 'efidisk0';
4530                 # some changes can be done without hotplug
4531                 my $drive = parse_drive($opt, $value);
4532                 if (drive_is_cloudinit($drive)) {
4533                     &$apply_pending_cloudinit($opt, $value);
4534                 }
4535                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4536                                      $vmid, $opt, $value, $arch, $machine_type);
4537             } elsif ($opt =~ m/^memory$/) { #dimms
4538                 die "skip\n" if !$hotplug_features->{memory};
4539                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4540             } elsif ($opt eq 'cpuunits') {
4541                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4542             } elsif ($opt eq 'cpulimit') {
4543                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4544                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4545             } else {
4546                 die "skip\n";  # skip non-hot-pluggable options
4547             }
4548         };
4549         if (my $err = $@) {
4550             &$add_error($opt, $err) if $err ne "skip\n";
4551         } else {
4552             $conf->{$opt} = $value;
4553             delete $conf->{pending}->{$opt};
4554         }
4555     }
4556
4557     PVE::QemuConfig->write_config($vmid, $conf);
4558 }
4559
4560 sub try_deallocate_drive {
4561     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4562
4563     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4564         my $volid = $drive->{file};
4565         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4566             my $sid = PVE::Storage::parse_volume_id($volid);
4567             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4568
4569             # check if the disk is really unused
4570             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4571                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4572             PVE::Storage::vdisk_free($storecfg, $volid);
4573             return 1;
4574         } else {
4575             # If vm is not owner of this disk remove from config
4576             return 1;
4577         }
4578     }
4579
4580     return undef;
4581 }
4582
4583 sub vmconfig_delete_or_detach_drive {
4584     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4585
4586     my $drive = parse_drive($opt, $conf->{$opt});
4587
4588     my $rpcenv = PVE::RPCEnvironment::get();
4589     my $authuser = $rpcenv->get_user();
4590
4591     if ($force) {
4592         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4593         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4594     } else {
4595         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4596     }
4597 }
4598
4599
4600
4601 sub vmconfig_apply_pending {
4602     my ($vmid, $conf, $storecfg, $errors) = @_;
4603
4604     my $add_apply_error = sub {
4605         my ($opt, $msg) = @_;
4606         my $err_msg = "unable to apply pending change $opt : $msg";
4607         $errors->{$opt} = $err_msg;
4608         warn $err_msg;
4609     };
4610
4611     # cold plug
4612
4613     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4614     foreach my $opt (sort keys %$pending_delete_hash) {
4615         my $force = $pending_delete_hash->{$opt}->{force};
4616         eval {
4617             if ($opt =~ m/^unused/) {
4618                 die "internal error";
4619             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4620                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4621             }
4622         };
4623         if (my $err = $@) {
4624             $add_apply_error->($opt, $err);
4625         } else {
4626             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4627             delete $conf->{$opt};
4628         }
4629     }
4630
4631     PVE::QemuConfig->cleanup_pending($conf);
4632
4633     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4634         next if $opt eq 'delete'; # just to be sure
4635         eval {
4636             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4637                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4638             }
4639         };
4640         if (my $err = $@) {
4641             $add_apply_error->($opt, $err);
4642         } else {
4643             $conf->{$opt} = delete $conf->{pending}->{$opt};
4644         }
4645     }
4646
4647     # write all changes at once to avoid unnecessary i/o
4648     PVE::QemuConfig->write_config($vmid, $conf);
4649 }
4650
4651 sub vmconfig_update_net {
4652     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4653
4654     my $newnet = parse_net($value);
4655
4656     if ($conf->{$opt}) {
4657         my $oldnet = parse_net($conf->{$opt});
4658
4659         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4660             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4661             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4662             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4663
4664             # for non online change, we try to hot-unplug
4665             die "skip\n" if !$hotplug;
4666             vm_deviceunplug($vmid, $conf, $opt);
4667         } else {
4668
4669             die "internal error" if $opt !~ m/net(\d+)/;
4670             my $iface = "tap${vmid}i$1";
4671
4672             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4673                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4674                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4675                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4676                 PVE::Network::tap_unplug($iface);
4677
4678                 if ($have_sdn) {
4679                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4680                 } else {
4681                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4682                 }
4683             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4684                 # Rate can be applied on its own but any change above needs to
4685                 # include the rate in tap_plug since OVS resets everything.
4686                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4687             }
4688
4689             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4690                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4691             }
4692
4693             return 1;
4694         }
4695     }
4696
4697     if ($hotplug) {
4698         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4699     } else {
4700         die "skip\n";
4701     }
4702 }
4703
4704 sub vmconfig_update_disk {
4705     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4706
4707     my $drive = parse_drive($opt, $value);
4708
4709     if ($conf->{$opt}) {
4710
4711         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4712
4713             my $media = $drive->{media} || 'disk';
4714             my $oldmedia = $old_drive->{media} || 'disk';
4715             die "unable to change media type\n" if $media ne $oldmedia;
4716
4717             if (!drive_is_cdrom($old_drive)) {
4718
4719                 if ($drive->{file} ne $old_drive->{file}) {
4720
4721                     die "skip\n" if !$hotplug;
4722
4723                     # unplug and register as unused
4724                     vm_deviceunplug($vmid, $conf, $opt);
4725                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4726
4727                 } else {
4728                     # update existing disk
4729
4730                     # skip non hotpluggable value
4731                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4732                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4733                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4734                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4735                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4736                         die "skip\n";
4737                     }
4738
4739                     # apply throttle
4740                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4741                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4742                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4743                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4744                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4745                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4746                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4747                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4748                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4749                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4750                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4751                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4752                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4753                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4754                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4755                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4756                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4757                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4758
4759                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4760                                                    ($drive->{mbps} || 0)*1024*1024,
4761                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4762                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4763                                                    $drive->{iops} || 0,
4764                                                    $drive->{iops_rd} || 0,
4765                                                    $drive->{iops_wr} || 0,
4766                                                    ($drive->{mbps_max} || 0)*1024*1024,
4767                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4768                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4769                                                    $drive->{iops_max} || 0,
4770                                                    $drive->{iops_rd_max} || 0,
4771                                                    $drive->{iops_wr_max} || 0,
4772                                                    $drive->{bps_max_length} || 1,
4773                                                    $drive->{bps_rd_max_length} || 1,
4774                                                    $drive->{bps_wr_max_length} || 1,
4775                                                    $drive->{iops_max_length} || 1,
4776                                                    $drive->{iops_rd_max_length} || 1,
4777                                                    $drive->{iops_wr_max_length} || 1);
4778
4779                     }
4780
4781                     return 1;
4782                 }
4783
4784             } else { # cdrom
4785
4786                 if ($drive->{file} eq 'none') {
4787                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4788                     if (drive_is_cloudinit($old_drive)) {
4789                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4790                     }
4791                 } else {
4792                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4793
4794                     # force eject if locked
4795                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4796
4797                     if ($path) {
4798                         mon_cmd($vmid, "blockdev-change-medium",
4799                             id => "$opt", filename => "$path");
4800                     }
4801                 }
4802
4803                 return 1;
4804             }
4805         }
4806     }
4807
4808     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4809     # hotplug new disks
4810     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4811     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4812 }
4813
4814 # called in locked context by incoming migration
4815 sub vm_migrate_get_nbd_disks {
4816     my ($storecfg, $conf, $replicated_volumes) = @_;
4817
4818     my $local_volumes = {};
4819     PVE::QemuConfig->foreach_volume($conf, sub {
4820         my ($ds, $drive) = @_;
4821
4822         return if drive_is_cdrom($drive);
4823
4824         my $volid = $drive->{file};
4825
4826         return if !$volid;
4827
4828         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4829
4830         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4831         return if $scfg->{shared};
4832
4833         # replicated disks re-use existing state via bitmap
4834         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4835         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4836     });
4837     return $local_volumes;
4838 }
4839
4840 # called in locked context by incoming migration
4841 sub vm_migrate_alloc_nbd_disks {
4842     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4843
4844     my $format = undef;
4845
4846     my $nbd = {};
4847     foreach my $opt (sort keys %$source_volumes) {
4848         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4849
4850         if ($use_existing) {
4851             $nbd->{$opt}->{drivestr} = print_drive($drive);
4852             $nbd->{$opt}->{volid} = $volid;
4853             $nbd->{$opt}->{replicated} = 1;
4854             next;
4855         }
4856
4857         # If a remote storage is specified and the format of the original
4858         # volume is not available there, fall back to the default format.
4859         # Otherwise use the same format as the original.
4860         if (!$storagemap->{identity}) {
4861             $storeid = map_storage($storagemap, $storeid);
4862             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4863             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4864             my $fileFormat = qemu_img_format($scfg, $volname);
4865             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4866         } else {
4867             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4868             $format = qemu_img_format($scfg, $volname);
4869         }
4870
4871         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4872         my $newdrive = $drive;
4873         $newdrive->{format} = $format;
4874         $newdrive->{file} = $newvolid;
4875         my $drivestr = print_drive($newdrive);
4876         $nbd->{$opt}->{drivestr} = $drivestr;
4877         $nbd->{$opt}->{volid} = $newvolid;
4878     }
4879
4880     return $nbd;
4881 }
4882
4883 # see vm_start_nolock for parameters, additionally:
4884 # migrate_opts:
4885 #   storagemap = parsed storage map for allocating NBD disks
4886 sub vm_start {
4887     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4888
4889     return PVE::QemuConfig->lock_config($vmid, sub {
4890         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4891
4892         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4893
4894         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4895
4896         PVE::QemuConfig->check_lock($conf)
4897             if !($params->{skiplock} || $params->{resume});
4898
4899         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4900
4901         if (my $storagemap = $migrate_opts->{storagemap}) {
4902             my $replicated = $migrate_opts->{replicated_volumes};
4903             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4904             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4905
4906             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4907                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4908             }
4909         }
4910
4911         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4912     });
4913 }
4914
4915
4916 # params:
4917 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4918 #   skiplock => 0/1, skip checking for config lock
4919 #   forcemachine => to force Qemu machine (rollback/migration)
4920 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4921 #   timeout => in seconds
4922 #   paused => start VM in paused state (backup)
4923 #   resume => resume from hibernation
4924 # migrate_opts:
4925 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4926 #   migratedfrom => source node
4927 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4928 #   network => CIDR of migration network
4929 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4930 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4931 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4932 sub vm_start_nolock {
4933     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4934
4935     my $statefile = $params->{statefile};
4936     my $resume = $params->{resume};
4937
4938     my $migratedfrom = $migrate_opts->{migratedfrom};
4939     my $migration_type = $migrate_opts->{type};
4940
4941     my $res = {};
4942
4943     # clean up leftover reboot request files
4944     eval { clear_reboot_request($vmid); };
4945     warn $@ if $@;
4946
4947     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4948         vmconfig_apply_pending($vmid, $conf, $storecfg);
4949         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4950     }
4951
4952     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4953
4954     my $defaults = load_defaults();
4955
4956     # set environment variable useful inside network script
4957     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4958
4959     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4960
4961     my $forcemachine = $params->{forcemachine};
4962     my $forcecpu = $params->{forcecpu};
4963     if ($resume) {
4964         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4965         $forcemachine = $conf->{runningmachine};
4966         $forcecpu = $conf->{runningcpu};
4967         print "Resuming suspended VM\n";
4968     }
4969
4970     my ($cmd, $vollist, $spice_port) =
4971         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4972
4973     my $migration_ip;
4974     my $get_migration_ip = sub {
4975         my ($nodename) = @_;
4976
4977         return $migration_ip if defined($migration_ip);
4978
4979         my $cidr = $migrate_opts->{network};
4980
4981         if (!defined($cidr)) {
4982             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4983             $cidr = $dc_conf->{migration}->{network};
4984         }
4985
4986         if (defined($cidr)) {
4987             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4988
4989             die "could not get IP: no address configured on local " .
4990                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4991
4992             die "could not get IP: multiple addresses configured on local " .
4993                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4994
4995             $migration_ip = @$ips[0];
4996         }
4997
4998         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4999             if !defined($migration_ip);
5000
5001         return $migration_ip;
5002     };
5003
5004     my $migrate_uri;
5005     if ($statefile) {
5006         if ($statefile eq 'tcp') {
5007             my $localip = "localhost";
5008             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5009             my $nodename = nodename();
5010
5011             if (!defined($migration_type)) {
5012                 if (defined($datacenterconf->{migration}->{type})) {
5013                     $migration_type = $datacenterconf->{migration}->{type};
5014                 } else {
5015                     $migration_type = 'secure';
5016                 }
5017             }
5018
5019             if ($migration_type eq 'insecure') {
5020                 $localip = $get_migration_ip->($nodename);
5021                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5022             }
5023
5024             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5025             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5026             $migrate_uri = "tcp:${localip}:${migrate_port}";
5027             push @$cmd, '-incoming', $migrate_uri;
5028             push @$cmd, '-S';
5029
5030         } elsif ($statefile eq 'unix') {
5031             # should be default for secure migrations as a ssh TCP forward
5032             # tunnel is not deterministic reliable ready and fails regurarly
5033             # to set up in time, so use UNIX socket forwards
5034             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5035             unlink $socket_addr;
5036
5037             $migrate_uri = "unix:$socket_addr";
5038
5039             push @$cmd, '-incoming', $migrate_uri;
5040             push @$cmd, '-S';
5041
5042         } elsif (-e $statefile) {
5043             push @$cmd, '-loadstate', $statefile;
5044         } else {
5045             my $statepath = PVE::Storage::path($storecfg, $statefile);
5046             push @$vollist, $statefile;
5047             push @$cmd, '-loadstate', $statepath;
5048         }
5049     } elsif ($params->{paused}) {
5050         push @$cmd, '-S';
5051     }
5052
5053     # host pci devices
5054     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5055       my $d = parse_hostpci($conf->{"hostpci$i"});
5056       next if !$d;
5057       my $pcidevices = $d->{pciid};
5058       foreach my $pcidevice (@$pcidevices) {
5059             my $pciid = $pcidevice->{id};
5060
5061             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5062             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5063             die "no pci device info for device '$pciid'\n" if !$info;
5064
5065             if ($d->{mdev}) {
5066                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5067                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5068             } else {
5069                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5070                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5071                 die "can't reset pci device '$pciid'\n"
5072                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5073             }
5074       }
5075     }
5076
5077     PVE::Storage::activate_volumes($storecfg, $vollist);
5078
5079     eval {
5080         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5081             outfunc => sub {}, errfunc => sub {});
5082     };
5083     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5084     # timeout should be more than enough here...
5085     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5086
5087     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5088                                               : $defaults->{cpuunits};
5089
5090     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5091     my %run_params = (
5092         timeout => $statefile ? undef : $start_timeout,
5093         umask => 0077,
5094         noerr => 1,
5095     );
5096
5097     # when migrating, prefix QEMU output so other side can pick up any
5098     # errors that might occur and show the user
5099     if ($migratedfrom) {
5100         $run_params{quiet} = 1;
5101         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5102     }
5103
5104     my %properties = (
5105         Slice => 'qemu.slice',
5106         KillMode => 'none',
5107         CPUShares => $cpuunits
5108     );
5109
5110     if (my $cpulimit = $conf->{cpulimit}) {
5111         $properties{CPUQuota} = int($cpulimit * 100);
5112     }
5113     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5114
5115     my $run_qemu = sub {
5116         PVE::Tools::run_fork sub {
5117             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5118
5119             my $exitcode = run_command($cmd, %run_params);
5120             die "QEMU exited with code $exitcode\n" if $exitcode;
5121         };
5122     };
5123
5124     if ($conf->{hugepages}) {
5125
5126         my $code = sub {
5127             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5128             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5129
5130             PVE::QemuServer::Memory::hugepages_mount();
5131             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5132
5133             eval { $run_qemu->() };
5134             if (my $err = $@) {
5135                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5136                 die $err;
5137             }
5138
5139             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5140         };
5141         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5142
5143     } else {
5144         eval { $run_qemu->() };
5145     }
5146
5147     if (my $err = $@) {
5148         # deactivate volumes if start fails
5149         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5150         die "start failed: $err";
5151     }
5152
5153     print "migration listens on $migrate_uri\n" if $migrate_uri;
5154     $res->{migrate_uri} = $migrate_uri;
5155
5156     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5157         eval { mon_cmd($vmid, "cont"); };
5158         warn $@ if $@;
5159     }
5160
5161     #start nbd server for storage migration
5162     if (my $nbd = $migrate_opts->{nbd}) {
5163         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5164
5165         my $migrate_storage_uri;
5166         # nbd_protocol_version > 0 for unix socket support
5167         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5168             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5169             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5170             $migrate_storage_uri = "nbd:unix:$socket_path";
5171         } else {
5172             my $nodename = nodename();
5173             my $localip = $get_migration_ip->($nodename);
5174             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5175             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5176
5177             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5178             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5179             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5180         }
5181
5182         $res->{migrate_storage_uri} = $migrate_storage_uri;
5183
5184         foreach my $opt (sort keys %$nbd) {
5185             my $drivestr = $nbd->{$opt}->{drivestr};
5186             my $volid = $nbd->{$opt}->{volid};
5187             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5188             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5189             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5190             print "re-using replicated volume: $opt - $volid\n"
5191                 if $nbd->{$opt}->{replicated};
5192
5193             $res->{drives}->{$opt} = $nbd->{$opt};
5194             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5195         }
5196     }
5197
5198     if ($migratedfrom) {
5199         eval {
5200             set_migration_caps($vmid);
5201         };
5202         warn $@ if $@;
5203
5204         if ($spice_port) {
5205             print "spice listens on port $spice_port\n";
5206             $res->{spice_port} = $spice_port;
5207             if ($migrate_opts->{spice_ticket}) {
5208                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5209                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5210             }
5211         }
5212
5213     } else {
5214         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5215             if !$statefile && $conf->{balloon};
5216
5217         foreach my $opt (keys %$conf) {
5218             next if $opt !~  m/^net\d+$/;
5219             my $nicconf = parse_net($conf->{$opt});
5220             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5221         }
5222     }
5223
5224     mon_cmd($vmid, 'qom-set',
5225                 path => "machine/peripheral/balloon0",
5226                 property => "guest-stats-polling-interval",
5227                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5228
5229     if ($resume) {
5230         print "Resumed VM, removing state\n";
5231         if (my $vmstate = $conf->{vmstate}) {
5232             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5233             PVE::Storage::vdisk_free($storecfg, $vmstate);
5234         }
5235         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5236         PVE::QemuConfig->write_config($vmid, $conf);
5237     }
5238
5239     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5240
5241     return $res;
5242 }
5243
5244 sub vm_commandline {
5245     my ($storecfg, $vmid, $snapname) = @_;
5246
5247     my $conf = PVE::QemuConfig->load_config($vmid);
5248     my $forcemachine;
5249     my $forcecpu;
5250
5251     if ($snapname) {
5252         my $snapshot = $conf->{snapshots}->{$snapname};
5253         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5254
5255         # check for machine or CPU overrides in snapshot
5256         $forcemachine = $snapshot->{runningmachine};
5257         $forcecpu = $snapshot->{runningcpu};
5258
5259         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5260
5261         $conf = $snapshot;
5262     }
5263
5264     my $defaults = load_defaults();
5265
5266     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5267         $forcemachine, $forcecpu);
5268
5269     return PVE::Tools::cmd2string($cmd);
5270 }
5271
5272 sub vm_reset {
5273     my ($vmid, $skiplock) = @_;
5274
5275     PVE::QemuConfig->lock_config($vmid, sub {
5276
5277         my $conf = PVE::QemuConfig->load_config($vmid);
5278
5279         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5280
5281         mon_cmd($vmid, "system_reset");
5282     });
5283 }
5284
5285 sub get_vm_volumes {
5286     my ($conf) = @_;
5287
5288     my $vollist = [];
5289     foreach_volid($conf, sub {
5290         my ($volid, $attr) = @_;
5291
5292         return if $volid =~ m|^/|;
5293
5294         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5295         return if !$sid;
5296
5297         push @$vollist, $volid;
5298     });
5299
5300     return $vollist;
5301 }
5302
5303 sub vm_stop_cleanup {
5304     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5305
5306     eval {
5307
5308         if (!$keepActive) {
5309             my $vollist = get_vm_volumes($conf);
5310             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5311         }
5312
5313         foreach my $ext (qw(mon qmp pid vnc qga)) {
5314             unlink "/var/run/qemu-server/${vmid}.$ext";
5315         }
5316
5317         if ($conf->{ivshmem}) {
5318             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5319             # just delete it for now, VMs which have this already open do not
5320             # are affected, but new VMs will get a separated one. If this
5321             # becomes an issue we either add some sort of ref-counting or just
5322             # add a "don't delete on stop" flag to the ivshmem format.
5323             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5324         }
5325
5326         foreach my $key (keys %$conf) {
5327             next if $key !~ m/^hostpci(\d+)$/;
5328             my $hostpciindex = $1;
5329             my $d = parse_hostpci($conf->{$key});
5330             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5331
5332             foreach my $pci (@{$d->{pciid}}) {
5333                 my $pciid = $pci->{id};
5334                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5335             }
5336         }
5337
5338         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5339     };
5340     warn $@ if $@; # avoid errors - just warn
5341 }
5342
5343 # call only in locked context
5344 sub _do_vm_stop {
5345     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5346
5347     my $pid = check_running($vmid, $nocheck);
5348     return if !$pid;
5349
5350     my $conf;
5351     if (!$nocheck) {
5352         $conf = PVE::QemuConfig->load_config($vmid);
5353         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5354         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5355             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5356             $timeout = $opts->{down} if $opts->{down};
5357         }
5358         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5359     }
5360
5361     eval {
5362         if ($shutdown) {
5363             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5364                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5365             } else {
5366                 mon_cmd($vmid, "system_powerdown");
5367             }
5368         } else {
5369             mon_cmd($vmid, "quit");
5370         }
5371     };
5372     my $err = $@;
5373
5374     if (!$err) {
5375         $timeout = 60 if !defined($timeout);
5376
5377         my $count = 0;
5378         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5379             $count++;
5380             sleep 1;
5381         }
5382
5383         if ($count >= $timeout) {
5384             if ($force) {
5385                 warn "VM still running - terminating now with SIGTERM\n";
5386                 kill 15, $pid;
5387             } else {
5388                 die "VM quit/powerdown failed - got timeout\n";
5389             }
5390         } else {
5391             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5392             return;
5393         }
5394     } else {
5395         if ($force) {
5396             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5397             kill 15, $pid;
5398         } else {
5399             die "VM quit/powerdown failed\n";
5400         }
5401     }
5402
5403     # wait again
5404     $timeout = 10;
5405
5406     my $count = 0;
5407     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5408         $count++;
5409         sleep 1;
5410     }
5411
5412     if ($count >= $timeout) {
5413         warn "VM still running - terminating now with SIGKILL\n";
5414         kill 9, $pid;
5415         sleep 1;
5416     }
5417
5418     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5419 }
5420
5421 # Note: use $nocheck to skip tests if VM configuration file exists.
5422 # We need that when migration VMs to other nodes (files already moved)
5423 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5424 sub vm_stop {
5425     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5426
5427     $force = 1 if !defined($force) && !$shutdown;
5428
5429     if ($migratedfrom){
5430         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5431         kill 15, $pid if $pid;
5432         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5433         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5434         return;
5435     }
5436
5437     PVE::QemuConfig->lock_config($vmid, sub {
5438         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5439    });
5440 }
5441
5442 sub vm_reboot {
5443     my ($vmid, $timeout) = @_;
5444
5445     PVE::QemuConfig->lock_config($vmid, sub {
5446         eval {
5447
5448             # only reboot if running, as qmeventd starts it again on a stop event
5449             return if !check_running($vmid);
5450
5451             create_reboot_request($vmid);
5452
5453             my $storecfg = PVE::Storage::config();
5454             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5455
5456         };
5457         if (my $err = $@) {
5458             # avoid that the next normal shutdown will be confused for a reboot
5459             clear_reboot_request($vmid);
5460             die $err;
5461         }
5462    });
5463 }
5464
5465 # note: if using the statestorage parameter, the caller has to check privileges
5466 sub vm_suspend {
5467     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5468
5469     my $conf;
5470     my $path;
5471     my $storecfg;
5472     my $vmstate;
5473
5474     PVE::QemuConfig->lock_config($vmid, sub {
5475
5476         $conf = PVE::QemuConfig->load_config($vmid);
5477
5478         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5479         PVE::QemuConfig->check_lock($conf)
5480             if !($skiplock || $is_backing_up);
5481
5482         die "cannot suspend to disk during backup\n"
5483             if $is_backing_up && $includestate;
5484
5485         if ($includestate) {
5486             $conf->{lock} = 'suspending';
5487             my $date = strftime("%Y-%m-%d", localtime(time()));
5488             $storecfg = PVE::Storage::config();
5489             if (!$statestorage) {
5490                 $statestorage = find_vmstate_storage($conf, $storecfg);
5491                 # check permissions for the storage
5492                 my $rpcenv = PVE::RPCEnvironment::get();
5493                 if ($rpcenv->{type} ne 'cli') {
5494                     my $authuser = $rpcenv->get_user();
5495                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5496                 }
5497             }
5498
5499
5500             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5501             $path = PVE::Storage::path($storecfg, $vmstate);
5502             PVE::QemuConfig->write_config($vmid, $conf);
5503         } else {
5504             mon_cmd($vmid, "stop");
5505         }
5506     });
5507
5508     if ($includestate) {
5509         # save vm state
5510         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5511
5512         eval {
5513             mon_cmd($vmid, "savevm-start", statefile => $path);
5514             for(;;) {
5515                 my $state = mon_cmd($vmid, "query-savevm");
5516                 if (!$state->{status}) {
5517                     die "savevm not active\n";
5518                 } elsif ($state->{status} eq 'active') {
5519                     sleep(1);
5520                     next;
5521                 } elsif ($state->{status} eq 'completed') {
5522                     print "State saved, quitting\n";
5523                     last;
5524                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5525                     die "query-savevm failed with error '$state->{error}'\n"
5526                 } else {
5527                     die "query-savevm returned status '$state->{status}'\n";
5528                 }
5529             }
5530         };
5531         my $err = $@;
5532
5533         PVE::QemuConfig->lock_config($vmid, sub {
5534             $conf = PVE::QemuConfig->load_config($vmid);
5535             if ($err) {
5536                 # cleanup, but leave suspending lock, to indicate something went wrong
5537                 eval {
5538                     mon_cmd($vmid, "savevm-end");
5539                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5540                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5541                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5542                     PVE::QemuConfig->write_config($vmid, $conf);
5543                 };
5544                 warn $@ if $@;
5545                 die $err;
5546             }
5547
5548             die "lock changed unexpectedly\n"
5549                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5550
5551             mon_cmd($vmid, "quit");
5552             $conf->{lock} = 'suspended';
5553             PVE::QemuConfig->write_config($vmid, $conf);
5554         });
5555     }
5556 }
5557
5558 sub vm_resume {
5559     my ($vmid, $skiplock, $nocheck) = @_;
5560
5561     PVE::QemuConfig->lock_config($vmid, sub {
5562         my $res = mon_cmd($vmid, 'query-status');
5563         my $resume_cmd = 'cont';
5564
5565         if ($res->{status} && $res->{status} eq 'suspended') {
5566             $resume_cmd = 'system_wakeup';
5567         }
5568
5569         if (!$nocheck) {
5570
5571             my $conf = PVE::QemuConfig->load_config($vmid);
5572
5573             PVE::QemuConfig->check_lock($conf)
5574                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5575         }
5576
5577         mon_cmd($vmid, $resume_cmd);
5578     });
5579 }
5580
5581 sub vm_sendkey {
5582     my ($vmid, $skiplock, $key) = @_;
5583
5584     PVE::QemuConfig->lock_config($vmid, sub {
5585
5586         my $conf = PVE::QemuConfig->load_config($vmid);
5587
5588         # there is no qmp command, so we use the human monitor command
5589         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5590         die $res if $res ne '';
5591     });
5592 }
5593
5594 # vzdump restore implementaion
5595
5596 sub tar_archive_read_firstfile {
5597     my $archive = shift;
5598
5599     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5600
5601     # try to detect archive type first
5602     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5603         die "unable to open file '$archive'\n";
5604     my $firstfile = <$fh>;
5605     kill 15, $pid;
5606     close $fh;
5607
5608     die "ERROR: archive contaions no data\n" if !$firstfile;
5609     chomp $firstfile;
5610
5611     return $firstfile;
5612 }
5613
5614 sub tar_restore_cleanup {
5615     my ($storecfg, $statfile) = @_;
5616
5617     print STDERR "starting cleanup\n";
5618
5619     if (my $fd = IO::File->new($statfile, "r")) {
5620         while (defined(my $line = <$fd>)) {
5621             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5622                 my $volid = $2;
5623                 eval {
5624                     if ($volid =~ m|^/|) {
5625                         unlink $volid || die 'unlink failed\n';
5626                     } else {
5627                         PVE::Storage::vdisk_free($storecfg, $volid);
5628                     }
5629                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5630                 };
5631                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5632             } else {
5633                 print STDERR "unable to parse line in statfile - $line";
5634             }
5635         }
5636         $fd->close();
5637     }
5638 }
5639
5640 sub restore_file_archive {
5641     my ($archive, $vmid, $user, $opts) = @_;
5642
5643     my $format = $opts->{format};
5644     my $comp;
5645
5646     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5647         $format = 'tar' if !$format;
5648         $comp = 'gzip';
5649     } elsif ($archive =~ m/\.tar$/) {
5650         $format = 'tar' if !$format;
5651     } elsif ($archive =~ m/.tar.lzo$/) {
5652         $format = 'tar' if !$format;
5653         $comp = 'lzop';
5654     } elsif ($archive =~ m/\.vma$/) {
5655         $format = 'vma' if !$format;
5656     } elsif ($archive =~ m/\.vma\.gz$/) {
5657         $format = 'vma' if !$format;
5658         $comp = 'gzip';
5659     } elsif ($archive =~ m/\.vma\.lzo$/) {
5660         $format = 'vma' if !$format;
5661         $comp = 'lzop';
5662     } else {
5663         $format = 'vma' if !$format; # default
5664     }
5665
5666     # try to detect archive format
5667     if ($format eq 'tar') {
5668         return restore_tar_archive($archive, $vmid, $user, $opts);
5669     } else {
5670         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5671     }
5672 }
5673
5674 # hepler to remove disks that will not be used after restore
5675 my $restore_cleanup_oldconf = sub {
5676     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5677
5678     PVE::QemuConfig->foreach_volume($oldconf, sub {
5679         my ($ds, $drive) = @_;
5680
5681         return if drive_is_cdrom($drive, 1);
5682
5683         my $volid = $drive->{file};
5684         return if !$volid || $volid =~ m|^/|;
5685
5686         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5687         return if !$path || !$owner || ($owner != $vmid);
5688
5689         # Note: only delete disk we want to restore
5690         # other volumes will become unused
5691         if ($virtdev_hash->{$ds}) {
5692             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5693             if (my $err = $@) {
5694                 warn $err;
5695             }
5696         }
5697     });
5698
5699     # delete vmstate files, after the restore we have no snapshots anymore
5700     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5701         my $snap = $oldconf->{snapshots}->{$snapname};
5702         if ($snap->{vmstate}) {
5703             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5704             if (my $err = $@) {
5705                 warn $err;
5706             }
5707         }
5708     }
5709 };
5710
5711 # Helper to parse vzdump backup device hints
5712 #
5713 # $rpcenv: Environment, used to ckeck storage permissions
5714 # $user: User ID, to check storage permissions
5715 # $storecfg: Storage configuration
5716 # $fh: the file handle for reading the configuration
5717 # $devinfo: should contain device sizes for all backu-up'ed devices
5718 # $options: backup options (pool, default storage)
5719 #
5720 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5721 my $parse_backup_hints = sub {
5722     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5723
5724     my $virtdev_hash = {};
5725
5726     while (defined(my $line = <$fh>)) {
5727         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5728             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5729             die "archive does not contain data for drive '$virtdev'\n"
5730                 if !$devinfo->{$devname};
5731
5732             if (defined($options->{storage})) {
5733                 $storeid = $options->{storage} || 'local';
5734             } elsif (!$storeid) {
5735                 $storeid = 'local';
5736             }
5737             $format = 'raw' if !$format;
5738             $devinfo->{$devname}->{devname} = $devname;
5739             $devinfo->{$devname}->{virtdev} = $virtdev;
5740             $devinfo->{$devname}->{format} = $format;
5741             $devinfo->{$devname}->{storeid} = $storeid;
5742
5743             # check permission on storage
5744             my $pool = $options->{pool}; # todo: do we need that?
5745             if ($user ne 'root@pam') {
5746                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5747             }
5748
5749             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5750         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5751             my $virtdev = $1;
5752             my $drive = parse_drive($virtdev, $2);
5753             if (drive_is_cloudinit($drive)) {
5754                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5755                 $storeid = $options->{storage} if defined ($options->{storage});
5756                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5757                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5758
5759                 $virtdev_hash->{$virtdev} = {
5760                     format => $format,
5761                     storeid => $storeid,
5762                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5763                     is_cloudinit => 1,
5764                 };
5765             }
5766         }
5767     }
5768
5769     return $virtdev_hash;
5770 };
5771
5772 # Helper to allocate and activate all volumes required for a restore
5773 #
5774 # $storecfg: Storage configuration
5775 # $virtdev_hash: as returned by parse_backup_hints()
5776 #
5777 # Returns: { $virtdev => $volid }
5778 my $restore_allocate_devices = sub {
5779     my ($storecfg, $virtdev_hash, $vmid) = @_;
5780
5781     my $map = {};
5782     foreach my $virtdev (sort keys %$virtdev_hash) {
5783         my $d = $virtdev_hash->{$virtdev};
5784         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5785         my $storeid = $d->{storeid};
5786         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5787
5788         # test if requested format is supported
5789         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5790         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5791         $d->{format} = $defFormat if !$supported;
5792
5793         my $name;
5794         if ($d->{is_cloudinit}) {
5795             $name = "vm-$vmid-cloudinit";
5796             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5797         }
5798
5799         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5800
5801         print STDERR "new volume ID is '$volid'\n";
5802         $d->{volid} = $volid;
5803
5804         PVE::Storage::activate_volumes($storecfg, [$volid]);
5805
5806         $map->{$virtdev} = $volid;
5807     }
5808
5809     return $map;
5810 };
5811
5812 my $restore_update_config_line = sub {
5813     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5814
5815     return if $line =~ m/^\#qmdump\#/;
5816     return if $line =~ m/^\#vzdump\#/;
5817     return if $line =~ m/^lock:/;
5818     return if $line =~ m/^unused\d+:/;
5819     return if $line =~ m/^parent:/;
5820
5821     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5822     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5823         # try to convert old 1.X settings
5824         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5825         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5826             my ($model, $macaddr) = split(/\=/, $devconfig);
5827             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5828             my $net = {
5829                 model => $model,
5830                 bridge => "vmbr$ind",
5831                 macaddr => $macaddr,
5832             };
5833             my $netstr = print_net($net);
5834
5835             print $outfd "net$cookie->{netcount}: $netstr\n";
5836             $cookie->{netcount}++;
5837         }
5838     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5839         my ($id, $netstr) = ($1, $2);
5840         my $net = parse_net($netstr);
5841         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5842         $netstr = print_net($net);
5843         print $outfd "$id: $netstr\n";
5844     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5845         my $virtdev = $1;
5846         my $value = $3;
5847         my $di = parse_drive($virtdev, $value);
5848         if (defined($di->{backup}) && !$di->{backup}) {
5849             print $outfd "#$line";
5850         } elsif ($map->{$virtdev}) {
5851             delete $di->{format}; # format can change on restore
5852             $di->{file} = $map->{$virtdev};
5853             $value = print_drive($di);
5854             print $outfd "$virtdev: $value\n";
5855         } else {
5856             print $outfd $line;
5857         }
5858     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5859         my $vmgenid = $1;
5860         if ($vmgenid ne '0') {
5861             # always generate a new vmgenid if there was a valid one setup
5862             $vmgenid = generate_uuid();
5863         }
5864         print $outfd "vmgenid: $vmgenid\n";
5865     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5866         my ($uuid, $uuid_str);
5867         UUID::generate($uuid);
5868         UUID::unparse($uuid, $uuid_str);
5869         my $smbios1 = parse_smbios1($2);
5870         $smbios1->{uuid} = $uuid_str;
5871         print $outfd $1.print_smbios1($smbios1)."\n";
5872     } else {
5873         print $outfd $line;
5874     }
5875 };
5876
5877 my $restore_deactivate_volumes = sub {
5878     my ($storecfg, $devinfo) = @_;
5879
5880     my $vollist = [];
5881     foreach my $devname (keys %$devinfo) {
5882         my $volid = $devinfo->{$devname}->{volid};
5883         push @$vollist, $volid if $volid;
5884     }
5885
5886     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5887 };
5888
5889 my $restore_destroy_volumes = sub {
5890     my ($storecfg, $devinfo) = @_;
5891
5892     foreach my $devname (keys %$devinfo) {
5893         my $volid = $devinfo->{$devname}->{volid};
5894         next if !$volid;
5895         eval {
5896             if ($volid =~ m|^/|) {
5897                 unlink $volid || die 'unlink failed\n';
5898             } else {
5899                 PVE::Storage::vdisk_free($storecfg, $volid);
5900             }
5901             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5902         };
5903         print STDERR "unable to cleanup '$volid' - $@" if $@;
5904     }
5905 };
5906
5907 sub scan_volids {
5908     my ($cfg, $vmid) = @_;
5909
5910     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5911
5912     my $volid_hash = {};
5913     foreach my $storeid (keys %$info) {
5914         foreach my $item (@{$info->{$storeid}}) {
5915             next if !($item->{volid} && $item->{size});
5916             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5917             $volid_hash->{$item->{volid}} = $item;
5918         }
5919     }
5920
5921     return $volid_hash;
5922 }
5923
5924 sub update_disk_config {
5925     my ($vmid, $conf, $volid_hash) = @_;
5926
5927     my $changes;
5928     my $prefix = "VM $vmid:";
5929
5930     # used and unused disks
5931     my $referenced = {};
5932
5933     # Note: it is allowed to define multiple storages with same path (alias), so
5934     # we need to check both 'volid' and real 'path' (two different volid can point
5935     # to the same path).
5936
5937     my $referencedpath = {};
5938
5939     # update size info
5940     PVE::QemuConfig->foreach_volume($conf, sub {
5941         my ($opt, $drive) = @_;
5942
5943         my $volid = $drive->{file};
5944         return if !$volid;
5945
5946         # mark volid as "in-use" for next step
5947         $referenced->{$volid} = 1;
5948         if ($volid_hash->{$volid} &&
5949             (my $path = $volid_hash->{$volid}->{path})) {
5950             $referencedpath->{$path} = 1;
5951         }
5952
5953         return if drive_is_cdrom($drive);
5954         return if !$volid_hash->{$volid};
5955
5956         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5957         if (defined($updated)) {
5958             $changes = 1;
5959             $conf->{$opt} = print_drive($updated);
5960             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5961         }
5962     });
5963
5964     # remove 'unusedX' entry if volume is used
5965     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5966         my ($opt, $drive) = @_;
5967
5968         my $volid = $drive->{file};
5969         return if !$volid;
5970
5971         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5972         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5973             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5974             $changes = 1;
5975             delete $conf->{$opt};
5976         }
5977
5978         $referenced->{$volid} = 1;
5979         $referencedpath->{$path} = 1 if $path;
5980     });
5981
5982     foreach my $volid (sort keys %$volid_hash) {
5983         next if $volid =~ m/vm-$vmid-state-/;
5984         next if $referenced->{$volid};
5985         my $path = $volid_hash->{$volid}->{path};
5986         next if !$path; # just to be sure
5987         next if $referencedpath->{$path};
5988         $changes = 1;
5989         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5990         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5991         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5992     }
5993
5994     return $changes;
5995 }
5996
5997 sub rescan {
5998     my ($vmid, $nolock, $dryrun) = @_;
5999
6000     my $cfg = PVE::Storage::config();
6001
6002     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6003     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6004     foreach my $stor (keys %{$cfg->{ids}}) {
6005         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6006     }
6007
6008     print "rescan volumes...\n";
6009     my $volid_hash = scan_volids($cfg, $vmid);
6010
6011     my $updatefn =  sub {
6012         my ($vmid) = @_;
6013
6014         my $conf = PVE::QemuConfig->load_config($vmid);
6015
6016         PVE::QemuConfig->check_lock($conf);
6017
6018         my $vm_volids = {};
6019         foreach my $volid (keys %$volid_hash) {
6020             my $info = $volid_hash->{$volid};
6021             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6022         }
6023
6024         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6025
6026         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6027     };
6028
6029     if (defined($vmid)) {
6030         if ($nolock) {
6031             &$updatefn($vmid);
6032         } else {
6033             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6034         }
6035     } else {
6036         my $vmlist = config_list();
6037         foreach my $vmid (keys %$vmlist) {
6038             if ($nolock) {
6039                 &$updatefn($vmid);
6040             } else {
6041                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6042             }
6043         }
6044     }
6045 }
6046
6047 sub restore_proxmox_backup_archive {
6048     my ($archive, $vmid, $user, $options) = @_;
6049
6050     my $storecfg = PVE::Storage::config();
6051
6052     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6053     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6054
6055     my $server = $scfg->{server};
6056     my $datastore = $scfg->{datastore};
6057     my $username = $scfg->{username} // 'root@pam';
6058     my $fingerprint = $scfg->{fingerprint};
6059
6060     my $repo = "$username\@$server:$datastore";
6061     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6062     local $ENV{PBS_PASSWORD} = $password;
6063     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6064
6065     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6066         PVE::Storage::parse_volname($storecfg, $archive);
6067
6068     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6069
6070     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6071
6072     my $tmpdir = "/var/tmp/vzdumptmp$$";
6073     rmtree $tmpdir;
6074     mkpath $tmpdir;
6075
6076     my $conffile = PVE::QemuConfig->config_file($vmid);
6077     my $tmpfn = "$conffile.$$.tmp";
6078      # disable interrupts (always do cleanups)
6079     local $SIG{INT} =
6080         local $SIG{TERM} =
6081         local $SIG{QUIT} =
6082         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6083
6084     # Note: $oldconf is undef if VM does not exists
6085     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6086     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6087
6088     my $rpcenv = PVE::RPCEnvironment::get();
6089     my $devinfo = {};
6090
6091     eval {
6092         # enable interrupts
6093         local $SIG{INT} =
6094             local $SIG{TERM} =
6095             local $SIG{QUIT} =
6096             local $SIG{HUP} =
6097             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6098
6099         my $cfgfn = "$tmpdir/qemu-server.conf";
6100         my $firewall_config_fn = "$tmpdir/fw.conf";
6101         my $index_fn = "$tmpdir/index.json";
6102
6103         my $cmd = "restore";
6104
6105         my $param = [$pbs_backup_name, "index.json", $index_fn];
6106         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6107         my $index = PVE::Tools::file_get_contents($index_fn);
6108         $index = decode_json($index);
6109
6110         # print Dumper($index);
6111         foreach my $info (@{$index->{files}}) {
6112             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6113                 my $devname = $1;
6114                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6115                     $devinfo->{$devname}->{size} = $1;
6116                 } else {
6117                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6118                 }
6119             }
6120         }
6121
6122         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6123         if (!$is_qemu_server_backup) {
6124             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6125         }
6126         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6127
6128         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6129         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6130
6131         if ($has_firewall_config) {
6132             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6133             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6134
6135             my $pve_firewall_dir = '/etc/pve/firewall';
6136             mkdir $pve_firewall_dir; # make sure the dir exists
6137             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6138         }
6139
6140         my $fh = IO::File->new($cfgfn, "r") ||
6141             "unable to read qemu-server.conf - $!\n";
6142
6143         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6144
6145         # fixme: rate limit?
6146
6147         # create empty/temp config
6148         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6149
6150         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6151
6152         # allocate volumes
6153         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6154
6155         foreach my $virtdev (sort keys %$virtdev_hash) {
6156             my $d = $virtdev_hash->{$virtdev};
6157             next if $d->{is_cloudinit}; # no need to restore cloudinit
6158
6159             my $volid = $d->{volid};
6160
6161             my $path = PVE::Storage::path($storecfg, $volid);
6162
6163             my $pbs_restore_cmd = [
6164                 '/usr/bin/pbs-restore',
6165                 '--repository', $repo,
6166                 $pbs_backup_name,
6167                 "$d->{devname}.img.fidx",
6168                 $path,
6169                 '--verbose',
6170                 ];
6171
6172             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6173                 push @$pbs_restore_cmd, '--skip-zero';
6174             }
6175
6176             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6177             print "restore proxmox backup image: $dbg_cmdstring\n";
6178             run_command($pbs_restore_cmd);
6179         }
6180
6181         $fh->seek(0, 0) || die "seek failed - $!\n";
6182
6183         my $outfd = new IO::File ($tmpfn, "w") ||
6184             die "unable to write config for VM $vmid\n";
6185
6186         my $cookie = { netcount => 0 };
6187         while (defined(my $line = <$fh>)) {
6188             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6189         }
6190
6191         $fh->close();
6192         $outfd->close();
6193     };
6194     my $err = $@;
6195
6196     $restore_deactivate_volumes->($storecfg, $devinfo);
6197
6198     rmtree $tmpdir;
6199
6200     if ($err) {
6201         unlink $tmpfn;
6202         $restore_destroy_volumes->($storecfg, $devinfo);
6203         die $err;
6204     }
6205
6206     rename($tmpfn, $conffile) ||
6207         die "unable to commit configuration file '$conffile'\n";
6208
6209     PVE::Cluster::cfs_update(); # make sure we read new file
6210
6211     eval { rescan($vmid, 1); };
6212     warn $@ if $@;
6213 }
6214
6215 sub restore_vma_archive {
6216     my ($archive, $vmid, $user, $opts, $comp) = @_;
6217
6218     my $readfrom = $archive;
6219
6220     my $cfg = PVE::Storage::config();
6221     my $commands = [];
6222     my $bwlimit = $opts->{bwlimit};
6223
6224     my $dbg_cmdstring = '';
6225     my $add_pipe = sub {
6226         my ($cmd) = @_;
6227         push @$commands, $cmd;
6228         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6229         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6230         $readfrom = '-';
6231     };
6232
6233     my $input = undef;
6234     if ($archive eq '-') {
6235         $input = '<&STDIN';
6236     } else {
6237         # If we use a backup from a PVE defined storage we also consider that
6238         # storage's rate limit:
6239         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6240         if (defined($volid)) {
6241             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6242             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6243             if ($readlimit) {
6244                 print STDERR "applying read rate limit: $readlimit\n";
6245                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6246                 $add_pipe->($cstream);
6247             }
6248         }
6249     }
6250
6251     if ($comp) {
6252         my $cmd;
6253         if ($comp eq 'gzip') {
6254             $cmd = ['zcat', $readfrom];
6255         } elsif ($comp eq 'lzop') {
6256             $cmd = ['lzop', '-d', '-c', $readfrom];
6257         } else {
6258             die "unknown compression method '$comp'\n";
6259         }
6260         $add_pipe->($cmd);
6261     }
6262
6263     my $tmpdir = "/var/tmp/vzdumptmp$$";
6264     rmtree $tmpdir;
6265
6266     # disable interrupts (always do cleanups)
6267     local $SIG{INT} =
6268         local $SIG{TERM} =
6269         local $SIG{QUIT} =
6270         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6271
6272     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6273     POSIX::mkfifo($mapfifo, 0600);
6274     my $fifofh;
6275
6276     my $openfifo = sub {
6277         open($fifofh, '>', $mapfifo) || die $!;
6278     };
6279
6280     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6281
6282     my $oldtimeout;
6283     my $timeout = 5;
6284
6285     my $devinfo = {};
6286
6287     my $rpcenv = PVE::RPCEnvironment::get();
6288
6289     my $conffile = PVE::QemuConfig->config_file($vmid);
6290     my $tmpfn = "$conffile.$$.tmp";
6291
6292     # Note: $oldconf is undef if VM does not exist
6293     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6294     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6295
6296     my %storage_limits;
6297
6298     my $print_devmap = sub {
6299         my $cfgfn = "$tmpdir/qemu-server.conf";
6300
6301         # we can read the config - that is already extracted
6302         my $fh = IO::File->new($cfgfn, "r") ||
6303             "unable to read qemu-server.conf - $!\n";
6304
6305         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6306         if (-f $fwcfgfn) {
6307             my $pve_firewall_dir = '/etc/pve/firewall';
6308             mkdir $pve_firewall_dir; # make sure the dir exists
6309             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6310         }
6311
6312         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6313
6314         foreach my $key (keys %storage_limits) {
6315             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6316             next if !$limit;
6317             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6318             $storage_limits{$key} = $limit * 1024;
6319         }
6320
6321         foreach my $devname (keys %$devinfo) {
6322             die "found no device mapping information for device '$devname'\n"
6323                 if !$devinfo->{$devname}->{virtdev};
6324         }
6325
6326         # create empty/temp config
6327         if ($oldconf) {
6328             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6329             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6330         }
6331
6332         # allocate volumes
6333         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6334
6335         # print restore information to $fifofh
6336         foreach my $virtdev (sort keys %$virtdev_hash) {
6337             my $d = $virtdev_hash->{$virtdev};
6338             next if $d->{is_cloudinit}; # no need to restore cloudinit
6339
6340             my $storeid = $d->{storeid};
6341             my $volid = $d->{volid};
6342
6343             my $map_opts = '';
6344             if (my $limit = $storage_limits{$storeid}) {
6345                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6346             }
6347
6348             my $write_zeros = 1;
6349             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6350                 $write_zeros = 0;
6351             }
6352
6353             my $path = PVE::Storage::path($cfg, $volid);
6354
6355             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6356
6357             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6358         }
6359
6360         $fh->seek(0, 0) || die "seek failed - $!\n";
6361
6362         my $outfd = new IO::File ($tmpfn, "w") ||
6363             die "unable to write config for VM $vmid\n";
6364
6365         my $cookie = { netcount => 0 };
6366         while (defined(my $line = <$fh>)) {
6367             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6368         }
6369
6370         $fh->close();
6371         $outfd->close();
6372     };
6373
6374     eval {
6375         # enable interrupts
6376         local $SIG{INT} =
6377             local $SIG{TERM} =
6378             local $SIG{QUIT} =
6379             local $SIG{HUP} =
6380             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6381         local $SIG{ALRM} = sub { die "got timeout\n"; };
6382
6383         $oldtimeout = alarm($timeout);
6384
6385         my $parser = sub {
6386             my $line = shift;
6387
6388             print "$line\n";
6389
6390             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6391                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6392                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6393             } elsif ($line =~ m/^CTIME: /) {
6394                 # we correctly received the vma config, so we can disable
6395                 # the timeout now for disk allocation (set to 10 minutes, so
6396                 # that we always timeout if something goes wrong)
6397                 alarm(600);
6398                 &$print_devmap();
6399                 print $fifofh "done\n";
6400                 my $tmp = $oldtimeout || 0;
6401                 $oldtimeout = undef;
6402                 alarm($tmp);
6403                 close($fifofh);
6404             }
6405         };
6406
6407         print "restore vma archive: $dbg_cmdstring\n";
6408         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6409     };
6410     my $err = $@;
6411
6412     alarm($oldtimeout) if $oldtimeout;
6413
6414     $restore_deactivate_volumes->($cfg, $devinfo);
6415
6416     unlink $mapfifo;
6417     rmtree $tmpdir;
6418
6419     if ($err) {
6420         unlink $tmpfn;
6421         $restore_destroy_volumes->($cfg, $devinfo);
6422         die $err;
6423     }
6424
6425     rename($tmpfn, $conffile) ||
6426         die "unable to commit configuration file '$conffile'\n";
6427
6428     PVE::Cluster::cfs_update(); # make sure we read new file
6429
6430     eval { rescan($vmid, 1); };
6431     warn $@ if $@;
6432 }
6433
6434 sub restore_tar_archive {
6435     my ($archive, $vmid, $user, $opts) = @_;
6436
6437     if ($archive ne '-') {
6438         my $firstfile = tar_archive_read_firstfile($archive);
6439         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6440             if $firstfile ne 'qemu-server.conf';
6441     }
6442
6443     my $storecfg = PVE::Storage::config();
6444
6445     # avoid zombie disks when restoring over an existing VM -> cleanup first
6446     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6447     # skiplock=1 because qmrestore has set the 'create' lock itself already
6448     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6449     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6450
6451     my $tocmd = "/usr/lib/qemu-server/qmextract";
6452
6453     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6454     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6455     $tocmd .= ' --prealloc' if $opts->{prealloc};
6456     $tocmd .= ' --info' if $opts->{info};
6457
6458     # tar option "xf" does not autodetect compression when read from STDIN,
6459     # so we pipe to zcat
6460     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6461         PVE::Tools::shellquote("--to-command=$tocmd");
6462
6463     my $tmpdir = "/var/tmp/vzdumptmp$$";
6464     mkpath $tmpdir;
6465
6466     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6467     local $ENV{VZDUMP_VMID} = $vmid;
6468     local $ENV{VZDUMP_USER} = $user;
6469
6470     my $conffile = PVE::QemuConfig->config_file($vmid);
6471     my $tmpfn = "$conffile.$$.tmp";
6472
6473     # disable interrupts (always do cleanups)
6474     local $SIG{INT} =
6475         local $SIG{TERM} =
6476         local $SIG{QUIT} =
6477         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6478
6479     eval {
6480         # enable interrupts
6481         local $SIG{INT} =
6482             local $SIG{TERM} =
6483             local $SIG{QUIT} =
6484             local $SIG{HUP} =
6485             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6486
6487         if ($archive eq '-') {
6488             print "extracting archive from STDIN\n";
6489             run_command($cmd, input => "<&STDIN");
6490         } else {
6491             print "extracting archive '$archive'\n";
6492             run_command($cmd);
6493         }
6494
6495         return if $opts->{info};
6496
6497         # read new mapping
6498         my $map = {};
6499         my $statfile = "$tmpdir/qmrestore.stat";
6500         if (my $fd = IO::File->new($statfile, "r")) {
6501             while (defined (my $line = <$fd>)) {
6502                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6503                     $map->{$1} = $2 if $1;
6504                 } else {
6505                     print STDERR "unable to parse line in statfile - $line\n";
6506                 }
6507             }
6508             $fd->close();
6509         }
6510
6511         my $confsrc = "$tmpdir/qemu-server.conf";
6512
6513         my $srcfd = new IO::File($confsrc, "r") ||
6514             die "unable to open file '$confsrc'\n";
6515
6516         my $outfd = new IO::File ($tmpfn, "w") ||
6517             die "unable to write config for VM $vmid\n";
6518
6519         my $cookie = { netcount => 0 };
6520         while (defined (my $line = <$srcfd>)) {
6521             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6522         }
6523
6524         $srcfd->close();
6525         $outfd->close();
6526     };
6527     if (my $err = $@) {
6528         unlink $tmpfn;
6529         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6530         die $err;
6531     }
6532
6533     rmtree $tmpdir;
6534
6535     rename $tmpfn, $conffile ||
6536         die "unable to commit configuration file '$conffile'\n";
6537
6538     PVE::Cluster::cfs_update(); # make sure we read new file
6539
6540     eval { rescan($vmid, 1); };
6541     warn $@ if $@;
6542 };
6543
6544 sub foreach_storage_used_by_vm {
6545     my ($conf, $func) = @_;
6546
6547     my $sidhash = {};
6548
6549     PVE::QemuConfig->foreach_volume($conf, sub {
6550         my ($ds, $drive) = @_;
6551         return if drive_is_cdrom($drive);
6552
6553         my $volid = $drive->{file};
6554
6555         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6556         $sidhash->{$sid} = $sid if $sid;
6557     });
6558
6559     foreach my $sid (sort keys %$sidhash) {
6560         &$func($sid);
6561     }
6562 }
6563
6564 my $qemu_snap_storage = {
6565     rbd => 1,
6566 };
6567 sub do_snapshots_with_qemu {
6568     my ($storecfg, $volid) = @_;
6569
6570     my $storage_name = PVE::Storage::parse_volume_id($volid);
6571     my $scfg = $storecfg->{ids}->{$storage_name};
6572
6573     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6574         return 1;
6575     }
6576
6577     if ($volid =~ m/\.(qcow2|qed)$/){
6578         return 1;
6579     }
6580
6581     return undef;
6582 }
6583
6584 sub qga_check_running {
6585     my ($vmid, $nowarn) = @_;
6586
6587     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6588     if ($@) {
6589         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6590         return 0;
6591     }
6592     return 1;
6593 }
6594
6595 sub template_create {
6596     my ($vmid, $conf, $disk) = @_;
6597
6598     my $storecfg = PVE::Storage::config();
6599
6600     PVE::QemuConfig->foreach_volume($conf, sub {
6601         my ($ds, $drive) = @_;
6602
6603         return if drive_is_cdrom($drive);
6604         return if $disk && $ds ne $disk;
6605
6606         my $volid = $drive->{file};
6607         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6608
6609         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6610         $drive->{file} = $voliddst;
6611         $conf->{$ds} = print_drive($drive);
6612         PVE::QemuConfig->write_config($vmid, $conf);
6613     });
6614 }
6615
6616 sub convert_iscsi_path {
6617     my ($path) = @_;
6618
6619     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6620         my $portal = $1;
6621         my $target = $2;
6622         my $lun = $3;
6623
6624         my $initiator_name = get_initiator_name();
6625
6626         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6627                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6628     }
6629
6630     die "cannot convert iscsi path '$path', unkown format\n";
6631 }
6632
6633 sub qemu_img_convert {
6634     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6635
6636     my $storecfg = PVE::Storage::config();
6637     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6638     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6639
6640     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6641
6642     my $cachemode;
6643     my $src_path;
6644     my $src_is_iscsi = 0;
6645     my $src_format;
6646
6647     if ($src_storeid) {
6648         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6649         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6650         $src_format = qemu_img_format($src_scfg, $src_volname);
6651         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6652         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6653         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6654     } elsif (-f $src_volid) {
6655         $src_path = $src_volid;
6656         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6657             $src_format = $1;
6658         }
6659     }
6660
6661     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6662
6663     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6664     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6665     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6666     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6667
6668     my $cmd = [];
6669     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6670     push @$cmd, '-l', "snapshot.name=$snapname"
6671         if $snapname && $src_format && $src_format eq "qcow2";
6672     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6673     push @$cmd, '-T', $cachemode if defined($cachemode);
6674
6675     if ($src_is_iscsi) {
6676         push @$cmd, '--image-opts';
6677         $src_path = convert_iscsi_path($src_path);
6678     } elsif ($src_format) {
6679         push @$cmd, '-f', $src_format;
6680     }
6681
6682     if ($dst_is_iscsi) {
6683         push @$cmd, '--target-image-opts';
6684         $dst_path = convert_iscsi_path($dst_path);
6685     } else {
6686         push @$cmd, '-O', $dst_format;
6687     }
6688
6689     push @$cmd, $src_path;
6690
6691     if (!$dst_is_iscsi && $is_zero_initialized) {
6692         push @$cmd, "zeroinit:$dst_path";
6693     } else {
6694         push @$cmd, $dst_path;
6695     }
6696
6697     my $parser = sub {
6698         my $line = shift;
6699         if($line =~ m/\((\S+)\/100\%\)/){
6700             my $percent = $1;
6701             my $transferred = int($size * $percent / 100);
6702             my $remaining = $size - $transferred;
6703
6704             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6705         }
6706
6707     };
6708
6709     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6710     my $err = $@;
6711     die "copy failed: $err" if $err;
6712 }
6713
6714 sub qemu_img_format {
6715     my ($scfg, $volname) = @_;
6716
6717     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6718         return $1;
6719     } else {
6720         return "raw";
6721     }
6722 }
6723
6724 sub qemu_drive_mirror {
6725     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6726
6727     $jobs = {} if !$jobs;
6728
6729     my $qemu_target;
6730     my $format;
6731     $jobs->{"drive-$drive"} = {};
6732
6733     if ($dst_volid =~ /^nbd:/) {
6734         $qemu_target = $dst_volid;
6735         $format = "nbd";
6736     } else {
6737         my $storecfg = PVE::Storage::config();
6738         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6739
6740         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6741
6742         $format = qemu_img_format($dst_scfg, $dst_volname);
6743
6744         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6745
6746         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6747     }
6748
6749     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6750     $opts->{format} = $format if $format;
6751
6752     if (defined($src_bitmap)) {
6753         $opts->{sync} = 'incremental';
6754         $opts->{bitmap} = $src_bitmap;
6755         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6756     }
6757
6758     if (defined($bwlimit)) {
6759         $opts->{speed} = $bwlimit * 1024;
6760         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6761     } else {
6762         print "drive mirror is starting for drive-$drive\n";
6763     }
6764
6765     # if a job already runs for this device we get an error, catch it for cleanup
6766     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6767     if (my $err = $@) {
6768         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6769         warn "$@\n" if $@;
6770         die "mirroring error: $err\n";
6771     }
6772
6773     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6774 }
6775
6776 # $completion can be either
6777 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6778 # 'cancel': wait until all jobs are ready, block-job-cancel them
6779 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6780 sub qemu_drive_mirror_monitor {
6781     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6782
6783     $completion //= 'complete';
6784
6785     eval {
6786         my $err_complete = 0;
6787
6788         while (1) {
6789             die "storage migration timed out\n" if $err_complete > 300;
6790
6791             my $stats = mon_cmd($vmid, "query-block-jobs");
6792
6793             my $running_mirror_jobs = {};
6794             foreach my $stat (@$stats) {
6795                 next if $stat->{type} ne 'mirror';
6796                 $running_mirror_jobs->{$stat->{device}} = $stat;
6797             }
6798
6799             my $readycounter = 0;
6800
6801             foreach my $job (keys %$jobs) {
6802
6803                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6804                     print "$job : finished\n";
6805                     delete $jobs->{$job};
6806                     next;
6807                 }
6808
6809                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6810
6811                 my $busy = $running_mirror_jobs->{$job}->{busy};
6812                 my $ready = $running_mirror_jobs->{$job}->{ready};
6813                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6814                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6815                     my $remaining = $total - $transferred;
6816                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6817
6818                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6819                 }
6820
6821                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6822             }
6823
6824             last if scalar(keys %$jobs) == 0;
6825
6826             if ($readycounter == scalar(keys %$jobs)) {
6827                 print "all mirroring jobs are ready \n";
6828                 last if $completion eq 'skip'; #do the complete later
6829
6830                 if ($vmiddst && $vmiddst != $vmid) {
6831                     my $agent_running = $qga && qga_check_running($vmid);
6832                     if ($agent_running) {
6833                         print "freeze filesystem\n";
6834                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6835                     } else {
6836                         print "suspend vm\n";
6837                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6838                     }
6839
6840                     # if we clone a disk for a new target vm, we don't switch the disk
6841                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6842
6843                     if ($agent_running) {
6844                         print "unfreeze filesystem\n";
6845                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6846                     } else {
6847                         print "resume vm\n";
6848                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6849                     }
6850
6851                     last;
6852                 } else {
6853
6854                     foreach my $job (keys %$jobs) {
6855                         # try to switch the disk if source and destination are on the same guest
6856                         print "$job: Completing block job...\n";
6857
6858                         my $op;
6859                         if ($completion eq 'complete') {
6860                             $op = 'block-job-complete';
6861                         } elsif ($completion eq 'cancel') {
6862                             $op = 'block-job-cancel';
6863                         } else {
6864                             die "invalid completion value: $completion\n";
6865                         }
6866                         eval { mon_cmd($vmid, $op, device => $job) };
6867                         if ($@ =~ m/cannot be completed/) {
6868                             print "$job: Block job cannot be completed, try again.\n";
6869                             $err_complete++;
6870                         }else {
6871                             print "$job: Completed successfully.\n";
6872                             $jobs->{$job}->{complete} = 1;
6873                         }
6874                     }
6875                 }
6876             }
6877             sleep 1;
6878         }
6879     };
6880     my $err = $@;
6881
6882     if ($err) {
6883         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6884         die "mirroring error: $err";
6885     }
6886
6887 }
6888
6889 sub qemu_blockjobs_cancel {
6890     my ($vmid, $jobs) = @_;
6891
6892     foreach my $job (keys %$jobs) {
6893         print "$job: Cancelling block job\n";
6894         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6895         $jobs->{$job}->{cancel} = 1;
6896     }
6897
6898     while (1) {
6899         my $stats = mon_cmd($vmid, "query-block-jobs");
6900
6901         my $running_jobs = {};
6902         foreach my $stat (@$stats) {
6903             $running_jobs->{$stat->{device}} = $stat;
6904         }
6905
6906         foreach my $job (keys %$jobs) {
6907
6908             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6909                 print "$job: Done.\n";
6910                 delete $jobs->{$job};
6911             }
6912         }
6913
6914         last if scalar(keys %$jobs) == 0;
6915
6916         sleep 1;
6917     }
6918 }
6919
6920 sub clone_disk {
6921     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6922         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6923
6924     my $newvolid;
6925
6926     if (!$full) {
6927         print "create linked clone of drive $drivename ($drive->{file})\n";
6928         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6929         push @$newvollist, $newvolid;
6930     } else {
6931
6932         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6933         $storeid = $storage if $storage;
6934
6935         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6936         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6937
6938         print "create full clone of drive $drivename ($drive->{file})\n";
6939         my $name = undef;
6940         if (drive_is_cloudinit($drive)) {
6941             $name = "vm-$newvmid-cloudinit";
6942             $name .= ".$dst_format" if $dst_format ne 'raw';
6943             $snapname = undef;
6944             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6945         } elsif ($drivename eq 'efidisk0') {
6946             $size = get_efivars_size($conf);
6947         }
6948         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6949         push @$newvollist, $newvolid;
6950
6951         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6952
6953         if (drive_is_cloudinit($drive)) {
6954             goto no_data_clone;
6955         }
6956
6957         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6958         if (!$running || $snapname) {
6959             # TODO: handle bwlimits
6960             if ($drivename eq 'efidisk0') {
6961                 # the relevant data on the efidisk may be smaller than the source
6962                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6963                 # that is given by the OVMF_VARS.fd
6964                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6965                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6966                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6967             } else {
6968                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6969             }
6970         } else {
6971
6972             my $kvmver = get_running_qemu_version ($vmid);
6973             if (!min_version($kvmver, 2, 7)) {
6974                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6975                     if $drive->{iothread};
6976             }
6977
6978             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6979         }
6980     }
6981
6982 no_data_clone:
6983     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6984
6985     my $disk = $drive;
6986     $disk->{format} = undef;
6987     $disk->{file} = $newvolid;
6988     $disk->{size} = $size;
6989
6990     return $disk;
6991 }
6992
6993 sub get_running_qemu_version {
6994     my ($vmid) = @_;
6995     my $res = mon_cmd($vmid, "query-version");
6996     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6997 }
6998
6999 sub qemu_use_old_bios_files {
7000     my ($machine_type) = @_;
7001
7002     return if !$machine_type;
7003
7004     my $use_old_bios_files = undef;
7005
7006     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7007         $machine_type = $1;
7008         $use_old_bios_files = 1;
7009     } else {
7010         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7011         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7012         # load new efi bios files on migration. So this hack is required to allow
7013         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7014         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7015         $use_old_bios_files = !min_version($version, 2, 4);
7016     }
7017
7018     return ($use_old_bios_files, $machine_type);
7019 }
7020
7021 sub get_efivars_size {
7022     my ($conf) = @_;
7023     my $arch = get_vm_arch($conf);
7024     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7025     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7026     return -s $ovmf_vars;
7027 }
7028
7029 sub update_efidisk_size {
7030     my ($conf) = @_;
7031
7032     return if !defined($conf->{efidisk0});
7033
7034     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7035     $disk->{size} = get_efivars_size($conf);
7036     $conf->{efidisk0} = print_drive($disk);
7037
7038     return;
7039 }
7040
7041 sub create_efidisk($$$$$) {
7042     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7043
7044     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7045     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7046
7047     my $vars_size_b = -s $ovmf_vars;
7048     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7049     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7050     PVE::Storage::activate_volumes($storecfg, [$volid]);
7051
7052     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7053     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7054
7055     return ($volid, $size/1024);
7056 }
7057
7058 sub vm_iothreads_list {
7059     my ($vmid) = @_;
7060
7061     my $res = mon_cmd($vmid, 'query-iothreads');
7062
7063     my $iothreads = {};
7064     foreach my $iothread (@$res) {
7065         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7066     }
7067
7068     return $iothreads;
7069 }
7070
7071 sub scsihw_infos {
7072     my ($conf, $drive) = @_;
7073
7074     my $maxdev = 0;
7075
7076     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7077         $maxdev = 7;
7078     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7079         $maxdev = 1;
7080     } else {
7081         $maxdev = 256;
7082     }
7083
7084     my $controller = int($drive->{index} / $maxdev);
7085     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7086
7087     return ($maxdev, $controller, $controller_prefix);
7088 }
7089
7090 sub windows_version {
7091     my ($ostype) = @_;
7092
7093     return 0 if !$ostype;
7094
7095     my $winversion = 0;
7096
7097     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7098         $winversion = 5;
7099     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7100         $winversion = 6;
7101     } elsif ($ostype =~ m/^win(\d+)$/) {
7102         $winversion = $1;
7103     }
7104
7105     return $winversion;
7106 }
7107
7108 sub resolve_dst_disk_format {
7109         my ($storecfg, $storeid, $src_volname, $format) = @_;
7110         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7111
7112         if (!$format) {
7113             # if no target format is specified, use the source disk format as hint
7114             if ($src_volname) {
7115                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7116                 $format = qemu_img_format($scfg, $src_volname);
7117             } else {
7118                 return $defFormat;
7119             }
7120         }
7121
7122         # test if requested format is supported - else use default
7123         my $supported = grep { $_ eq $format } @$validFormats;
7124         $format = $defFormat if !$supported;
7125         return $format;
7126 }
7127
7128 # NOTE: if this logic changes, please update docs & possibly gui logic
7129 sub find_vmstate_storage {
7130     my ($conf, $storecfg) = @_;
7131
7132     # first, return storage from conf if set
7133     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7134
7135     my ($target, $shared, $local);
7136
7137     foreach_storage_used_by_vm($conf, sub {
7138         my ($sid) = @_;
7139         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7140         my $dst = $scfg->{shared} ? \$shared : \$local;
7141         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7142     });
7143
7144     # second, use shared storage where VM has at least one disk
7145     # third, use local storage where VM has at least one disk
7146     # fall back to local storage
7147     $target = $shared // $local // 'local';
7148
7149     return $target;
7150 }
7151
7152 sub generate_uuid {
7153     my ($uuid, $uuid_str);
7154     UUID::generate($uuid);
7155     UUID::unparse($uuid, $uuid_str);
7156     return $uuid_str;
7157 }
7158
7159 sub generate_smbios1_uuid {
7160     return "uuid=".generate_uuid();
7161 }
7162
7163 sub nbd_stop {
7164     my ($vmid) = @_;
7165
7166     mon_cmd($vmid, 'nbd-server-stop');
7167 }
7168
7169 sub create_reboot_request {
7170     my ($vmid) = @_;
7171     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7172         or die "failed to create reboot trigger file: $!\n";
7173     close($fh);
7174 }
7175
7176 sub clear_reboot_request {
7177     my ($vmid) = @_;
7178     my $path = "/run/qemu-server/$vmid.reboot";
7179     my $res = 0;
7180
7181     $res = unlink($path);
7182     die "could not remove reboot request for $vmid: $!"
7183         if !$res && $! != POSIX::ENOENT;
7184
7185     return $res;
7186 }
7187
7188 # bash completion helper
7189
7190 sub complete_backup_archives {
7191     my ($cmdname, $pname, $cvalue) = @_;
7192
7193     my $cfg = PVE::Storage::config();
7194
7195     my $storeid;
7196
7197     if ($cvalue =~ m/^([^:]+):/) {
7198         $storeid = $1;
7199     }
7200
7201     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7202
7203     my $res = [];
7204     foreach my $id (keys %$data) {
7205         foreach my $item (@{$data->{$id}}) {
7206             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7207             push @$res, $item->{volid} if defined($item->{volid});
7208         }
7209     }
7210
7211     return $res;
7212 }
7213
7214 my $complete_vmid_full = sub {
7215     my ($running) = @_;
7216
7217     my $idlist = vmstatus();
7218
7219     my $res = [];
7220
7221     foreach my $id (keys %$idlist) {
7222         my $d = $idlist->{$id};
7223         if (defined($running)) {
7224             next if $d->{template};
7225             next if $running && $d->{status} ne 'running';
7226             next if !$running && $d->{status} eq 'running';
7227         }
7228         push @$res, $id;
7229
7230     }
7231     return $res;
7232 };
7233
7234 sub complete_vmid {
7235     return &$complete_vmid_full();
7236 }
7237
7238 sub complete_vmid_stopped {
7239     return &$complete_vmid_full(0);
7240 }
7241
7242 sub complete_vmid_running {
7243     return &$complete_vmid_full(1);
7244 }
7245
7246 sub complete_storage {
7247
7248     my $cfg = PVE::Storage::config();
7249     my $ids = $cfg->{ids};
7250
7251     my $res = [];
7252     foreach my $sid (keys %$ids) {
7253         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7254         next if !$ids->{$sid}->{content}->{images};
7255         push @$res, $sid;
7256     }
7257
7258     return $res;
7259 }
7260
7261 sub complete_migration_storage {
7262     my ($cmd, $param, $current_value, $all_args) = @_;
7263
7264     my $targetnode = @$all_args[1];
7265
7266     my $cfg = PVE::Storage::config();
7267     my $ids = $cfg->{ids};
7268
7269     my $res = [];
7270     foreach my $sid (keys %$ids) {
7271         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7272         next if !$ids->{$sid}->{content}->{images};
7273         push @$res, $sid;
7274     }
7275
7276     return $res;
7277 }
7278
7279 1;