PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_HOSTPCI_DEVICES = 16;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         pattern => '[-_.\w\d]+',
 850         optional => 1,
 851     },
 852     queues => {
 853         type => 'integer',
 854         minimum => 0, maximum => 16,
 855         description => 'Number of packet queues to be used on the device.',
 856         optional => 1,
 857     },
 858     rate => {
 859         type => 'number',
 860         minimum => 0,
 861         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 862         optional => 1,
 863     },
 864     tag => {
 865         type => 'integer',
 866         minimum => 1, maximum => 4094,
 867         description => 'VLAN tag to apply to packets on this interface.',
 868         optional => 1,
 869     },
 870     trunks => {
 871         type => 'string',
 872         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 873         description => 'VLAN trunks to pass through this interface.',
 874         format_description => 'vlanid[;vlanid...]',
 875         optional => 1,
 876     },
 877     firewall => {
 878         type => 'boolean',
 879         description => 'Whether this interface should be protected by the firewall.',
 880         optional => 1,
 881     },
 882     link_down => {
 883         type => 'boolean',
 884         description => 'Whether this interface should be disconnected (like pulling the plug).',
 885         optional => 1,
 886     },
 887 };
 888
 889 my $netdesc = {
 890     optional => 1,
 891     type => 'string', format => $net_fmt,
 892     description => "Specify network devices.",
 893 };
 894
 895 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 896
 897 my $ipconfig_fmt = {
 898     ip => {
 899         type => 'string',
 900         format => 'pve-ipv4-config',
 901         format_description => 'IPv4Format/CIDR',
 902         description => 'IPv4 address in CIDR format.',
 903         optional => 1,
 904         default => 'dhcp',
 905     },
 906     gw => {
 907         type => 'string',
 908         format => 'ipv4',
 909         format_description => 'GatewayIPv4',
 910         description => 'Default gateway for IPv4 traffic.',
 911         optional => 1,
 912         requires => 'ip',
 913     },
 914     ip6 => {
 915         type => 'string',
 916         format => 'pve-ipv6-config',
 917         format_description => 'IPv6Format/CIDR',
 918         description => 'IPv6 address in CIDR format.',
 919         optional => 1,
 920         default => 'dhcp',
 921     },
 922     gw6 => {
 923         type => 'string',
 924         format => 'ipv6',
 925         format_description => 'GatewayIPv6',
 926         description => 'Default gateway for IPv6 traffic.',
 927         optional => 1,
 928         requires => 'ip6',
 929     },
 930 };
 931 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 932 my $ipconfigdesc = {
 933     optional => 1,
 934     type => 'string', format => 'pve-qm-ipconfig',
 935     description => <<'EODESCR',
 936 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 937
 938 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 939
 940 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 941 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 942
 943 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 944 EODESCR
 945 };
 946 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 947
 948 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 949     $confdesc->{"net$i"} = $netdesc;
 950     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 951 }
 952
 953 foreach my $key (keys %$confdesc_cloudinit) {
 954     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 955 }
 956
 957 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 958 sub verify_volume_id_or_qm_path {
 959     my ($volid, $noerr) = @_;
 960
 961     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 962         return $volid;
 963     }
 964
 965     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 966     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 967     if ($@) {
 968         return undef if $noerr;
 969         die $@;
 970     }
 971     return $volid;
 972 }
 973
 974 my $usb_fmt = {
 975     host => {
 976         default_key => 1,
 977         type => 'string', format => 'pve-qm-usb-device',
 978         format_description => 'HOSTUSBDEVICE|spice',
 979         description => <<EODESCR,
 980 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 981
 982  'bus-port(.port)*' (decimal numbers) or
 983  'vendor_id:product_id' (hexadeciaml numbers) or
 984  'spice'
 985
 986 You can use the 'lsusb -t' command to list existing usb devices.
 987
 988 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 989
 990 The value 'spice' can be used to add a usb redirection devices for spice.
 991 EODESCR
 992     },
 993     usb3 => {
 994         optional => 1,
 995         type => 'boolean',
 996         description => "Specifies whether if given host option is a USB3 device or port.",
 997         default => 0,
 998     },
 999 };
1000
1001 my $usbdesc = {
1002     optional => 1,
1003     type => 'string', format => $usb_fmt,
1004     description => "Configure an USB device (n is 0 to 4).",
1005 };
1006 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1007
1008 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1009 my $hostpci_fmt = {
1010     host => {
1011         default_key => 1,
1012         type => 'string',
1013         pattern => qr/$PCIRE(;$PCIRE)*/,
1014         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1015         description => <<EODESCR,
1016 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1017 of PCI virtual functions of the host. HOSTPCIID syntax is:
1018
1019 'bus:dev.func' (hexadecimal numbers)
1020
1021 You can us the 'lspci' command to list existing PCI devices.
1022 EODESCR
1023     },
1024     rombar => {
1025         type => 'boolean',
1026         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1027         optional => 1,
1028         default => 1,
1029     },
1030     romfile => {
1031         type => 'string',
1032         pattern => '[^,;]+',
1033         format_description => 'string',
1034         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1035         optional => 1,
1036     },
1037     pcie => {
1038         type => 'boolean',
1039         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1040         optional => 1,
1041         default => 0,
1042     },
1043     'x-vga' => {
1044         type => 'boolean',
1045         description =>  "Enable vfio-vga device support.",
1046         optional => 1,
1047         default => 0,
1048     },
1049     'mdev' => {
1050         type => 'string',
1051         format_description => 'string',
1052         pattern => '[^/\.:]+',
1053         optional => 1,
1054         description => <<EODESCR
1055 The type of mediated device to use.
1056 An instance of this type will be created on startup of the VM and
1057 will be cleaned up when the VM stops.
1058 EODESCR
1059     }
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1121     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1122 }
1123
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1125     $confdesc->{"usb$i"} = $usbdesc;
1126 }
1127
1128 my $kvm_api_version = 0;
1129
1130 sub kvm_version {
1131     return $kvm_api_version if $kvm_api_version;
1132
1133     open my $fh, '<', '/dev/kvm'
1134         or return undef;
1135
1136     # 0xae00 => KVM_GET_API_VERSION
1137     $kvm_api_version = ioctl($fh, 0xae00, 0);
1138
1139     return $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version = {};
1143 my $kvm_mtime = {};
1144
1145 sub kvm_user_version {
1146     my ($binary) = @_;
1147
1148     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1149     my $st = stat($binary);
1150
1151     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1152     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1153         $cachedmtime == $st->mtime;
1154
1155     $kvm_user_version->{$binary} = 'unknown';
1156     $kvm_mtime->{$binary} = $st->mtime;
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version->{$binary} = $2;
1162         }
1163     };
1164
1165     eval { run_command([$binary, '--version'], outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version->{$binary};
1169
1170 }
1171
1172 sub kernel_has_vhost_net {
1173     return -c '/dev/vhost-net';
1174 }
1175
1176 sub option_exists {
1177     my $key = shift;
1178     return defined($confdesc->{$key});
1179 }
1180
1181 my $cdrom_path;
1182 sub get_cdrom_path {
1183
1184     return  $cdrom_path if $cdrom_path;
1185
1186     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1187     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1188     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1189 }
1190
1191 sub get_iso_path {
1192     my ($storecfg, $vmid, $cdrom) = @_;
1193
1194     if ($cdrom eq 'cdrom') {
1195         return get_cdrom_path();
1196     } elsif ($cdrom eq 'none') {
1197         return '';
1198     } elsif ($cdrom =~ m|^/|) {
1199         return $cdrom;
1200     } else {
1201         return PVE::Storage::path($storecfg, $cdrom);
1202     }
1203 }
1204
1205 # try to convert old style file names to volume IDs
1206 sub filename_to_volume_id {
1207     my ($vmid, $file, $media) = @_;
1208
1209      if (!($file eq 'none' || $file eq 'cdrom' ||
1210           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1211
1212         return undef if $file =~ m|/|;
1213
1214         if ($media && $media eq 'cdrom') {
1215             $file = "local:iso/$file";
1216         } else {
1217             $file = "local:$vmid/$file";
1218         }
1219     }
1220
1221     return $file;
1222 }
1223
1224 sub verify_media_type {
1225     my ($opt, $vtype, $media) = @_;
1226
1227     return if !$media;
1228
1229     my $etype;
1230     if ($media eq 'disk') {
1231         $etype = 'images';
1232     } elsif ($media eq 'cdrom') {
1233         $etype = 'iso';
1234     } else {
1235         die "internal error";
1236     }
1237
1238     return if ($vtype eq $etype);
1239
1240     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1241 }
1242
1243 sub cleanup_drive_path {
1244     my ($opt, $storecfg, $drive) = @_;
1245
1246     # try to convert filesystem paths to volume IDs
1247
1248     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1249         ($drive->{file} !~ m|^/dev/.+|) &&
1250         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1251         ($drive->{file} !~ m/^\d+$/)) {
1252         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1253         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1254         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1255         verify_media_type($opt, $vtype, $drive->{media});
1256         $drive->{file} = $volid;
1257     }
1258
1259     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1260 }
1261
1262 sub parse_hotplug_features {
1263     my ($data) = @_;
1264
1265     my $res = {};
1266
1267     return $res if $data eq '0';
1268
1269     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1270
1271     foreach my $feature (PVE::Tools::split_list($data)) {
1272         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1273             $res->{$1} = 1;
1274         } else {
1275             die "invalid hotplug feature '$feature'\n";
1276         }
1277     }
1278     return $res;
1279 }
1280
1281 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1282 sub pve_verify_hotplug_features {
1283     my ($value, $noerr) = @_;
1284
1285     return $value if parse_hotplug_features($value);
1286
1287     return undef if $noerr;
1288
1289     die "unable to parse hotplug option\n";
1290 }
1291
1292 sub scsi_inquiry {
1293     my($fh, $noerr) = @_;
1294
1295     my $SG_IO = 0x2285;
1296     my $SG_GET_VERSION_NUM = 0x2282;
1297
1298     my $versionbuf = "\x00" x 8;
1299     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1300     if (!$ret) {
1301         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1302         return undef;
1303     }
1304     my $version = unpack("I", $versionbuf);
1305     if ($version < 30000) {
1306         die "scsi generic interface too old\n"  if !$noerr;
1307         return undef;
1308     }
1309
1310     my $buf = "\x00" x 36;
1311     my $sensebuf = "\x00" x 8;
1312     my $cmd = pack("C x3 C x1", 0x12, 36);
1313
1314     # see /usr/include/scsi/sg.h
1315     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1316
1317     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1318                       length($sensebuf), 0, length($buf), $buf,
1319                       $cmd, $sensebuf, 6000);
1320
1321     $ret = ioctl($fh, $SG_IO, $packet);
1322     if (!$ret) {
1323         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1324         return undef;
1325     }
1326
1327     my @res = unpack($sg_io_hdr_t, $packet);
1328     if ($res[17] || $res[18]) {
1329         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1330         return undef;
1331     }
1332
1333     my $res = {};
1334     (my $byte0, my $byte1, $res->{vendor},
1335      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1336
1337     $res->{removable} = $byte1 & 128 ? 1 : 0;
1338     $res->{type} = $byte0 & 31;
1339
1340     return $res;
1341 }
1342
1343 sub path_is_scsi {
1344     my ($path) = @_;
1345
1346     my $fh = IO::File->new("+<$path") || return undef;
1347     my $res = scsi_inquiry($fh, 1);
1348     close($fh);
1349
1350     return $res;
1351 }
1352
1353 sub print_tabletdevice_full {
1354     my ($conf, $arch) = @_;
1355
1356     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1357
1358     # we use uhci for old VMs because tablet driver was buggy in older qemu
1359     my $usbbus;
1360     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1361         $usbbus = 'ehci';
1362     } else {
1363         $usbbus = 'uhci';
1364     }
1365
1366     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1367 }
1368
1369 sub print_keyboarddevice_full {
1370     my ($conf, $arch, $machine) = @_;
1371
1372     return undef if $arch ne 'aarch64';
1373
1374     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1375 }
1376
1377 sub print_drivedevice_full {
1378     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1379
1380     my $device = '';
1381     my $maxdev = 0;
1382
1383     if ($drive->{interface} eq 'virtio') {
1384         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1385         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1386         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1387     } elsif ($drive->{interface} eq 'scsi') {
1388
1389         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1390         my $unit = $drive->{index} % $maxdev;
1391         my $devicetype = 'hd';
1392         my $path = '';
1393         if (drive_is_cdrom($drive)) {
1394             $devicetype = 'cd';
1395         } else {
1396             if ($drive->{file} =~ m|^/|) {
1397                 $path = $drive->{file};
1398                 if (my $info = path_is_scsi($path)) {
1399                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1400                         $devicetype = 'block';
1401                     } elsif ($info->{type} == 1) { # tape
1402                         $devicetype = 'generic';
1403                     }
1404                 }
1405             } else {
1406                  $path = PVE::Storage::path($storecfg, $drive->{file});
1407             }
1408
1409             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1410             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1411             if ($path =~ m/^iscsi\:\/\// &&
1412                !min_version($version, 4, 1)) {
1413                 $devicetype = 'generic';
1414             }
1415         }
1416
1417         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1418             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1419         } else {
1420             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1421         }
1422
1423         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1424             $device .= ",rotation_rate=1";
1425         }
1426         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1427
1428     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1429         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1430         my $controller = int($drive->{index} / $maxdev);
1431         my $unit = $drive->{index} % $maxdev;
1432         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1433
1434         $device = "ide-$devicetype";
1435         if ($drive->{interface} eq 'ide') {
1436             $device .= ",bus=ide.$controller,unit=$unit";
1437         } else {
1438             $device .= ",bus=ahci$controller.$unit";
1439         }
1440         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1441
1442         if ($devicetype eq 'hd') {
1443             if (my $model = $drive->{model}) {
1444                 $model = URI::Escape::uri_unescape($model);
1445                 $device .= ",model=$model";
1446             }
1447             if ($drive->{ssd}) {
1448                 $device .= ",rotation_rate=1";
1449             }
1450         }
1451         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1452     } elsif ($drive->{interface} eq 'usb') {
1453         die "implement me";
1454         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1455     } else {
1456         die "unsupported interface type";
1457     }
1458
1459     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1460
1461     if (my $serial = $drive->{serial}) {
1462         $serial = URI::Escape::uri_unescape($serial);
1463         $device .= ",serial=$serial";
1464     }
1465
1466
1467     return $device;
1468 }
1469
1470 sub get_initiator_name {
1471     my $initiator;
1472
1473     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1474     while (defined(my $line = <$fh>)) {
1475         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1476         $initiator = $1;
1477         last;
1478     }
1479     $fh->close();
1480
1481     return $initiator;
1482 }
1483
1484 sub print_drive_commandline_full {
1485     my ($storecfg, $vmid, $drive) = @_;
1486
1487     my $path;
1488     my $volid = $drive->{file};
1489     my $format;
1490
1491     if (drive_is_cdrom($drive)) {
1492         $path = get_iso_path($storecfg, $vmid, $volid);
1493     } else {
1494         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1495         if ($storeid) {
1496             $path = PVE::Storage::path($storecfg, $volid);
1497             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1498             $format = qemu_img_format($scfg, $volname);
1499         } else {
1500             $path = $volid;
1501             $format = "raw";
1502         }
1503    }
1504
1505     my $opts = '';
1506     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1507     foreach my $o (@qemu_drive_options) {
1508         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1509     }
1510
1511     # snapshot only accepts on|off
1512     if (defined($drive->{snapshot})) {
1513         my $v = $drive->{snapshot} ? 'on' : 'off';
1514         $opts .= ",snapshot=$v";
1515     }
1516
1517     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1518         my ($dir, $qmpname) = @$type;
1519         if (my $v = $drive->{"mbps$dir"}) {
1520             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1521         }
1522         if (my $v = $drive->{"mbps${dir}_max"}) {
1523             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1524         }
1525         if (my $v = $drive->{"bps${dir}_max_length"}) {
1526             $opts .= ",throttling.bps$qmpname-max-length=$v";
1527         }
1528         if (my $v = $drive->{"iops${dir}"}) {
1529             $opts .= ",throttling.iops$qmpname=$v";
1530         }
1531         if (my $v = $drive->{"iops${dir}_max"}) {
1532             $opts .= ",throttling.iops$qmpname-max=$v";
1533         }
1534         if (my $v = $drive->{"iops${dir}_max_length"}) {
1535             $opts .= ",throttling.iops$qmpname-max-length=$v";
1536         }
1537     }
1538
1539     $opts .= ",format=$format" if $format && !$drive->{format};
1540
1541     my $cache_direct = 0;
1542
1543     if (my $cache = $drive->{cache}) {
1544         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1545     } elsif (!drive_is_cdrom($drive)) {
1546         $opts .= ",cache=none";
1547         $cache_direct = 1;
1548     }
1549
1550     # aio native works only with O_DIRECT
1551     if (!$drive->{aio}) {
1552         if($cache_direct) {
1553             $opts .= ",aio=native";
1554         } else {
1555             $opts .= ",aio=threads";
1556         }
1557     }
1558
1559     if (!drive_is_cdrom($drive)) {
1560         my $detectzeroes;
1561         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1562             $detectzeroes = 'off';
1563         } elsif ($drive->{discard}) {
1564             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1565         } else {
1566             # This used to be our default with discard not being specified:
1567             $detectzeroes = 'on';
1568         }
1569         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1570     }
1571
1572     my $pathinfo = $path ? "file=$path," : '';
1573
1574     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1575 }
1576
1577 sub print_netdevice_full {
1578     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1579
1580     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1581
1582     my $device = $net->{model};
1583     if ($net->{model} eq 'virtio') {
1584          $device = 'virtio-net-pci';
1585      };
1586
1587     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1588     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1589     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1590         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1591         my $vectors = $net->{queues} * 2 + 2;
1592         $tmpstr .= ",vectors=$vectors,mq=on";
1593     }
1594     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1595
1596     if ($use_old_bios_files) {
1597         my $romfile;
1598         if ($device eq 'virtio-net-pci') {
1599             $romfile = 'pxe-virtio.rom';
1600         } elsif ($device eq 'e1000') {
1601             $romfile = 'pxe-e1000.rom';
1602         } elsif ($device eq 'ne2k') {
1603             $romfile = 'pxe-ne2k_pci.rom';
1604         } elsif ($device eq 'pcnet') {
1605             $romfile = 'pxe-pcnet.rom';
1606         } elsif ($device eq 'rtl8139') {
1607             $romfile = 'pxe-rtl8139.rom';
1608         }
1609         $tmpstr .= ",romfile=$romfile" if $romfile;
1610     }
1611
1612     return $tmpstr;
1613 }
1614
1615 sub print_netdev_full {
1616     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1617
1618     my $i = '';
1619     if ($netid =~ m/^net(\d+)$/) {
1620         $i = int($1);
1621     }
1622
1623     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1624
1625     my $ifname = "tap${vmid}i$i";
1626
1627     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1628     die "interface name '$ifname' is too long (max 15 character)\n"
1629         if length($ifname) >= 16;
1630
1631     my $vhostparam = '';
1632     if (is_native($arch)) {
1633         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1634     }
1635
1636     my $vmname = $conf->{name} || "vm$vmid";
1637
1638     my $netdev = "";
1639     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1640
1641     if ($net->{bridge}) {
1642         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1643     } else {
1644         $netdev = "type=user,id=$netid,hostname=$vmname";
1645     }
1646
1647     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1648
1649     return $netdev;
1650 }
1651
1652 my $vga_map = {
1653     'cirrus' => 'cirrus-vga',
1654     'std' => 'VGA',
1655     'vmware' => 'vmware-svga',
1656     'virtio' => 'virtio-vga',
1657 };
1658
1659 sub print_vga_device {
1660     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1661
1662     my $type = $vga_map->{$vga->{type}};
1663     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1664         $type = 'virtio-gpu';
1665     }
1666     my $vgamem_mb = $vga->{memory};
1667
1668     my $max_outputs = '';
1669     if ($qxlnum) {
1670         $type = $id ? 'qxl' : 'qxl-vga';
1671
1672         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1673             # set max outputs so linux can have up to 4 qxl displays with one device
1674             if (min_version($machine_version, 4, 1)) {
1675                 $max_outputs = ",max_outputs=4";
1676             }
1677         }
1678     }
1679
1680     die "no devicetype for $vga->{type}\n" if !$type;
1681
1682     my $memory = "";
1683     if ($vgamem_mb) {
1684         if ($vga->{type} eq 'virtio') {
1685             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1686             $memory = ",max_hostmem=$bytes";
1687         } elsif ($qxlnum) {
1688             # from https://www.spice-space.org/multiple-monitors.html
1689             $memory = ",vgamem_mb=$vga->{memory}";
1690             my $ram = $vgamem_mb * 4;
1691             my $vram = $vgamem_mb * 2;
1692             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1693         } else {
1694             $memory = ",vgamem_mb=$vga->{memory}";
1695         }
1696     } elsif ($qxlnum && $id) {
1697         $memory = ",ram_size=67108864,vram_size=33554432";
1698     }
1699
1700     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1701     my $vgaid = "vga" . ($id // '');
1702     my $pciaddr;
1703
1704     if ($q35 && $vgaid eq 'vga') {
1705         # the first display uses pcie.0 bus on q35 machines
1706         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1707     } else {
1708         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1709     }
1710
1711     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1712 }
1713
1714 sub parse_number_sets {
1715     my ($set) = @_;
1716     my $res = [];
1717     foreach my $part (split(/;/, $set)) {
1718         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1719             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1720             push @$res, [ $1, $2 ];
1721         } else {
1722             die "invalid range: $part\n";
1723         }
1724     }
1725     return $res;
1726 }
1727
1728 sub parse_numa {
1729     my ($data) = @_;
1730
1731     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1732     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1733     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1734     return $res;
1735 }
1736
1737 sub parse_hostpci {
1738     my ($value) = @_;
1739
1740     return undef if !$value;
1741
1742     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1743
1744     my @idlist = split(/;/, $res->{host});
1745     delete $res->{host};
1746     foreach my $id (@idlist) {
1747         my $devs = PVE::SysFSTools::lspci($id);
1748         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1749         push @{$res->{pciid}}, @$devs;
1750     }
1751     return $res;
1752 }
1753
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1755 sub parse_net {
1756     my ($data) = @_;
1757
1758     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1759     if ($@) {
1760         warn $@;
1761         return undef;
1762     }
1763     if (!defined($res->{macaddr})) {
1764         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1765         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1766     }
1767     return $res;
1768 }
1769
1770 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1771 sub parse_ipconfig {
1772     my ($data) = @_;
1773
1774     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1775     if ($@) {
1776         warn $@;
1777         return undef;
1778     }
1779
1780     if ($res->{gw} && !$res->{ip}) {
1781         warn 'gateway specified without specifying an IP address';
1782         return undef;
1783     }
1784     if ($res->{gw6} && !$res->{ip6}) {
1785         warn 'IPv6 gateway specified without specifying an IPv6 address';
1786         return undef;
1787     }
1788     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1789         warn 'gateway specified together with DHCP';
1790         return undef;
1791     }
1792     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1793         # gw6 + auto/dhcp
1794         warn "IPv6 gateway specified together with $res->{ip6} address";
1795         return undef;
1796     }
1797
1798     if (!$res->{ip} && !$res->{ip6}) {
1799         return { ip => 'dhcp', ip6 => 'dhcp' };
1800     }
1801
1802     return $res;
1803 }
1804
1805 sub print_net {
1806     my $net = shift;
1807
1808     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1809 }
1810
1811 sub add_random_macs {
1812     my ($settings) = @_;
1813
1814     foreach my $opt (keys %$settings) {
1815         next if $opt !~ m/^net(\d+)$/;
1816         my $net = parse_net($settings->{$opt});
1817         next if !$net;
1818         $settings->{$opt} = print_net($net);
1819     }
1820 }
1821
1822 sub vm_is_volid_owner {
1823     my ($storecfg, $vmid, $volid) = @_;
1824
1825     if ($volid !~  m|^/|) {
1826         my ($path, $owner);
1827         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1828         if ($owner && ($owner == $vmid)) {
1829             return 1;
1830         }
1831     }
1832
1833     return undef;
1834 }
1835
1836 sub vmconfig_register_unused_drive {
1837     my ($storecfg, $vmid, $conf, $drive) = @_;
1838
1839     if (drive_is_cloudinit($drive)) {
1840         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1841         warn $@ if $@;
1842     } elsif (!drive_is_cdrom($drive)) {
1843         my $volid = $drive->{file};
1844         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1845             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1846         }
1847     }
1848 }
1849
1850 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1851 my $smbios1_fmt = {
1852     uuid => {
1853         type => 'string',
1854         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1855         format_description => 'UUID',
1856         description => "Set SMBIOS1 UUID.",
1857         optional => 1,
1858     },
1859     version => {
1860         type => 'string',
1861         pattern => '[A-Za-z0-9+\/]+={0,2}',
1862         format_description => 'Base64 encoded string',
1863         description => "Set SMBIOS1 version.",
1864         optional => 1,
1865     },
1866     serial => {
1867         type => 'string',
1868         pattern => '[A-Za-z0-9+\/]+={0,2}',
1869         format_description => 'Base64 encoded string',
1870         description => "Set SMBIOS1 serial number.",
1871         optional => 1,
1872     },
1873     manufacturer => {
1874         type => 'string',
1875         pattern => '[A-Za-z0-9+\/]+={0,2}',
1876         format_description => 'Base64 encoded string',
1877         description => "Set SMBIOS1 manufacturer.",
1878         optional => 1,
1879     },
1880     product => {
1881         type => 'string',
1882         pattern => '[A-Za-z0-9+\/]+={0,2}',
1883         format_description => 'Base64 encoded string',
1884         description => "Set SMBIOS1 product ID.",
1885         optional => 1,
1886     },
1887     sku => {
1888         type => 'string',
1889         pattern => '[A-Za-z0-9+\/]+={0,2}',
1890         format_description => 'Base64 encoded string',
1891         description => "Set SMBIOS1 SKU string.",
1892         optional => 1,
1893     },
1894     family => {
1895         type => 'string',
1896         pattern => '[A-Za-z0-9+\/]+={0,2}',
1897         format_description => 'Base64 encoded string',
1898         description => "Set SMBIOS1 family string.",
1899         optional => 1,
1900     },
1901     base64 => {
1902         type => 'boolean',
1903         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1904         optional => 1,
1905     },
1906 };
1907
1908 sub parse_smbios1 {
1909     my ($data) = @_;
1910
1911     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1912     warn $@ if $@;
1913     return $res;
1914 }
1915
1916 sub print_smbios1 {
1917     my ($smbios1) = @_;
1918     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1919 }
1920
1921 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1922
1923 sub parse_watchdog {
1924     my ($value) = @_;
1925
1926     return undef if !$value;
1927
1928     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1929     warn $@ if $@;
1930     return $res;
1931 }
1932
1933 sub parse_guest_agent {
1934     my ($value) = @_;
1935
1936     return {} if !defined($value->{agent});
1937
1938     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1939     warn $@ if $@;
1940
1941     # if the agent is disabled ignore the other potentially set properties
1942     return {} if !$res->{enabled};
1943     return $res;
1944 }
1945
1946 sub parse_vga {
1947     my ($value) = @_;
1948
1949     return {} if !$value;
1950     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1951     warn $@ if $@;
1952     return $res;
1953 }
1954
1955 sub parse_rng {
1956     my ($value) = @_;
1957
1958     return undef if !$value;
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1961     warn $@ if $@;
1962     return $res;
1963 }
1964
1965 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1966 sub verify_usb_device {
1967     my ($value, $noerr) = @_;
1968
1969     return $value if parse_usb_device($value);
1970
1971     return undef if $noerr;
1972
1973     die "unable to parse usb device\n";
1974 }
1975
1976 # add JSON properties for create and set function
1977 sub json_config_properties {
1978     my $prop = shift;
1979
1980     foreach my $opt (keys %$confdesc) {
1981         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1982             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1983         $prop->{$opt} = $confdesc->{$opt};
1984     }
1985
1986     return $prop;
1987 }
1988
1989 # return copy of $confdesc_cloudinit to generate documentation
1990 sub cloudinit_config_properties {
1991
1992     return dclone($confdesc_cloudinit);
1993 }
1994
1995 sub check_type {
1996     my ($key, $value) = @_;
1997
1998     die "unknown setting '$key'\n" if !$confdesc->{$key};
1999
2000     my $type = $confdesc->{$key}->{type};
2001
2002     if (!defined($value)) {
2003         die "got undefined value\n";
2004     }
2005
2006     if ($value =~ m/[\n\r]/) {
2007         die "property contains a line feed\n";
2008     }
2009
2010     if ($type eq 'boolean') {
2011         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2012         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2013         die "type check ('boolean') failed - got '$value'\n";
2014     } elsif ($type eq 'integer') {
2015         return int($1) if $value =~ m/^(\d+)$/;
2016         die "type check ('integer') failed - got '$value'\n";
2017     } elsif ($type eq 'number') {
2018         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2019         die "type check ('number') failed - got '$value'\n";
2020     } elsif ($type eq 'string') {
2021         if (my $fmt = $confdesc->{$key}->{format}) {
2022             PVE::JSONSchema::check_format($fmt, $value);
2023             return $value;
2024         }
2025         $value =~ s/^\"(.*)\"$/$1/;
2026         return $value;
2027     } else {
2028         die "internal error"
2029     }
2030 }
2031
2032 sub destroy_vm {
2033     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2034
2035     my $conf = PVE::QemuConfig->load_config($vmid);
2036
2037     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2038
2039     if ($conf->{template}) {
2040         # check if any base image is still used by a linked clone
2041         PVE::QemuConfig->foreach_volume($conf, sub {
2042                 my ($ds, $drive) = @_;
2043                 return if drive_is_cdrom($drive);
2044
2045                 my $volid = $drive->{file};
2046                 return if !$volid || $volid =~ m|^/|;
2047
2048                 die "base volume '$volid' is still in use by linked cloned\n"
2049                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2050
2051         });
2052     }
2053
2054     # only remove disks owned by this VM
2055     PVE::QemuConfig->foreach_volume($conf, sub {
2056         my ($ds, $drive) = @_;
2057         return if drive_is_cdrom($drive, 1);
2058
2059         my $volid = $drive->{file};
2060         return if !$volid || $volid =~ m|^/|;
2061
2062         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2063         return if !$path || !$owner || ($owner != $vmid);
2064
2065         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2066         warn "Could not remove disk '$volid', check manually: $@" if $@;
2067     });
2068
2069     # also remove unused disk
2070     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2071     PVE::Storage::foreach_volid($vmdisks, sub {
2072         my ($volid, $sid, $volname, $d) = @_;
2073         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2074         warn $@ if $@;
2075     });
2076
2077     if (defined $replacement_conf) {
2078         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2079     } else {
2080         PVE::QemuConfig->destroy_config($vmid);
2081     }
2082 }
2083
2084 sub parse_vm_config {
2085     my ($filename, $raw) = @_;
2086
2087     return undef if !defined($raw);
2088
2089     my $res = {
2090         digest => Digest::SHA::sha1_hex($raw),
2091         snapshots => {},
2092         pending => {},
2093     };
2094
2095     $filename =~ m|/qemu-server/(\d+)\.conf$|
2096         || die "got strange filename '$filename'";
2097
2098     my $vmid = $1;
2099
2100     my $conf = $res;
2101     my $descr;
2102     my $section = '';
2103
2104     my @lines = split(/\n/, $raw);
2105     foreach my $line (@lines) {
2106         next if $line =~ m/^\s*$/;
2107
2108         if ($line =~ m/^\[PENDING\]\s*$/i) {
2109             $section = 'pending';
2110             if (defined($descr)) {
2111                 $descr =~ s/\s+$//;
2112                 $conf->{description} = $descr;
2113             }
2114             $descr = undef;
2115             $conf = $res->{$section} = {};
2116             next;
2117
2118         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2119             $section = $1;
2120             if (defined($descr)) {
2121                 $descr =~ s/\s+$//;
2122                 $conf->{description} = $descr;
2123             }
2124             $descr = undef;
2125             $conf = $res->{snapshots}->{$section} = {};
2126             next;
2127         }
2128
2129         if ($line =~ m/^\#(.*)\s*$/) {
2130             $descr = '' if !defined($descr);
2131             $descr .= PVE::Tools::decode_text($1) . "\n";
2132             next;
2133         }
2134
2135         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2136             $descr = '' if !defined($descr);
2137             $descr .= PVE::Tools::decode_text($2);
2138         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2139             $conf->{snapstate} = $1;
2140         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2141             my $key = $1;
2142             my $value = $2;
2143             $conf->{$key} = $value;
2144         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2145             my $value = $1;
2146             if ($section eq 'pending') {
2147                 $conf->{delete} = $value; # we parse this later
2148             } else {
2149                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2150             }
2151         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2152             my $key = $1;
2153             my $value = $2;
2154             eval { $value = check_type($key, $value); };
2155             if ($@) {
2156                 warn "vm $vmid - unable to parse value of '$key' - $@";
2157             } else {
2158                 $key = 'ide2' if $key eq 'cdrom';
2159                 my $fmt = $confdesc->{$key}->{format};
2160                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2161                     my $v = parse_drive($key, $value);
2162                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2163                         $v->{file} = $volid;
2164                         $value = print_drive($v);
2165                     } else {
2166                         warn "vm $vmid - unable to parse value of '$key'\n";
2167                         next;
2168                     }
2169                 }
2170
2171                 $conf->{$key} = $value;
2172             }
2173         }
2174     }
2175
2176     if (defined($descr)) {
2177         $descr =~ s/\s+$//;
2178         $conf->{description} = $descr;
2179     }
2180     delete $res->{snapstate}; # just to be sure
2181
2182     return $res;
2183 }
2184
2185 sub write_vm_config {
2186     my ($filename, $conf) = @_;
2187
2188     delete $conf->{snapstate}; # just to be sure
2189
2190     if ($conf->{cdrom}) {
2191         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2192         $conf->{ide2} = $conf->{cdrom};
2193         delete $conf->{cdrom};
2194     }
2195
2196     # we do not use 'smp' any longer
2197     if ($conf->{sockets}) {
2198         delete $conf->{smp};
2199     } elsif ($conf->{smp}) {
2200         $conf->{sockets} = $conf->{smp};
2201         delete $conf->{cores};
2202         delete $conf->{smp};
2203     }
2204
2205     my $used_volids = {};
2206
2207     my $cleanup_config = sub {
2208         my ($cref, $pending, $snapname) = @_;
2209
2210         foreach my $key (keys %$cref) {
2211             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2212                 $key eq 'snapstate' || $key eq 'pending';
2213             my $value = $cref->{$key};
2214             if ($key eq 'delete') {
2215                 die "propertry 'delete' is only allowed in [PENDING]\n"
2216                     if !$pending;
2217                 # fixme: check syntax?
2218                 next;
2219             }
2220             eval { $value = check_type($key, $value); };
2221             die "unable to parse value of '$key' - $@" if $@;
2222
2223             $cref->{$key} = $value;
2224
2225             if (!$snapname && is_valid_drivename($key)) {
2226                 my $drive = parse_drive($key, $value);
2227                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2228             }
2229         }
2230     };
2231
2232     &$cleanup_config($conf);
2233
2234     &$cleanup_config($conf->{pending}, 1);
2235
2236     foreach my $snapname (keys %{$conf->{snapshots}}) {
2237         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2238         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2239     }
2240
2241     # remove 'unusedX' settings if we re-add a volume
2242     foreach my $key (keys %$conf) {
2243         my $value = $conf->{$key};
2244         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2245             delete $conf->{$key};
2246         }
2247     }
2248
2249     my $generate_raw_config = sub {
2250         my ($conf, $pending) = @_;
2251
2252         my $raw = '';
2253
2254         # add description as comment to top of file
2255         if (defined(my $descr = $conf->{description})) {
2256             if ($descr) {
2257                 foreach my $cl (split(/\n/, $descr)) {
2258                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2259                 }
2260             } else {
2261                 $raw .= "#\n" if $pending;
2262             }
2263         }
2264
2265         foreach my $key (sort keys %$conf) {
2266             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2267             $raw .= "$key: $conf->{$key}\n";
2268         }
2269         return $raw;
2270     };
2271
2272     my $raw = &$generate_raw_config($conf);
2273
2274     if (scalar(keys %{$conf->{pending}})){
2275         $raw .= "\n[PENDING]\n";
2276         $raw .= &$generate_raw_config($conf->{pending}, 1);
2277     }
2278
2279     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2280         $raw .= "\n[$snapname]\n";
2281         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2282     }
2283
2284     return $raw;
2285 }
2286
2287 sub load_defaults {
2288
2289     my $res = {};
2290
2291     # we use static defaults from our JSON schema configuration
2292     foreach my $key (keys %$confdesc) {
2293         if (defined(my $default = $confdesc->{$key}->{default})) {
2294             $res->{$key} = $default;
2295         }
2296     }
2297
2298     return $res;
2299 }
2300
2301 sub config_list {
2302     my $vmlist = PVE::Cluster::get_vmlist();
2303     my $res = {};
2304     return $res if !$vmlist || !$vmlist->{ids};
2305     my $ids = $vmlist->{ids};
2306     my $nodename = nodename();
2307
2308     foreach my $vmid (keys %$ids) {
2309         my $d = $ids->{$vmid};
2310         next if !$d->{node} || $d->{node} ne $nodename;
2311         next if !$d->{type} || $d->{type} ne 'qemu';
2312         $res->{$vmid}->{exists} = 1;
2313     }
2314     return $res;
2315 }
2316
2317 # test if VM uses local resources (to prevent migration)
2318 sub check_local_resources {
2319     my ($conf, $noerr) = @_;
2320
2321     my @loc_res = ();
2322
2323     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2324     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2325
2326     push @loc_res, "ivshmem" if $conf->{ivshmem};
2327
2328     foreach my $k (keys %$conf) {
2329         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2330         # sockets are safe: they will recreated be on the target side post-migrate
2331         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2332         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2333     }
2334
2335     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2336
2337     return \@loc_res;
2338 }
2339
2340 # check if used storages are available on all nodes (use by migrate)
2341 sub check_storage_availability {
2342     my ($storecfg, $conf, $node) = @_;
2343
2344     PVE::QemuConfig->foreach_volume($conf, sub {
2345         my ($ds, $drive) = @_;
2346
2347         my $volid = $drive->{file};
2348         return if !$volid;
2349
2350         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2351         return if !$sid;
2352
2353         # check if storage is available on both nodes
2354         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2355         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2356    });
2357 }
2358
2359 # list nodes where all VM images are available (used by has_feature API)
2360 sub shared_nodes {
2361     my ($conf, $storecfg) = @_;
2362
2363     my $nodelist = PVE::Cluster::get_nodelist();
2364     my $nodehash = { map { $_ => 1 } @$nodelist };
2365     my $nodename = nodename();
2366
2367     PVE::QemuConfig->foreach_volume($conf, sub {
2368         my ($ds, $drive) = @_;
2369
2370         my $volid = $drive->{file};
2371         return if !$volid;
2372
2373         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2374         if ($storeid) {
2375             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2376             if ($scfg->{disable}) {
2377                 $nodehash = {};
2378             } elsif (my $avail = $scfg->{nodes}) {
2379                 foreach my $node (keys %$nodehash) {
2380                     delete $nodehash->{$node} if !$avail->{$node};
2381                 }
2382             } elsif (!$scfg->{shared}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     delete $nodehash->{$node} if $node ne $nodename
2385                 }
2386             }
2387         }
2388     });
2389
2390     return $nodehash
2391 }
2392
2393 sub check_local_storage_availability {
2394     my ($conf, $storecfg) = @_;
2395
2396     my $nodelist = PVE::Cluster::get_nodelist();
2397     my $nodehash = { map { $_ => {} } @$nodelist };
2398
2399     PVE::QemuConfig->foreach_volume($conf, sub {
2400         my ($ds, $drive) = @_;
2401
2402         my $volid = $drive->{file};
2403         return if !$volid;
2404
2405         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2406         if ($storeid) {
2407             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2408
2409             if ($scfg->{disable}) {
2410                 foreach my $node (keys %$nodehash) {
2411                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2412                 }
2413             } elsif (my $avail = $scfg->{nodes}) {
2414                 foreach my $node (keys %$nodehash) {
2415                     if (!$avail->{$node}) {
2416                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2417                     }
2418                 }
2419             }
2420         }
2421     });
2422
2423     foreach my $node (values %$nodehash) {
2424         if (my $unavail = $node->{unavailable_storages}) {
2425             $node->{unavailable_storages} = [ sort keys %$unavail ];
2426         }
2427     }
2428
2429     return $nodehash
2430 }
2431
2432 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2433 sub check_running {
2434     my ($vmid, $nocheck, $node) = @_;
2435
2436     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2437     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2438 }
2439
2440 sub vzlist {
2441
2442     my $vzlist = config_list();
2443
2444     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2445
2446     while (defined(my $de = $fd->read)) {
2447         next if $de !~ m/^(\d+)\.pid$/;
2448         my $vmid = $1;
2449         next if !defined($vzlist->{$vmid});
2450         if (my $pid = check_running($vmid)) {
2451             $vzlist->{$vmid}->{pid} = $pid;
2452         }
2453     }
2454
2455     return $vzlist;
2456 }
2457
2458 our $vmstatus_return_properties = {
2459     vmid => get_standard_option('pve-vmid'),
2460     status => {
2461         description => "Qemu process status.",
2462         type => 'string',
2463         enum => ['stopped', 'running'],
2464     },
2465     maxmem => {
2466         description => "Maximum memory in bytes.",
2467         type => 'integer',
2468         optional => 1,
2469         renderer => 'bytes',
2470     },
2471     maxdisk => {
2472         description => "Root disk size in bytes.",
2473         type => 'integer',
2474         optional => 1,
2475         renderer => 'bytes',
2476     },
2477     name => {
2478         description => "VM name.",
2479         type => 'string',
2480         optional => 1,
2481     },
2482     qmpstatus => {
2483         description => "Qemu QMP agent status.",
2484         type => 'string',
2485         optional => 1,
2486     },
2487     pid => {
2488         description => "PID of running qemu process.",
2489         type => 'integer',
2490         optional => 1,
2491     },
2492     uptime => {
2493         description => "Uptime.",
2494         type => 'integer',
2495         optional => 1,
2496         renderer => 'duration',
2497     },
2498     cpus => {
2499         description => "Maximum usable CPUs.",
2500         type => 'number',
2501         optional => 1,
2502     },
2503     lock => {
2504         description => "The current config lock, if any.",
2505         type => 'string',
2506         optional => 1,
2507     },
2508     tags => {
2509         description  => "The current configured tags, if any",
2510         type => 'string',
2511         optional => 1,
2512     },
2513 };
2514
2515 my $last_proc_pid_stat;
2516
2517 # get VM status information
2518 # This must be fast and should not block ($full == false)
2519 # We only query KVM using QMP if $full == true (this can be slow)
2520 sub vmstatus {
2521     my ($opt_vmid, $full) = @_;
2522
2523     my $res = {};
2524
2525     my $storecfg = PVE::Storage::config();
2526
2527     my $list = vzlist();
2528     my $defaults = load_defaults();
2529
2530     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2531
2532     my $cpucount = $cpuinfo->{cpus} || 1;
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536
2537         my $conf = PVE::QemuConfig->load_config($vmid);
2538
2539         my $d = { vmid => $vmid };
2540         $d->{pid} = $list->{$vmid}->{pid};
2541
2542         # fixme: better status?
2543         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2544
2545         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2546         if (defined($size)) {
2547             $d->{disk} = 0; # no info available
2548             $d->{maxdisk} = $size;
2549         } else {
2550             $d->{disk} = 0;
2551             $d->{maxdisk} = 0;
2552         }
2553
2554         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2555             * ($conf->{cores} || $defaults->{cores});
2556         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2557         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2558
2559         $d->{name} = $conf->{name} || "VM $vmid";
2560         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2561             : $defaults->{memory}*(1024*1024);
2562
2563         if ($conf->{balloon}) {
2564             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2565             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2566                 : $defaults->{shares};
2567         }
2568
2569         $d->{uptime} = 0;
2570         $d->{cpu} = 0;
2571         $d->{mem} = 0;
2572
2573         $d->{netout} = 0;
2574         $d->{netin} = 0;
2575
2576         $d->{diskread} = 0;
2577         $d->{diskwrite} = 0;
2578
2579         $d->{template} = PVE::QemuConfig->is_template($conf);
2580
2581         $d->{serial} = 1 if conf_has_serial($conf);
2582         $d->{lock} = $conf->{lock} if $conf->{lock};
2583         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2584
2585         $res->{$vmid} = $d;
2586     }
2587
2588     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2589     foreach my $dev (keys %$netdev) {
2590         next if $dev !~ m/^tap([1-9]\d*)i/;
2591         my $vmid = $1;
2592         my $d = $res->{$vmid};
2593         next if !$d;
2594
2595         $d->{netout} += $netdev->{$dev}->{receive};
2596         $d->{netin} += $netdev->{$dev}->{transmit};
2597
2598         if ($full) {
2599             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2600             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2601         }
2602
2603     }
2604
2605     my $ctime = gettimeofday;
2606
2607     foreach my $vmid (keys %$list) {
2608
2609         my $d = $res->{$vmid};
2610         my $pid = $d->{pid};
2611         next if !$pid;
2612
2613         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2614         next if !$pstat; # not running
2615
2616         my $used = $pstat->{utime} + $pstat->{stime};
2617
2618         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2619
2620         if ($pstat->{vsize}) {
2621             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2622         }
2623
2624         my $old = $last_proc_pid_stat->{$pid};
2625         if (!$old) {
2626             $last_proc_pid_stat->{$pid} = {
2627                 time => $ctime,
2628                 used => $used,
2629                 cpu => 0,
2630             };
2631             next;
2632         }
2633
2634         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2635
2636         if ($dtime > 1000) {
2637             my $dutime = $used -  $old->{used};
2638
2639             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2640             $last_proc_pid_stat->{$pid} = {
2641                 time => $ctime,
2642                 used => $used,
2643                 cpu => $d->{cpu},
2644             };
2645         } else {
2646             $d->{cpu} = $old->{cpu};
2647         }
2648     }
2649
2650     return $res if !$full;
2651
2652     my $qmpclient = PVE::QMPClient->new();
2653
2654     my $ballooncb = sub {
2655         my ($vmid, $resp) = @_;
2656
2657         my $info = $resp->{'return'};
2658         return if !$info->{max_mem};
2659
2660         my $d = $res->{$vmid};
2661
2662         # use memory assigned to VM
2663         $d->{maxmem} = $info->{max_mem};
2664         $d->{balloon} = $info->{actual};
2665
2666         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2667             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2668             $d->{freemem} = $info->{free_mem};
2669         }
2670
2671         $d->{ballooninfo} = $info;
2672     };
2673
2674     my $blockstatscb = sub {
2675         my ($vmid, $resp) = @_;
2676         my $data = $resp->{'return'} || [];
2677         my $totalrdbytes = 0;
2678         my $totalwrbytes = 0;
2679
2680         for my $blockstat (@$data) {
2681             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2682             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2683
2684             $blockstat->{device} =~ s/drive-//;
2685             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2686         }
2687         $res->{$vmid}->{diskread} = $totalrdbytes;
2688         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2689     };
2690
2691     my $statuscb = sub {
2692         my ($vmid, $resp) = @_;
2693
2694         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2695         # this fails if ballon driver is not loaded, so this must be
2696         # the last commnand (following command are aborted if this fails).
2697         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2698
2699         my $status = 'unknown';
2700         if (!defined($status = $resp->{'return'}->{status})) {
2701             warn "unable to get VM status\n";
2702             return;
2703         }
2704
2705         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2706     };
2707
2708     foreach my $vmid (keys %$list) {
2709         next if $opt_vmid && ($vmid ne $opt_vmid);
2710         next if !$res->{$vmid}->{pid}; # not running
2711         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2712     }
2713
2714     $qmpclient->queue_execute(undef, 2);
2715
2716     foreach my $vmid (keys %$list) {
2717         next if $opt_vmid && ($vmid ne $opt_vmid);
2718         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2719     }
2720
2721     return $res;
2722 }
2723
2724 sub conf_has_serial {
2725     my ($conf) = @_;
2726
2727     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2728         if ($conf->{"serial$i"}) {
2729             return 1;
2730         }
2731     }
2732
2733     return 0;
2734 }
2735
2736 sub conf_has_audio {
2737     my ($conf, $id) = @_;
2738
2739     $id //= 0;
2740     my $audio = $conf->{"audio$id"};
2741     return undef if !defined($audio);
2742
2743     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2744     my $audiodriver = $audioproperties->{driver} // 'spice';
2745
2746     return {
2747         dev => $audioproperties->{device},
2748         dev_id => "audiodev$id",
2749         backend => $audiodriver,
2750         backend_id => "$audiodriver-backend${id}",
2751     };
2752 }
2753
2754 sub audio_devs {
2755     my ($audio, $audiopciaddr, $machine_version) = @_;
2756
2757     my $devs = [];
2758
2759     my $id = $audio->{dev_id};
2760     my $audiodev = "";
2761     if (min_version($machine_version, 4, 2)) {
2762         $audiodev = ",audiodev=$audio->{backend_id}";
2763     }
2764
2765     if ($audio->{dev} eq 'AC97') {
2766         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2767     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2768         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2769         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2770         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2771     } else {
2772         die "unkown audio device '$audio->{dev}', implement me!";
2773     }
2774
2775     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2776
2777     return $devs;
2778 }
2779
2780 sub vga_conf_has_spice {
2781     my ($vga) = @_;
2782
2783     my $vgaconf = parse_vga($vga);
2784     my $vgatype = $vgaconf->{type};
2785     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2786
2787     return $1 || 1;
2788 }
2789
2790 sub is_native($) {
2791     my ($arch) = @_;
2792     return get_host_arch() eq $arch;
2793 }
2794
2795 sub get_vm_arch {
2796     my ($conf) = @_;
2797     return $conf->{arch} // get_host_arch();
2798 }
2799
2800 my $default_machines = {
2801     x86_64 => 'pc',
2802     aarch64 => 'virt',
2803 };
2804
2805 sub get_vm_machine {
2806     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2807
2808     my $machine = $forcemachine || $conf->{machine};
2809
2810     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2811         $arch //= 'x86_64';
2812         $machine ||= $default_machines->{$arch};
2813         if ($add_pve_version) {
2814             $kvmversion //= kvm_user_version();
2815             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2816             $machine .= "+pve$pvever";
2817         }
2818     }
2819
2820     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2821         # for version-pinned machines that do not include a pve-version (e.g.
2822         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2823         $machine .= '+pve0';
2824     }
2825
2826     return $machine;
2827 }
2828
2829 sub get_ovmf_files($) {
2830     my ($arch) = @_;
2831
2832     my $ovmf = $OVMF->{$arch}
2833         or die "no OVMF images known for architecture '$arch'\n";
2834
2835     return @$ovmf;
2836 }
2837
2838 my $Arch2Qemu = {
2839     aarch64 => '/usr/bin/qemu-system-aarch64',
2840     x86_64 => '/usr/bin/qemu-system-x86_64',
2841 };
2842 sub get_command_for_arch($) {
2843     my ($arch) = @_;
2844     return '/usr/bin/kvm' if is_native($arch);
2845
2846     my $cmd = $Arch2Qemu->{$arch}
2847         or die "don't know how to emulate architecture '$arch'\n";
2848     return $cmd;
2849 }
2850
2851 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2852 # to use in a QEMU command line (-cpu element), first array_intersect the result
2853 # of query_supported_ with query_understood_. This is necessary because:
2854 #
2855 # a) query_understood_ returns flags the host cannot use and
2856 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2857 #    flags, but CPU settings - with most of them being flags. Those settings
2858 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2859 #
2860 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2861 # expensive. If you need the value returned from this, you can get it much
2862 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2863 # $accel being 'kvm' or 'tcg'.
2864 #
2865 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2866 # changes, automatically populating pmxcfs.
2867 #
2868 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2869 # since kvm and tcg machines support different flags
2870 #
2871 sub query_supported_cpu_flags {
2872     my ($arch) = @_;
2873
2874     $arch //= get_host_arch();
2875     my $default_machine = $default_machines->{$arch};
2876
2877     my $flags = {};
2878
2879     # FIXME: Once this is merged, the code below should work for ARM as well:
2880     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2881     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2882         $arch eq "aarch64";
2883
2884     my $kvm_supported = defined(kvm_version());
2885     my $qemu_cmd = get_command_for_arch($arch);
2886     my $fakevmid = -1;
2887     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2888
2889     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2890     my $query_supported_run_qemu = sub {
2891         my ($kvm) = @_;
2892
2893         my $flags = {};
2894         my $cmd = [
2895             $qemu_cmd,
2896             '-machine', $default_machine,
2897             '-display', 'none',
2898             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2899             '-mon', 'chardev=qmp,mode=control',
2900             '-pidfile', $pidfile,
2901             '-S', '-daemonize'
2902         ];
2903
2904         if (!$kvm) {
2905             push @$cmd, '-accel', 'tcg';
2906         }
2907
2908         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2909         die "QEMU flag querying VM exited with code " . $rc if $rc;
2910
2911         eval {
2912             my $cmd_result = mon_cmd(
2913                 $fakevmid,
2914                 'query-cpu-model-expansion',
2915                 type => 'full',
2916                 model => { name => 'host' }
2917             );
2918
2919             my $props = $cmd_result->{model}->{props};
2920             foreach my $prop (keys %$props) {
2921                 next if $props->{$prop} ne '1';
2922                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2923                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2924                 # We only keep those with underscores, to match /proc/cpuinfo
2925                 $prop =~ s/\.|-/_/g;
2926                 $flags->{$prop} = 1;
2927             }
2928         };
2929         my $err = $@;
2930
2931         # force stop with 10 sec timeout and 'nocheck'
2932         # always stop, even if QMP failed
2933         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2934
2935         die $err if $err;
2936
2937         return [ sort keys %$flags ];
2938     };
2939
2940     # We need to query QEMU twice, since KVM and TCG have different supported flags
2941     PVE::QemuConfig->lock_config($fakevmid, sub {
2942         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2943         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2944
2945         if ($kvm_supported) {
2946             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2947             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2948         }
2949     });
2950
2951     return $flags;
2952 }
2953
2954 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2955 my $understood_cpu_flag_dir = "/usr/share/kvm";
2956 sub query_understood_cpu_flags {
2957     my $arch = get_host_arch();
2958     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2959
2960     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2961         if ! -e $filepath;
2962
2963     my $raw = file_get_contents($filepath);
2964     $raw =~ s/^\s+|\s+$//g;
2965     my @flags = split(/\s+/, $raw);
2966
2967     return \@flags;
2968 }
2969
2970 sub config_to_command {
2971     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2972
2973     my $cmd = [];
2974     my $globalFlags = [];
2975     my $machineFlags = [];
2976     my $rtcFlags = [];
2977     my $devices = [];
2978     my $pciaddr = '';
2979     my $bridges = {};
2980     my $ostype = $conf->{ostype};
2981     my $winversion = windows_version($ostype);
2982     my $kvm = $conf->{kvm};
2983     my $nodename = nodename();
2984
2985     my $arch = get_vm_arch($conf);
2986     my $kvm_binary = get_command_for_arch($arch);
2987     my $kvmver = kvm_user_version($kvm_binary);
2988
2989     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2990         $kvmver //= "undefined";
2991         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2992     }
2993
2994     my $add_pve_version = min_version($kvmver, 4, 1);
2995
2996     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2997     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2998     $kvm //= 1 if is_native($arch);
2999
3000     $machine_version =~ m/(\d+)\.(\d+)/;
3001     my ($machine_major, $machine_minor) = ($1, $2);
3002
3003     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3004         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3005     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3006         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3007     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3008         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3009         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3010             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3011     }
3012
3013     # if a specific +pve version is required for a feature, use $version_guard
3014     # instead of min_version to allow machines to be run with the minimum
3015     # required version
3016     my $required_pve_version = 0;
3017     my $version_guard = sub {
3018         my ($major, $minor, $pve) = @_;
3019         return 0 if !min_version($machine_version, $major, $minor, $pve);
3020         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3021         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3022         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3023         return 1;
3024     };
3025
3026     if ($kvm) {
3027         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3028             if !defined kvm_version();
3029     }
3030
3031     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3032     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3033     my $use_old_bios_files = undef;
3034     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3035
3036     my $cpuunits = defined($conf->{cpuunits}) ?
3037             $conf->{cpuunits} : $defaults->{cpuunits};
3038
3039     push @$cmd, $kvm_binary;
3040
3041     push @$cmd, '-id', $vmid;
3042
3043     my $vmname = $conf->{name} || "vm$vmid";
3044
3045     push @$cmd, '-name', $vmname;
3046
3047     my $use_virtio = 0;
3048
3049     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3050     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3051     push @$cmd, '-mon', "chardev=qmp,mode=control";
3052
3053     if (min_version($machine_version, 2, 12)) {
3054         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3055         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3056     }
3057
3058     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3059
3060     push @$cmd, '-daemonize';
3061
3062     if ($conf->{smbios1}) {
3063         my $smbios_conf = parse_smbios1($conf->{smbios1});
3064         if ($smbios_conf->{base64}) {
3065             # Do not pass base64 flag to qemu
3066             delete $smbios_conf->{base64};
3067             my $smbios_string = "";
3068             foreach my $key (keys %$smbios_conf) {
3069                 my $value;
3070                 if ($key eq "uuid") {
3071                     $value = $smbios_conf->{uuid}
3072                 } else {
3073                     $value = decode_base64($smbios_conf->{$key});
3074                 }
3075                 # qemu accepts any binary data, only commas need escaping by double comma
3076                 $value =~ s/,/,,/g;
3077                 $smbios_string .= "," . $key . "=" . $value if $value;
3078             }
3079             push @$cmd, '-smbios', "type=1" . $smbios_string;
3080         } else {
3081             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3082         }
3083     }
3084
3085     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3086         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3087         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3088
3089         my ($path, $format);
3090         if (my $efidisk = $conf->{efidisk0}) {
3091             my $d = parse_drive('efidisk0', $efidisk);
3092             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3093             $format = $d->{format};
3094             if ($storeid) {
3095                 $path = PVE::Storage::path($storecfg, $d->{file});
3096                 if (!defined($format)) {
3097                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3098                     $format = qemu_img_format($scfg, $volname);
3099                 }
3100             } else {
3101                 $path = $d->{file};
3102                 die "efidisk format must be specified\n"
3103                     if !defined($format);
3104             }
3105         } else {
3106             warn "no efidisk configured! Using temporary efivars disk.\n";
3107             $path = "/tmp/$vmid-ovmf.fd";
3108             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3109             $format = 'raw';
3110         }
3111
3112         my $size_str = "";
3113
3114         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3115             $size_str = ",size=" . (-s $ovmf_vars);
3116         }
3117
3118         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3119         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3120     }
3121
3122     # load q35 config
3123     if ($q35) {
3124         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3125         if (min_version($machine_version, 4, 0)) {
3126             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3127         } else {
3128             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3129         }
3130     }
3131
3132     if ($conf->{vmgenid}) {
3133         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3134     }
3135
3136     # add usb controllers
3137     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3138     push @$devices, @usbcontrollers if @usbcontrollers;
3139     my $vga = parse_vga($conf->{vga});
3140
3141     my $qxlnum = vga_conf_has_spice($conf->{vga});
3142     $vga->{type} = 'qxl' if $qxlnum;
3143
3144     if (!$vga->{type}) {
3145         if ($arch eq 'aarch64') {
3146             $vga->{type} = 'virtio';
3147         } elsif (min_version($machine_version, 2, 9)) {
3148             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3149         } else {
3150             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3151         }
3152     }
3153
3154     # enable absolute mouse coordinates (needed by vnc)
3155     my $tablet;
3156     if (defined($conf->{tablet})) {
3157         $tablet = $conf->{tablet};
3158     } else {
3159         $tablet = $defaults->{tablet};
3160         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3161         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3162     }
3163
3164     if ($tablet) {
3165         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3166         my $kbd = print_keyboarddevice_full($conf, $arch);
3167         push @$devices, '-device', $kbd if defined($kbd);
3168     }
3169
3170     my $kvm_off = 0;
3171     my $gpu_passthrough;
3172
3173     # host pci devices
3174     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3175         my $id = "hostpci$i";
3176         my $d = parse_hostpci($conf->{$id});
3177         next if !$d;
3178
3179         if (my $pcie = $d->{pcie}) {
3180             die "q35 machine model is not enabled" if !$q35;
3181             # win7 wants to have the pcie devices directly on the pcie bus
3182             # instead of in the root port
3183             if ($winversion == 7) {
3184                 $pciaddr = print_pcie_addr("${id}bus0");
3185             } else {
3186                 # add more root ports if needed, 4 are present by default
3187                 # by pve-q35 cfgs, rest added here on demand.
3188                 if ($i > 3) {
3189                     push @$devices, '-device', print_pcie_root_port($i);
3190                 }
3191                 $pciaddr = print_pcie_addr($id);
3192             }
3193         } else {
3194             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3195         }
3196
3197         my $xvga = '';
3198         if ($d->{'x-vga'}) {
3199             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3200             $kvm_off = 1;
3201             $vga->{type} = 'none' if !defined($conf->{vga});
3202             $gpu_passthrough = 1;
3203         }
3204
3205         my $pcidevices = $d->{pciid};
3206         my $multifunction = 1 if @$pcidevices > 1;
3207
3208         my $sysfspath;
3209         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3210             my $pci_id = $pcidevices->[0]->{id};
3211             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3212             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3213         } elsif ($d->{mdev}) {
3214             warn "ignoring mediated device '$id' with multifunction device\n";
3215         }
3216
3217         my $j=0;
3218         foreach my $pcidevice (@$pcidevices) {
3219             my $devicestr = "vfio-pci";
3220
3221             if ($sysfspath) {
3222                 $devicestr .= ",sysfsdev=$sysfspath";
3223             } else {
3224                 $devicestr .= ",host=$pcidevice->{id}";
3225             }
3226
3227             my $mf_addr = $multifunction ? ".$j" : '';
3228             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3229
3230             if ($j == 0) {
3231                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3232                 $devicestr .= "$xvga";
3233                 $devicestr .= ",multifunction=on" if $multifunction;
3234                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3235             }
3236
3237             push @$devices, '-device', $devicestr;
3238             $j++;
3239         }
3240     }
3241
3242     # usb devices
3243     my $usb_dev_features = {};
3244     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3245
3246     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3247     push @$devices, @usbdevices if @usbdevices;
3248     # serial devices
3249     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3250         if (my $path = $conf->{"serial$i"}) {
3251             if ($path eq 'socket') {
3252                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3253                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3254                 # On aarch64, serial0 is the UART device. Qemu only allows
3255                 # connecting UART devices via the '-serial' command line, as
3256                 # the device has a fixed slot on the hardware...
3257                 if ($arch eq 'aarch64' && $i == 0) {
3258                     push @$devices, '-serial', "chardev:serial$i";
3259                 } else {
3260                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3261                 }
3262             } else {
3263                 die "no such serial device\n" if ! -c $path;
3264                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3265                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3266             }
3267         }
3268     }
3269
3270     # parallel devices
3271     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3272         if (my $path = $conf->{"parallel$i"}) {
3273             die "no such parallel device\n" if ! -c $path;
3274             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3275             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3276             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3277         }
3278     }
3279
3280     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3281         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3282         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3283         push @$devices, @$audio_devs;
3284     }
3285
3286     my $sockets = 1;
3287     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3288     $sockets = $conf->{sockets} if  $conf->{sockets};
3289
3290     my $cores = $conf->{cores} || 1;
3291
3292     my $maxcpus = $sockets * $cores;
3293
3294     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3295
3296     my $allowed_vcpus = $cpuinfo->{cpus};
3297
3298     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3299         if ($allowed_vcpus < $maxcpus);
3300
3301     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3302
3303         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3304         for (my $i = 2; $i <= $vcpus; $i++)  {
3305             my $cpustr = print_cpu_device($conf,$i);
3306             push @$cmd, '-device', $cpustr;
3307         }
3308
3309     } else {
3310
3311         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3312     }
3313     push @$cmd, '-nodefaults';
3314
3315     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3316
3317     my $bootindex_hash = {};
3318     my $i = 1;
3319     foreach my $o (split(//, $bootorder)) {
3320         $bootindex_hash->{$o} = $i*100;
3321         $i++;
3322     }
3323
3324     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3325
3326     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3327
3328     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3329
3330     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3331         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3332         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3333         push @$cmd,  '-vnc', "unix:$socket,password";
3334     } else {
3335         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3336         push @$cmd, '-nographic';
3337     }
3338
3339     # time drift fix
3340     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3341     my $useLocaltime = $conf->{localtime};
3342
3343     if ($winversion >= 5) { # windows
3344         $useLocaltime = 1 if !defined($conf->{localtime});
3345
3346         # use time drift fix when acpi is enabled
3347         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3348             $tdf = 1 if !defined($conf->{tdf});
3349         }
3350     }
3351
3352     if ($winversion >= 6) {
3353         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3354         push @$cmd, '-no-hpet';
3355     }
3356
3357     push @$rtcFlags, 'driftfix=slew' if $tdf;
3358
3359     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3360         push @$rtcFlags, "base=$conf->{startdate}";
3361     } elsif ($useLocaltime) {
3362         push @$rtcFlags, 'base=localtime';
3363     }
3364
3365     if ($forcecpu) {
3366         push @$cmd, '-cpu', $forcecpu;
3367     } else {
3368         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3369     }
3370
3371     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3372
3373     push @$cmd, '-S' if $conf->{freeze};
3374
3375     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3376
3377     my $guest_agent = parse_guest_agent($conf);
3378
3379     if ($guest_agent->{enabled}) {
3380         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3381         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3382
3383         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3384             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3385             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3386             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3387         } elsif ($guest_agent->{type} eq 'isa') {
3388             push @$devices, '-device', "isa-serial,chardev=qga0";
3389         }
3390     }
3391
3392     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3393     if ($rng && &$version_guard(4, 1, 2)) {
3394         check_rng_source($rng->{source});
3395
3396         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3397         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3398         my $limiter_str = "";
3399         if ($max_bytes) {
3400             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3401         }
3402
3403         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3404         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3405         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3406     }
3407
3408     my $spice_port;
3409
3410     if ($qxlnum) {
3411         if ($qxlnum > 1) {
3412             if ($winversion){
3413                 for (my $i = 1; $i < $qxlnum; $i++){
3414                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3415                 }
3416             } else {
3417                 # assume other OS works like Linux
3418                 my ($ram, $vram) = ("134217728", "67108864");
3419                 if ($vga->{memory}) {
3420                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3421                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3422                 }
3423                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3424                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3425             }
3426         }
3427
3428         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3429
3430         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3431         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3432         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3433
3434         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3435         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3436         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3437
3438         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3439         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3440
3441         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3442         if ($spice_enhancement->{foldersharing}) {
3443             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3444             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3445         }
3446
3447         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3448         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3449         push @$devices, '-spice', "$spice_opts";
3450     }
3451
3452     # enable balloon by default, unless explicitly disabled
3453     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3454         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3455         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3456     }
3457
3458     if ($conf->{watchdog}) {
3459         my $wdopts = parse_watchdog($conf->{watchdog});
3460         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3461         my $watchdog = $wdopts->{model} || 'i6300esb';
3462         push @$devices, '-device', "$watchdog$pciaddr";
3463         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3464     }
3465
3466     my $vollist = [];
3467     my $scsicontroller = {};
3468     my $ahcicontroller = {};
3469     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3470
3471     # Add iscsi initiator name if available
3472     if (my $initiator = get_initiator_name()) {
3473         push @$devices, '-iscsi', "initiator-name=$initiator";
3474     }
3475
3476     PVE::QemuConfig->foreach_volume($conf, sub {
3477         my ($ds, $drive) = @_;
3478
3479         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3480             push @$vollist, $drive->{file};
3481         }
3482
3483         # ignore efidisk here, already added in bios/fw handling code above
3484         return if $drive->{interface} eq 'efidisk';
3485
3486         $use_virtio = 1 if $ds =~ m/^virtio/;
3487
3488         if (drive_is_cdrom ($drive)) {
3489             if ($bootindex_hash->{d}) {
3490                 $drive->{bootindex} = $bootindex_hash->{d};
3491                 $bootindex_hash->{d} += 1;
3492             }
3493         } else {
3494             if ($bootindex_hash->{c}) {
3495                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3496                 $bootindex_hash->{c} += 1;
3497             }
3498         }
3499
3500         if ($drive->{interface} eq 'virtio'){
3501            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3502         }
3503
3504         if ($drive->{interface} eq 'scsi') {
3505
3506             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3507
3508             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3509                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3510
3511             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3512             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3513
3514             my $iothread = '';
3515             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3516                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3517                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3518             } elsif ($drive->{iothread}) {
3519                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3520             }
3521
3522             my $queues = '';
3523             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3524                 $queues = ",num_queues=$drive->{queues}";
3525             }
3526
3527             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3528             $scsicontroller->{$controller}=1;
3529         }
3530
3531         if ($drive->{interface} eq 'sata') {
3532             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3533             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3534             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3535             $ahcicontroller->{$controller}=1;
3536         }
3537
3538         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3539         push @$devices, '-drive',$drive_cmd;
3540         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3541     });
3542
3543     for (my $i = 0; $i < $MAX_NETS; $i++) {
3544          next if !$conf->{"net$i"};
3545          my $d = parse_net($conf->{"net$i"});
3546          next if !$d;
3547
3548          $use_virtio = 1 if $d->{model} eq 'virtio';
3549
3550          if ($bootindex_hash->{n}) {
3551             $d->{bootindex} = $bootindex_hash->{n};
3552             $bootindex_hash->{n} += 1;
3553          }
3554
3555          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3556          push @$devices, '-netdev', $netdevfull;
3557
3558          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3559          push @$devices, '-device', $netdevicefull;
3560     }
3561
3562     if ($conf->{ivshmem}) {
3563         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3564
3565         my $bus;
3566         if ($q35) {
3567             $bus = print_pcie_addr("ivshmem");
3568         } else {
3569             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3570         }
3571
3572         my $ivshmem_name = $ivshmem->{name} // $vmid;
3573         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3574
3575         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3576         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3577     }
3578
3579     # pci.4 is nested in pci.1
3580     $bridges->{1} = 1 if $bridges->{4};
3581
3582     if (!$q35) {
3583         # add pci bridges
3584         if (min_version($machine_version, 2, 3)) {
3585            $bridges->{1} = 1;
3586            $bridges->{2} = 1;
3587         }
3588
3589         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3590
3591     }
3592
3593     for my $k (sort {$b cmp $a} keys %$bridges) {
3594         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3595         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3596         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3597         if ($q35) {
3598             # add after -readconfig pve-q35.cfg
3599             splice @$devices, 2, 0, '-device', $devstr;
3600         } else {
3601             unshift @$devices, '-device', $devstr if $k > 0;
3602         }
3603     }
3604
3605     if (!$kvm) {
3606         push @$machineFlags, 'accel=tcg';
3607     }
3608
3609     my $machine_type_min = $machine_type;
3610     if ($add_pve_version) {
3611         $machine_type_min =~ s/\+pve\d+$//;
3612         $machine_type_min .= "+pve$required_pve_version";
3613     }
3614     push @$machineFlags, "type=${machine_type_min}";
3615
3616     push @$cmd, @$devices;
3617     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3618     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3619     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3620
3621     if (my $vmstate = $conf->{vmstate}) {
3622         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3623         push @$vollist, $vmstate;
3624         push @$cmd, '-loadstate', $statepath;
3625         print "activating and using '$vmstate' as vmstate\n";
3626     }
3627
3628     # add custom args
3629     if ($conf->{args}) {
3630         my $aa = PVE::Tools::split_args($conf->{args});
3631         push @$cmd, @$aa;
3632     }
3633
3634     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3635 }
3636
3637 sub check_rng_source {
3638     my ($source) = @_;
3639
3640     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3641     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3642         if ! -e $source;
3643
3644     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3645     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3646         # Needs to abort, otherwise QEMU crashes on first rng access.
3647         # Note that rng_current cannot be changed to 'none' manually, so
3648         # once the VM is past this point, it is no longer an issue.
3649         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3650             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3651             . " a compatible hardware-RNG is attached to the host.\n";
3652     }
3653 }
3654
3655 sub spice_port {
3656     my ($vmid) = @_;
3657
3658     my $res = mon_cmd($vmid, 'query-spice');
3659
3660     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3661 }
3662
3663 sub vm_devices_list {
3664     my ($vmid) = @_;
3665
3666     my $res = mon_cmd($vmid, 'query-pci');
3667     my $devices_to_check = [];
3668     my $devices = {};
3669     foreach my $pcibus (@$res) {
3670         push @$devices_to_check, @{$pcibus->{devices}},
3671     }
3672
3673     while (@$devices_to_check) {
3674         my $to_check = [];
3675         for my $d (@$devices_to_check) {
3676             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3677             next if !$d->{'pci_bridge'};
3678
3679             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3680             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3681         }
3682         $devices_to_check = $to_check;
3683     }
3684
3685     my $resblock = mon_cmd($vmid, 'query-block');
3686     foreach my $block (@$resblock) {
3687         if($block->{device} =~ m/^drive-(\S+)/){
3688                 $devices->{$1} = 1;
3689         }
3690     }
3691
3692     my $resmice = mon_cmd($vmid, 'query-mice');
3693     foreach my $mice (@$resmice) {
3694         if ($mice->{name} eq 'QEMU HID Tablet') {
3695             $devices->{tablet} = 1;
3696             last;
3697         }
3698     }
3699
3700     # for usb devices there is no query-usb
3701     # but we can iterate over the entries in
3702     # qom-list path=/machine/peripheral
3703     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3704     foreach my $per (@$resperipheral) {
3705         if ($per->{name} =~ m/^usb\d+$/) {
3706             $devices->{$per->{name}} = 1;
3707         }
3708     }
3709
3710     return $devices;
3711 }
3712
3713 sub vm_deviceplug {
3714     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3715
3716     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3717
3718     my $devices_list = vm_devices_list($vmid);
3719     return 1 if defined($devices_list->{$deviceid});
3720
3721     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3722
3723     if ($deviceid eq 'tablet') {
3724
3725         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3726
3727     } elsif ($deviceid eq 'keyboard') {
3728
3729         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3730
3731     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3732
3733         die "usb hotplug currently not reliable\n";
3734         # since we can't reliably hot unplug all added usb devices
3735         # and usb passthrough disables live migration
3736         # we disable usb hotplugging for now
3737         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3738
3739     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3740
3741         qemu_iothread_add($vmid, $deviceid, $device);
3742
3743         qemu_driveadd($storecfg, $vmid, $device);
3744         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3745
3746         qemu_deviceadd($vmid, $devicefull);
3747         eval { qemu_deviceaddverify($vmid, $deviceid); };
3748         if (my $err = $@) {
3749             eval { qemu_drivedel($vmid, $deviceid); };
3750             warn $@ if $@;
3751             die $err;
3752         }
3753
3754     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3755
3756
3757         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3758         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3759         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3760
3761         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3762
3763         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3764             qemu_iothread_add($vmid, $deviceid, $device);
3765             $devicefull .= ",iothread=iothread-$deviceid";
3766         }
3767
3768         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3769             $devicefull .= ",num_queues=$device->{queues}";
3770         }
3771
3772         qemu_deviceadd($vmid, $devicefull);
3773         qemu_deviceaddverify($vmid, $deviceid);
3774
3775     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3776
3777         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3778         qemu_driveadd($storecfg, $vmid, $device);
3779
3780         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3781         eval { qemu_deviceadd($vmid, $devicefull); };
3782         if (my $err = $@) {
3783             eval { qemu_drivedel($vmid, $deviceid); };
3784             warn $@ if $@;
3785             die $err;
3786         }
3787
3788     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3789
3790         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3791
3792         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3793         my $use_old_bios_files = undef;
3794         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3795
3796         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3797         qemu_deviceadd($vmid, $netdevicefull);
3798         eval {
3799             qemu_deviceaddverify($vmid, $deviceid);
3800             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3801         };
3802         if (my $err = $@) {
3803             eval { qemu_netdevdel($vmid, $deviceid); };
3804             warn $@ if $@;
3805             die $err;
3806         }
3807
3808     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3809
3810         my $bridgeid = $2;
3811         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3812         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3813
3814         qemu_deviceadd($vmid, $devicefull);
3815         qemu_deviceaddverify($vmid, $deviceid);
3816
3817     } else {
3818         die "can't hotplug device '$deviceid'\n";
3819     }
3820
3821     return 1;
3822 }
3823
3824 # fixme: this should raise exceptions on error!
3825 sub vm_deviceunplug {
3826     my ($vmid, $conf, $deviceid) = @_;
3827
3828     my $devices_list = vm_devices_list($vmid);
3829     return 1 if !defined($devices_list->{$deviceid});
3830
3831     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3832
3833     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3834
3835         qemu_devicedel($vmid, $deviceid);
3836
3837     } elsif ($deviceid =~ m/^usb\d+$/) {
3838
3839         die "usb hotplug currently not reliable\n";
3840         # when unplugging usb devices this way,
3841         # there may be remaining usb controllers/hubs
3842         # so we disable it for now
3843         qemu_devicedel($vmid, $deviceid);
3844         qemu_devicedelverify($vmid, $deviceid);
3845
3846     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3847
3848         qemu_devicedel($vmid, $deviceid);
3849         qemu_devicedelverify($vmid, $deviceid);
3850         qemu_drivedel($vmid, $deviceid);
3851         qemu_iothread_del($conf, $vmid, $deviceid);
3852
3853     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3854
3855         qemu_devicedel($vmid, $deviceid);
3856         qemu_devicedelverify($vmid, $deviceid);
3857         qemu_iothread_del($conf, $vmid, $deviceid);
3858
3859     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3860
3861         qemu_devicedel($vmid, $deviceid);
3862         qemu_drivedel($vmid, $deviceid);
3863         qemu_deletescsihw($conf, $vmid, $deviceid);
3864
3865     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3866
3867         qemu_devicedel($vmid, $deviceid);
3868         qemu_devicedelverify($vmid, $deviceid);
3869         qemu_netdevdel($vmid, $deviceid);
3870
3871     } else {
3872         die "can't unplug device '$deviceid'\n";
3873     }
3874
3875     return 1;
3876 }
3877
3878 sub qemu_deviceadd {
3879     my ($vmid, $devicefull) = @_;
3880
3881     $devicefull = "driver=".$devicefull;
3882     my %options =  split(/[=,]/, $devicefull);
3883
3884     mon_cmd($vmid, "device_add" , %options);
3885 }
3886
3887 sub qemu_devicedel {
3888     my ($vmid, $deviceid) = @_;
3889
3890     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3891 }
3892
3893 sub qemu_iothread_add {
3894     my($vmid, $deviceid, $device) = @_;
3895
3896     if ($device->{iothread}) {
3897         my $iothreads = vm_iothreads_list($vmid);
3898         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3899     }
3900 }
3901
3902 sub qemu_iothread_del {
3903     my($conf, $vmid, $deviceid) = @_;
3904
3905     my $confid = $deviceid;
3906     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3907         $confid = 'scsi' . $1;
3908     }
3909     my $device = parse_drive($confid, $conf->{$confid});
3910     if ($device->{iothread}) {
3911         my $iothreads = vm_iothreads_list($vmid);
3912         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3913     }
3914 }
3915
3916 sub qemu_objectadd {
3917     my($vmid, $objectid, $qomtype) = @_;
3918
3919     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3920
3921     return 1;
3922 }
3923
3924 sub qemu_objectdel {
3925     my($vmid, $objectid) = @_;
3926
3927     mon_cmd($vmid, "object-del", id => $objectid);
3928
3929     return 1;
3930 }
3931
3932 sub qemu_driveadd {
3933     my ($storecfg, $vmid, $device) = @_;
3934
3935     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3936     $drive =~ s/\\/\\\\/g;
3937     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3938
3939     # If the command succeeds qemu prints: "OK"
3940     return 1 if $ret =~ m/OK/s;
3941
3942     die "adding drive failed: $ret\n";
3943 }
3944
3945 sub qemu_drivedel {
3946     my($vmid, $deviceid) = @_;
3947
3948     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3949     $ret =~ s/^\s+//;
3950
3951     return 1 if $ret eq "";
3952
3953     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3954     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3955
3956     die "deleting drive $deviceid failed : $ret\n";
3957 }
3958
3959 sub qemu_deviceaddverify {
3960     my ($vmid, $deviceid) = @_;
3961
3962     for (my $i = 0; $i <= 5; $i++) {
3963          my $devices_list = vm_devices_list($vmid);
3964          return 1 if defined($devices_list->{$deviceid});
3965          sleep 1;
3966     }
3967
3968     die "error on hotplug device '$deviceid'\n";
3969 }
3970
3971
3972 sub qemu_devicedelverify {
3973     my ($vmid, $deviceid) = @_;
3974
3975     # need to verify that the device is correctly removed as device_del
3976     # is async and empty return is not reliable
3977
3978     for (my $i = 0; $i <= 5; $i++) {
3979          my $devices_list = vm_devices_list($vmid);
3980          return 1 if !defined($devices_list->{$deviceid});
3981          sleep 1;
3982     }
3983
3984     die "error on hot-unplugging device '$deviceid'\n";
3985 }
3986
3987 sub qemu_findorcreatescsihw {
3988     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3989
3990     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3991
3992     my $scsihwid="$controller_prefix$controller";
3993     my $devices_list = vm_devices_list($vmid);
3994
3995     if(!defined($devices_list->{$scsihwid})) {
3996         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3997     }
3998
3999     return 1;
4000 }
4001
4002 sub qemu_deletescsihw {
4003     my ($conf, $vmid, $opt) = @_;
4004
4005     my $device = parse_drive($opt, $conf->{$opt});
4006
4007     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4008         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4009         return 1;
4010     }
4011
4012     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4013
4014     my $devices_list = vm_devices_list($vmid);
4015     foreach my $opt (keys %{$devices_list}) {
4016         if (is_valid_drivename($opt)) {
4017             my $drive = parse_drive($opt, $conf->{$opt});
4018             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4019                 return 1;
4020             }
4021         }
4022     }
4023
4024     my $scsihwid="scsihw$controller";
4025
4026     vm_deviceunplug($vmid, $conf, $scsihwid);
4027
4028     return 1;
4029 }
4030
4031 sub qemu_add_pci_bridge {
4032     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4033
4034     my $bridges = {};
4035
4036     my $bridgeid;
4037
4038     print_pci_addr($device, $bridges, $arch, $machine_type);
4039
4040     while (my ($k, $v) = each %$bridges) {
4041         $bridgeid = $k;
4042     }
4043     return 1 if !defined($bridgeid) || $bridgeid < 1;
4044
4045     my $bridge = "pci.$bridgeid";
4046     my $devices_list = vm_devices_list($vmid);
4047
4048     if (!defined($devices_list->{$bridge})) {
4049         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4050     }
4051
4052     return 1;
4053 }
4054
4055 sub qemu_set_link_status {
4056     my ($vmid, $device, $up) = @_;
4057
4058     mon_cmd($vmid, "set_link", name => $device,
4059                up => $up ? JSON::true : JSON::false);
4060 }
4061
4062 sub qemu_netdevadd {
4063     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4064
4065     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4066     my %options =  split(/[=,]/, $netdev);
4067
4068     if (defined(my $vhost = $options{vhost})) {
4069         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4070     }
4071
4072     if (defined(my $queues = $options{queues})) {
4073         $options{queues} = $queues + 0;
4074     }
4075
4076     mon_cmd($vmid, "netdev_add",  %options);
4077     return 1;
4078 }
4079
4080 sub qemu_netdevdel {
4081     my ($vmid, $deviceid) = @_;
4082
4083     mon_cmd($vmid, "netdev_del", id => $deviceid);
4084 }
4085
4086 sub qemu_usb_hotplug {
4087     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4088
4089     return if !$device;
4090
4091     # remove the old one first
4092     vm_deviceunplug($vmid, $conf, $deviceid);
4093
4094     # check if xhci controller is necessary and available
4095     if ($device->{usb3}) {
4096
4097         my $devicelist = vm_devices_list($vmid);
4098
4099         if (!$devicelist->{xhci}) {
4100             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4101             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4102         }
4103     }
4104     my $d = parse_usb_device($device->{host});
4105     $d->{usb3} = $device->{usb3};
4106
4107     # add the new one
4108     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4109 }
4110
4111 sub qemu_cpu_hotplug {
4112     my ($vmid, $conf, $vcpus) = @_;
4113
4114     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4115
4116     my $sockets = 1;
4117     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4118     $sockets = $conf->{sockets} if  $conf->{sockets};
4119     my $cores = $conf->{cores} || 1;
4120     my $maxcpus = $sockets * $cores;
4121
4122     $vcpus = $maxcpus if !$vcpus;
4123
4124     die "you can't add more vcpus than maxcpus\n"
4125         if $vcpus > $maxcpus;
4126
4127     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4128
4129     if ($vcpus < $currentvcpus) {
4130
4131         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4132
4133             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4134                 qemu_devicedel($vmid, "cpu$i");
4135                 my $retry = 0;
4136                 my $currentrunningvcpus = undef;
4137                 while (1) {
4138                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4139                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4140                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4141                     $retry++;
4142                     sleep 1;
4143                 }
4144                 #update conf after each succesfull cpu unplug
4145                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4146                 PVE::QemuConfig->write_config($vmid, $conf);
4147             }
4148         } else {
4149             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4150         }
4151
4152         return;
4153     }
4154
4155     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4156     die "vcpus in running vm does not match its configuration\n"
4157         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4158
4159     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4160
4161         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4162             my $cpustr = print_cpu_device($conf, $i);
4163             qemu_deviceadd($vmid, $cpustr);
4164
4165             my $retry = 0;
4166             my $currentrunningvcpus = undef;
4167             while (1) {
4168                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4169                 last if scalar(@{$currentrunningvcpus}) == $i;
4170                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4171                 sleep 1;
4172                 $retry++;
4173             }
4174             #update conf after each succesfull cpu hotplug
4175             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4176             PVE::QemuConfig->write_config($vmid, $conf);
4177         }
4178     } else {
4179
4180         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4181             mon_cmd($vmid, "cpu-add", id => int($i));
4182         }
4183     }
4184 }
4185
4186 sub qemu_block_set_io_throttle {
4187     my ($vmid, $deviceid,
4188         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4189         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4190         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4191         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4192
4193     return if !check_running($vmid) ;
4194
4195     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4196         bps => int($bps),
4197         bps_rd => int($bps_rd),
4198         bps_wr => int($bps_wr),
4199         iops => int($iops),
4200         iops_rd => int($iops_rd),
4201         iops_wr => int($iops_wr),
4202         bps_max => int($bps_max),
4203         bps_rd_max => int($bps_rd_max),
4204         bps_wr_max => int($bps_wr_max),
4205         iops_max => int($iops_max),
4206         iops_rd_max => int($iops_rd_max),
4207         iops_wr_max => int($iops_wr_max),
4208         bps_max_length => int($bps_max_length),
4209         bps_rd_max_length => int($bps_rd_max_length),
4210         bps_wr_max_length => int($bps_wr_max_length),
4211         iops_max_length => int($iops_max_length),
4212         iops_rd_max_length => int($iops_rd_max_length),
4213         iops_wr_max_length => int($iops_wr_max_length),
4214     );
4215
4216 }
4217
4218 # old code, only used to shutdown old VM after update
4219 sub __read_avail {
4220     my ($fh, $timeout) = @_;
4221
4222     my $sel = new IO::Select;
4223     $sel->add($fh);
4224
4225     my $res = '';
4226     my $buf;
4227
4228     my @ready;
4229     while (scalar (@ready = $sel->can_read($timeout))) {
4230         my $count;
4231         if ($count = $fh->sysread($buf, 8192)) {
4232             if ($buf =~ /^(.*)\(qemu\) $/s) {
4233                 $res .= $1;
4234                 last;
4235             } else {
4236                 $res .= $buf;
4237             }
4238         } else {
4239             if (!defined($count)) {
4240                 die "$!\n";
4241             }
4242             last;
4243         }
4244     }
4245
4246     die "monitor read timeout\n" if !scalar(@ready);
4247
4248     return $res;
4249 }
4250
4251 sub qemu_block_resize {
4252     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4253
4254     my $running = check_running($vmid);
4255
4256     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4257
4258     return if !$running;
4259
4260     my $padding = (1024 - $size % 1024) % 1024;
4261     $size = $size + $padding;
4262
4263     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4264
4265 }
4266
4267 sub qemu_volume_snapshot {
4268     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4269
4270     my $running = check_running($vmid);
4271
4272     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4273         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4274     } else {
4275         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4276     }
4277 }
4278
4279 sub qemu_volume_snapshot_delete {
4280     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4281
4282     my $running = check_running($vmid);
4283
4284     if($running) {
4285
4286         $running = undef;
4287         my $conf = PVE::QemuConfig->load_config($vmid);
4288         PVE::QemuConfig->foreach_volume($conf, sub {
4289             my ($ds, $drive) = @_;
4290             $running = 1 if $drive->{file} eq $volid;
4291         });
4292     }
4293
4294     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4295         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4296     } else {
4297         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4298     }
4299 }
4300
4301 sub set_migration_caps {
4302     my ($vmid) = @_;
4303
4304     my $cap_ref = [];
4305
4306     my $enabled_cap = {
4307         "auto-converge" => 1,
4308         "xbzrle" => 1,
4309         "x-rdma-pin-all" => 0,
4310         "zero-blocks" => 0,
4311         "compress" => 0
4312     };
4313
4314     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4315
4316     for my $supported_capability (@$supported_capabilities) {
4317         push @$cap_ref, {
4318             capability => $supported_capability->{capability},
4319             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4320         };
4321     }
4322
4323     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4324 }
4325
4326 sub foreach_volid {
4327     my ($conf, $func, @param) = @_;
4328
4329     my $volhash = {};
4330
4331     my $test_volid = sub {
4332         my ($key, $drive, $snapname) = @_;
4333
4334         my $volid = $drive->{file};
4335         return if !$volid;
4336
4337         $volhash->{$volid}->{cdrom} //= 1;
4338         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4339
4340         my $replicate = $drive->{replicate} // 1;
4341         $volhash->{$volid}->{replicate} //= 0;
4342         $volhash->{$volid}->{replicate} = 1 if $replicate;
4343
4344         $volhash->{$volid}->{shared} //= 0;
4345         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4346
4347         $volhash->{$volid}->{referenced_in_config} //= 0;
4348         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4349
4350         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4351             if defined($snapname);
4352
4353         my $size = $drive->{size};
4354         $volhash->{$volid}->{size} //= $size if $size;
4355
4356         $volhash->{$volid}->{is_vmstate} //= 0;
4357         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4358
4359         $volhash->{$volid}->{is_unused} //= 0;
4360         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4361     };
4362
4363     my $include_opts = {
4364         extra_keys => ['vmstate'],
4365         include_unused => 1,
4366     };
4367
4368     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4369     foreach my $snapname (keys %{$conf->{snapshots}}) {
4370         my $snap = $conf->{snapshots}->{$snapname};
4371         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4372     }
4373
4374     foreach my $volid (keys %$volhash) {
4375         &$func($volid, $volhash->{$volid}, @param);
4376     }
4377 }
4378
4379 my $fast_plug_option = {
4380     'lock' => 1,
4381     'name' => 1,
4382     'onboot' => 1,
4383     'shares' => 1,
4384     'startup' => 1,
4385     'description' => 1,
4386     'protection' => 1,
4387     'vmstatestorage' => 1,
4388     'hookscript' => 1,
4389     'tags' => 1,
4390 };
4391
4392 # hotplug changes in [PENDING]
4393 # $selection hash can be used to only apply specified options, for
4394 # example: { cores => 1 } (only apply changed 'cores')
4395 # $errors ref is used to return error messages
4396 sub vmconfig_hotplug_pending {
4397     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4398
4399     my $defaults = load_defaults();
4400     my $arch = get_vm_arch($conf);
4401     my $machine_type = get_vm_machine($conf, undef, $arch);
4402
4403     # commit values which do not have any impact on running VM first
4404     # Note: those option cannot raise errors, we we do not care about
4405     # $selection and always apply them.
4406
4407     my $add_error = sub {
4408         my ($opt, $msg) = @_;
4409         $errors->{$opt} = "hotplug problem - $msg";
4410     };
4411
4412     my $changes = 0;
4413     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4414         if ($fast_plug_option->{$opt}) {
4415             $conf->{$opt} = $conf->{pending}->{$opt};
4416             delete $conf->{pending}->{$opt};
4417             $changes = 1;
4418         }
4419     }
4420
4421     if ($changes) {
4422         PVE::QemuConfig->write_config($vmid, $conf);
4423     }
4424
4425     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4426
4427     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4428     foreach my $opt (sort keys %$pending_delete_hash) {
4429         next if $selection && !$selection->{$opt};
4430         my $force = $pending_delete_hash->{$opt}->{force};
4431         eval {
4432             if ($opt eq 'hotplug') {
4433                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4434             } elsif ($opt eq 'tablet') {
4435                 die "skip\n" if !$hotplug_features->{usb};
4436                 if ($defaults->{tablet}) {
4437                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4438                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4439                         if $arch eq 'aarch64';
4440                 } else {
4441                     vm_deviceunplug($vmid, $conf, 'tablet');
4442                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4443                 }
4444             } elsif ($opt =~ m/^usb\d+/) {
4445                 die "skip\n";
4446                 # since we cannot reliably hot unplug usb devices
4447                 # we are disabling it
4448                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4449                 vm_deviceunplug($vmid, $conf, $opt);
4450             } elsif ($opt eq 'vcpus') {
4451                 die "skip\n" if !$hotplug_features->{cpu};
4452                 qemu_cpu_hotplug($vmid, $conf, undef);
4453             } elsif ($opt eq 'balloon') {
4454                 # enable balloon device is not hotpluggable
4455                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4456                 # here we reset the ballooning value to memory
4457                 my $balloon = $conf->{memory} || $defaults->{memory};
4458                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4459             } elsif ($fast_plug_option->{$opt}) {
4460                 # do nothing
4461             } elsif ($opt =~ m/^net(\d+)$/) {
4462                 die "skip\n" if !$hotplug_features->{network};
4463                 vm_deviceunplug($vmid, $conf, $opt);
4464             } elsif (is_valid_drivename($opt)) {
4465                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4466                 vm_deviceunplug($vmid, $conf, $opt);
4467                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4468             } elsif ($opt =~ m/^memory$/) {
4469                 die "skip\n" if !$hotplug_features->{memory};
4470                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4471             } elsif ($opt eq 'cpuunits') {
4472                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4473             } elsif ($opt eq 'cpulimit') {
4474                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4475             } else {
4476                 die "skip\n";
4477             }
4478         };
4479         if (my $err = $@) {
4480             &$add_error($opt, $err) if $err ne "skip\n";
4481         } else {
4482             delete $conf->{$opt};
4483             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4484         }
4485     }
4486
4487     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4488     $apply_pending_cloudinit = sub {
4489         return if $apply_pending_cloudinit_done; # once is enough
4490         $apply_pending_cloudinit_done = 1; # once is enough
4491
4492         my ($key, $value) = @_;
4493
4494         my @cloudinit_opts = keys %$confdesc_cloudinit;
4495         foreach my $opt (keys %{$conf->{pending}}) {
4496             next if !grep { $_ eq $opt } @cloudinit_opts;
4497             $conf->{$opt} = delete $conf->{pending}->{$opt};
4498         }
4499
4500         my $new_conf = { %$conf };
4501         $new_conf->{$key} = $value;
4502         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4503     };
4504
4505     foreach my $opt (keys %{$conf->{pending}}) {
4506         next if $selection && !$selection->{$opt};
4507         my $value = $conf->{pending}->{$opt};
4508         eval {
4509             if ($opt eq 'hotplug') {
4510                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4511             } elsif ($opt eq 'tablet') {
4512                 die "skip\n" if !$hotplug_features->{usb};
4513                 if ($value == 1) {
4514                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4515                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4516                         if $arch eq 'aarch64';
4517                 } elsif ($value == 0) {
4518                     vm_deviceunplug($vmid, $conf, 'tablet');
4519                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4520                 }
4521             } elsif ($opt =~ m/^usb\d+$/) {
4522                 die "skip\n";
4523                 # since we cannot reliably hot unplug usb devices
4524                 # we are disabling it
4525                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4526                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4527                 die "skip\n" if !$d;
4528                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4529             } elsif ($opt eq 'vcpus') {
4530                 die "skip\n" if !$hotplug_features->{cpu};
4531                 qemu_cpu_hotplug($vmid, $conf, $value);
4532             } elsif ($opt eq 'balloon') {
4533                 # enable/disable balloning device is not hotpluggable
4534                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4535                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4536                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4537
4538                 # allow manual ballooning if shares is set to zero
4539                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4540                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4541                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4542                 }
4543             } elsif ($opt =~ m/^net(\d+)$/) {
4544                 # some changes can be done without hotplug
4545                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4546                                     $vmid, $opt, $value, $arch, $machine_type);
4547             } elsif (is_valid_drivename($opt)) {
4548                 die "skip\n" if $opt eq 'efidisk0';
4549                 # some changes can be done without hotplug
4550                 my $drive = parse_drive($opt, $value);
4551                 if (drive_is_cloudinit($drive)) {
4552                     &$apply_pending_cloudinit($opt, $value);
4553                 }
4554                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4555                                      $vmid, $opt, $value, $arch, $machine_type);
4556             } elsif ($opt =~ m/^memory$/) { #dimms
4557                 die "skip\n" if !$hotplug_features->{memory};
4558                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4559             } elsif ($opt eq 'cpuunits') {
4560                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4561             } elsif ($opt eq 'cpulimit') {
4562                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4563                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4564             } else {
4565                 die "skip\n";  # skip non-hot-pluggable options
4566             }
4567         };
4568         if (my $err = $@) {
4569             &$add_error($opt, $err) if $err ne "skip\n";
4570         } else {
4571             $conf->{$opt} = $value;
4572             delete $conf->{pending}->{$opt};
4573         }
4574     }
4575
4576     PVE::QemuConfig->write_config($vmid, $conf);
4577 }
4578
4579 sub try_deallocate_drive {
4580     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4581
4582     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4583         my $volid = $drive->{file};
4584         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4585             my $sid = PVE::Storage::parse_volume_id($volid);
4586             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4587
4588             # check if the disk is really unused
4589             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4590                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4591             PVE::Storage::vdisk_free($storecfg, $volid);
4592             return 1;
4593         } else {
4594             # If vm is not owner of this disk remove from config
4595             return 1;
4596         }
4597     }
4598
4599     return undef;
4600 }
4601
4602 sub vmconfig_delete_or_detach_drive {
4603     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4604
4605     my $drive = parse_drive($opt, $conf->{$opt});
4606
4607     my $rpcenv = PVE::RPCEnvironment::get();
4608     my $authuser = $rpcenv->get_user();
4609
4610     if ($force) {
4611         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4612         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4613     } else {
4614         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4615     }
4616 }
4617
4618
4619
4620 sub vmconfig_apply_pending {
4621     my ($vmid, $conf, $storecfg, $errors) = @_;
4622
4623     my $add_apply_error = sub {
4624         my ($opt, $msg) = @_;
4625         my $err_msg = "unable to apply pending change $opt : $msg";
4626         $errors->{$opt} = $err_msg;
4627         warn $err_msg;
4628     };
4629
4630     # cold plug
4631
4632     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4633     foreach my $opt (sort keys %$pending_delete_hash) {
4634         my $force = $pending_delete_hash->{$opt}->{force};
4635         eval {
4636             if ($opt =~ m/^unused/) {
4637                 die "internal error";
4638             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4639                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4640             }
4641         };
4642         if (my $err = $@) {
4643             $add_apply_error->($opt, $err);
4644         } else {
4645             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4646             delete $conf->{$opt};
4647         }
4648     }
4649
4650     PVE::QemuConfig->cleanup_pending($conf);
4651
4652     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4653         next if $opt eq 'delete'; # just to be sure
4654         eval {
4655             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4656                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4657             }
4658         };
4659         if (my $err = $@) {
4660             $add_apply_error->($opt, $err);
4661         } else {
4662             $conf->{$opt} = delete $conf->{pending}->{$opt};
4663         }
4664     }
4665
4666     # write all changes at once to avoid unnecessary i/o
4667     PVE::QemuConfig->write_config($vmid, $conf);
4668 }
4669
4670 sub vmconfig_update_net {
4671     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4672
4673     my $newnet = parse_net($value);
4674
4675     if ($conf->{$opt}) {
4676         my $oldnet = parse_net($conf->{$opt});
4677
4678         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4679             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4680             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4681             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4682
4683             # for non online change, we try to hot-unplug
4684             die "skip\n" if !$hotplug;
4685             vm_deviceunplug($vmid, $conf, $opt);
4686         } else {
4687
4688             die "internal error" if $opt !~ m/net(\d+)/;
4689             my $iface = "tap${vmid}i$1";
4690
4691             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4692                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4693                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4694                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4695                 PVE::Network::tap_unplug($iface);
4696
4697                 if ($have_sdn) {
4698                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4699                 } else {
4700                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4701                 }
4702             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4703                 # Rate can be applied on its own but any change above needs to
4704                 # include the rate in tap_plug since OVS resets everything.
4705                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4706             }
4707
4708             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4709                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4710             }
4711
4712             return 1;
4713         }
4714     }
4715
4716     if ($hotplug) {
4717         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4718     } else {
4719         die "skip\n";
4720     }
4721 }
4722
4723 sub vmconfig_update_disk {
4724     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4725
4726     my $drive = parse_drive($opt, $value);
4727
4728     if ($conf->{$opt}) {
4729
4730         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4731
4732             my $media = $drive->{media} || 'disk';
4733             my $oldmedia = $old_drive->{media} || 'disk';
4734             die "unable to change media type\n" if $media ne $oldmedia;
4735
4736             if (!drive_is_cdrom($old_drive)) {
4737
4738                 if ($drive->{file} ne $old_drive->{file}) {
4739
4740                     die "skip\n" if !$hotplug;
4741
4742                     # unplug and register as unused
4743                     vm_deviceunplug($vmid, $conf, $opt);
4744                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4745
4746                 } else {
4747                     # update existing disk
4748
4749                     # skip non hotpluggable value
4750                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4751                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4752                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4753                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4754                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4755                         die "skip\n";
4756                     }
4757
4758                     # apply throttle
4759                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4760                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4761                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4762                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4763                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4764                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4765                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4766                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4767                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4768                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4769                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4770                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4771                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4772                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4773                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4774                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4775                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4776                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4777
4778                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4779                                                    ($drive->{mbps} || 0)*1024*1024,
4780                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4781                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4782                                                    $drive->{iops} || 0,
4783                                                    $drive->{iops_rd} || 0,
4784                                                    $drive->{iops_wr} || 0,
4785                                                    ($drive->{mbps_max} || 0)*1024*1024,
4786                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4787                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4788                                                    $drive->{iops_max} || 0,
4789                                                    $drive->{iops_rd_max} || 0,
4790                                                    $drive->{iops_wr_max} || 0,
4791                                                    $drive->{bps_max_length} || 1,
4792                                                    $drive->{bps_rd_max_length} || 1,
4793                                                    $drive->{bps_wr_max_length} || 1,
4794                                                    $drive->{iops_max_length} || 1,
4795                                                    $drive->{iops_rd_max_length} || 1,
4796                                                    $drive->{iops_wr_max_length} || 1);
4797
4798                     }
4799
4800                     return 1;
4801                 }
4802
4803             } else { # cdrom
4804
4805                 if ($drive->{file} eq 'none') {
4806                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4807                     if (drive_is_cloudinit($old_drive)) {
4808                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4809                     }
4810                 } else {
4811                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4812
4813                     # force eject if locked
4814                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4815
4816                     if ($path) {
4817                         mon_cmd($vmid, "blockdev-change-medium",
4818                             id => "$opt", filename => "$path");
4819                     }
4820                 }
4821
4822                 return 1;
4823             }
4824         }
4825     }
4826
4827     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4828     # hotplug new disks
4829     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4830     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4831 }
4832
4833 # called in locked context by incoming migration
4834 sub vm_migrate_get_nbd_disks {
4835     my ($storecfg, $conf, $replicated_volumes) = @_;
4836
4837     my $local_volumes = {};
4838     PVE::QemuConfig->foreach_volume($conf, sub {
4839         my ($ds, $drive) = @_;
4840
4841         return if drive_is_cdrom($drive);
4842
4843         my $volid = $drive->{file};
4844
4845         return if !$volid;
4846
4847         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4848
4849         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4850         return if $scfg->{shared};
4851
4852         # replicated disks re-use existing state via bitmap
4853         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4854         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4855     });
4856     return $local_volumes;
4857 }
4858
4859 # called in locked context by incoming migration
4860 sub vm_migrate_alloc_nbd_disks {
4861     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4862
4863     my $format = undef;
4864
4865     my $nbd = {};
4866     foreach my $opt (sort keys %$source_volumes) {
4867         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4868
4869         if ($use_existing) {
4870             $nbd->{$opt}->{drivestr} = print_drive($drive);
4871             $nbd->{$opt}->{volid} = $volid;
4872             $nbd->{$opt}->{replicated} = 1;
4873             next;
4874         }
4875
4876         # If a remote storage is specified and the format of the original
4877         # volume is not available there, fall back to the default format.
4878         # Otherwise use the same format as the original.
4879         if (!$storagemap->{identity}) {
4880             $storeid = map_storage($storagemap, $storeid);
4881             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4882             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4883             my $fileFormat = qemu_img_format($scfg, $volname);
4884             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4885         } else {
4886             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4887             $format = qemu_img_format($scfg, $volname);
4888         }
4889
4890         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4891         my $newdrive = $drive;
4892         $newdrive->{format} = $format;
4893         $newdrive->{file} = $newvolid;
4894         my $drivestr = print_drive($newdrive);
4895         $nbd->{$opt}->{drivestr} = $drivestr;
4896         $nbd->{$opt}->{volid} = $newvolid;
4897     }
4898
4899     return $nbd;
4900 }
4901
4902 # see vm_start_nolock for parameters, additionally:
4903 # migrate_opts:
4904 #   storagemap = parsed storage map for allocating NBD disks
4905 sub vm_start {
4906     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4907
4908     return PVE::QemuConfig->lock_config($vmid, sub {
4909         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4910
4911         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4912
4913         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4914
4915         PVE::QemuConfig->check_lock($conf)
4916             if !($params->{skiplock} || $params->{resume});
4917
4918         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4919
4920         if (my $storagemap = $migrate_opts->{storagemap}) {
4921             my $replicated = $migrate_opts->{replicated_volumes};
4922             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4923             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4924
4925             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4926                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4927             }
4928         }
4929
4930         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4931     });
4932 }
4933
4934
4935 # params:
4936 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4937 #   skiplock => 0/1, skip checking for config lock
4938 #   forcemachine => to force Qemu machine (rollback/migration)
4939 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4940 #   timeout => in seconds
4941 #   paused => start VM in paused state (backup)
4942 #   resume => resume from hibernation
4943 # migrate_opts:
4944 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4945 #   migratedfrom => source node
4946 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4947 #   network => CIDR of migration network
4948 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4949 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4950 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4951 sub vm_start_nolock {
4952     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4953
4954     my $statefile = $params->{statefile};
4955     my $resume = $params->{resume};
4956
4957     my $migratedfrom = $migrate_opts->{migratedfrom};
4958     my $migration_type = $migrate_opts->{type};
4959
4960     my $res = {};
4961
4962     # clean up leftover reboot request files
4963     eval { clear_reboot_request($vmid); };
4964     warn $@ if $@;
4965
4966     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4967         vmconfig_apply_pending($vmid, $conf, $storecfg);
4968         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4969     }
4970
4971     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4972
4973     my $defaults = load_defaults();
4974
4975     # set environment variable useful inside network script
4976     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4977
4978     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4979
4980     my $forcemachine = $params->{forcemachine};
4981     my $forcecpu = $params->{forcecpu};
4982     if ($resume) {
4983         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4984         $forcemachine = $conf->{runningmachine};
4985         $forcecpu = $conf->{runningcpu};
4986         print "Resuming suspended VM\n";
4987     }
4988
4989     my ($cmd, $vollist, $spice_port) =
4990         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4991
4992     my $migration_ip;
4993     my $get_migration_ip = sub {
4994         my ($nodename) = @_;
4995
4996         return $migration_ip if defined($migration_ip);
4997
4998         my $cidr = $migrate_opts->{network};
4999
5000         if (!defined($cidr)) {
5001             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5002             $cidr = $dc_conf->{migration}->{network};
5003         }
5004
5005         if (defined($cidr)) {
5006             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5007
5008             die "could not get IP: no address configured on local " .
5009                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5010
5011             die "could not get IP: multiple addresses configured on local " .
5012                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5013
5014             $migration_ip = @$ips[0];
5015         }
5016
5017         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5018             if !defined($migration_ip);
5019
5020         return $migration_ip;
5021     };
5022
5023     my $migrate_uri;
5024     if ($statefile) {
5025         if ($statefile eq 'tcp') {
5026             my $localip = "localhost";
5027             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5028             my $nodename = nodename();
5029
5030             if (!defined($migration_type)) {
5031                 if (defined($datacenterconf->{migration}->{type})) {
5032                     $migration_type = $datacenterconf->{migration}->{type};
5033                 } else {
5034                     $migration_type = 'secure';
5035                 }
5036             }
5037
5038             if ($migration_type eq 'insecure') {
5039                 $localip = $get_migration_ip->($nodename);
5040                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5041             }
5042
5043             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5044             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5045             $migrate_uri = "tcp:${localip}:${migrate_port}";
5046             push @$cmd, '-incoming', $migrate_uri;
5047             push @$cmd, '-S';
5048
5049         } elsif ($statefile eq 'unix') {
5050             # should be default for secure migrations as a ssh TCP forward
5051             # tunnel is not deterministic reliable ready and fails regurarly
5052             # to set up in time, so use UNIX socket forwards
5053             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5054             unlink $socket_addr;
5055
5056             $migrate_uri = "unix:$socket_addr";
5057
5058             push @$cmd, '-incoming', $migrate_uri;
5059             push @$cmd, '-S';
5060
5061         } elsif (-e $statefile) {
5062             push @$cmd, '-loadstate', $statefile;
5063         } else {
5064             my $statepath = PVE::Storage::path($storecfg, $statefile);
5065             push @$vollist, $statefile;
5066             push @$cmd, '-loadstate', $statepath;
5067         }
5068     } elsif ($params->{paused}) {
5069         push @$cmd, '-S';
5070     }
5071
5072     # host pci devices
5073     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5074       my $d = parse_hostpci($conf->{"hostpci$i"});
5075       next if !$d;
5076       my $pcidevices = $d->{pciid};
5077       foreach my $pcidevice (@$pcidevices) {
5078             my $pciid = $pcidevice->{id};
5079
5080             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5081             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5082             die "no pci device info for device '$pciid'\n" if !$info;
5083
5084             if ($d->{mdev}) {
5085                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5086                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5087             } else {
5088                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5089                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5090                 die "can't reset pci device '$pciid'\n"
5091                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5092             }
5093       }
5094     }
5095
5096     PVE::Storage::activate_volumes($storecfg, $vollist);
5097
5098     eval {
5099         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5100             outfunc => sub {}, errfunc => sub {});
5101     };
5102     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5103     # timeout should be more than enough here...
5104     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5105
5106     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5107                                               : $defaults->{cpuunits};
5108
5109     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5110     my %run_params = (
5111         timeout => $statefile ? undef : $start_timeout,
5112         umask => 0077,
5113         noerr => 1,
5114     );
5115
5116     # when migrating, prefix QEMU output so other side can pick up any
5117     # errors that might occur and show the user
5118     if ($migratedfrom) {
5119         $run_params{quiet} = 1;
5120         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5121     }
5122
5123     my %properties = (
5124         Slice => 'qemu.slice',
5125         KillMode => 'none',
5126         CPUShares => $cpuunits
5127     );
5128
5129     if (my $cpulimit = $conf->{cpulimit}) {
5130         $properties{CPUQuota} = int($cpulimit * 100);
5131     }
5132     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5133
5134     my $run_qemu = sub {
5135         PVE::Tools::run_fork sub {
5136             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5137
5138             my $exitcode = run_command($cmd, %run_params);
5139             die "QEMU exited with code $exitcode\n" if $exitcode;
5140         };
5141     };
5142
5143     if ($conf->{hugepages}) {
5144
5145         my $code = sub {
5146             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5147             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5148
5149             PVE::QemuServer::Memory::hugepages_mount();
5150             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5151
5152             eval { $run_qemu->() };
5153             if (my $err = $@) {
5154                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5155                 die $err;
5156             }
5157
5158             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5159         };
5160         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5161
5162     } else {
5163         eval { $run_qemu->() };
5164     }
5165
5166     if (my $err = $@) {
5167         # deactivate volumes if start fails
5168         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5169         die "start failed: $err";
5170     }
5171
5172     print "migration listens on $migrate_uri\n" if $migrate_uri;
5173     $res->{migrate_uri} = $migrate_uri;
5174
5175     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5176         eval { mon_cmd($vmid, "cont"); };
5177         warn $@ if $@;
5178     }
5179
5180     #start nbd server for storage migration
5181     if (my $nbd = $migrate_opts->{nbd}) {
5182         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5183
5184         my $migrate_storage_uri;
5185         # nbd_protocol_version > 0 for unix socket support
5186         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5187             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5188             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5189             $migrate_storage_uri = "nbd:unix:$socket_path";
5190         } else {
5191             my $nodename = nodename();
5192             my $localip = $get_migration_ip->($nodename);
5193             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5194             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5195
5196             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5197             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5198             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5199         }
5200
5201         $res->{migrate_storage_uri} = $migrate_storage_uri;
5202
5203         foreach my $opt (sort keys %$nbd) {
5204             my $drivestr = $nbd->{$opt}->{drivestr};
5205             my $volid = $nbd->{$opt}->{volid};
5206             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5207             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5208             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5209             print "re-using replicated volume: $opt - $volid\n"
5210                 if $nbd->{$opt}->{replicated};
5211
5212             $res->{drives}->{$opt} = $nbd->{$opt};
5213             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5214         }
5215     }
5216
5217     if ($migratedfrom) {
5218         eval {
5219             set_migration_caps($vmid);
5220         };
5221         warn $@ if $@;
5222
5223         if ($spice_port) {
5224             print "spice listens on port $spice_port\n";
5225             $res->{spice_port} = $spice_port;
5226             if ($migrate_opts->{spice_ticket}) {
5227                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5228                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5229             }
5230         }
5231
5232     } else {
5233         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5234             if !$statefile && $conf->{balloon};
5235
5236         foreach my $opt (keys %$conf) {
5237             next if $opt !~  m/^net\d+$/;
5238             my $nicconf = parse_net($conf->{$opt});
5239             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5240         }
5241     }
5242
5243     mon_cmd($vmid, 'qom-set',
5244                 path => "machine/peripheral/balloon0",
5245                 property => "guest-stats-polling-interval",
5246                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5247
5248     if ($resume) {
5249         print "Resumed VM, removing state\n";
5250         if (my $vmstate = $conf->{vmstate}) {
5251             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5252             PVE::Storage::vdisk_free($storecfg, $vmstate);
5253         }
5254         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5255         PVE::QemuConfig->write_config($vmid, $conf);
5256     }
5257
5258     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5259
5260     return $res;
5261 }
5262
5263 sub vm_commandline {
5264     my ($storecfg, $vmid, $snapname) = @_;
5265
5266     my $conf = PVE::QemuConfig->load_config($vmid);
5267     my $forcemachine;
5268     my $forcecpu;
5269
5270     if ($snapname) {
5271         my $snapshot = $conf->{snapshots}->{$snapname};
5272         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5273
5274         # check for machine or CPU overrides in snapshot
5275         $forcemachine = $snapshot->{runningmachine};
5276         $forcecpu = $snapshot->{runningcpu};
5277
5278         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5279
5280         $conf = $snapshot;
5281     }
5282
5283     my $defaults = load_defaults();
5284
5285     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5286         $forcemachine, $forcecpu);
5287
5288     return PVE::Tools::cmd2string($cmd);
5289 }
5290
5291 sub vm_reset {
5292     my ($vmid, $skiplock) = @_;
5293
5294     PVE::QemuConfig->lock_config($vmid, sub {
5295
5296         my $conf = PVE::QemuConfig->load_config($vmid);
5297
5298         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5299
5300         mon_cmd($vmid, "system_reset");
5301     });
5302 }
5303
5304 sub get_vm_volumes {
5305     my ($conf) = @_;
5306
5307     my $vollist = [];
5308     foreach_volid($conf, sub {
5309         my ($volid, $attr) = @_;
5310
5311         return if $volid =~ m|^/|;
5312
5313         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5314         return if !$sid;
5315
5316         push @$vollist, $volid;
5317     });
5318
5319     return $vollist;
5320 }
5321
5322 sub vm_stop_cleanup {
5323     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5324
5325     eval {
5326
5327         if (!$keepActive) {
5328             my $vollist = get_vm_volumes($conf);
5329             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5330         }
5331
5332         foreach my $ext (qw(mon qmp pid vnc qga)) {
5333             unlink "/var/run/qemu-server/${vmid}.$ext";
5334         }
5335
5336         if ($conf->{ivshmem}) {
5337             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5338             # just delete it for now, VMs which have this already open do not
5339             # are affected, but new VMs will get a separated one. If this
5340             # becomes an issue we either add some sort of ref-counting or just
5341             # add a "don't delete on stop" flag to the ivshmem format.
5342             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5343         }
5344
5345         foreach my $key (keys %$conf) {
5346             next if $key !~ m/^hostpci(\d+)$/;
5347             my $hostpciindex = $1;
5348             my $d = parse_hostpci($conf->{$key});
5349             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5350
5351             foreach my $pci (@{$d->{pciid}}) {
5352                 my $pciid = $pci->{id};
5353                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5354             }
5355         }
5356
5357         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5358     };
5359     warn $@ if $@; # avoid errors - just warn
5360 }
5361
5362 # call only in locked context
5363 sub _do_vm_stop {
5364     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5365
5366     my $pid = check_running($vmid, $nocheck);
5367     return if !$pid;
5368
5369     my $conf;
5370     if (!$nocheck) {
5371         $conf = PVE::QemuConfig->load_config($vmid);
5372         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5373         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5374             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5375             $timeout = $opts->{down} if $opts->{down};
5376         }
5377         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5378     }
5379
5380     eval {
5381         if ($shutdown) {
5382             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5383                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5384             } else {
5385                 mon_cmd($vmid, "system_powerdown");
5386             }
5387         } else {
5388             mon_cmd($vmid, "quit");
5389         }
5390     };
5391     my $err = $@;
5392
5393     if (!$err) {
5394         $timeout = 60 if !defined($timeout);
5395
5396         my $count = 0;
5397         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5398             $count++;
5399             sleep 1;
5400         }
5401
5402         if ($count >= $timeout) {
5403             if ($force) {
5404                 warn "VM still running - terminating now with SIGTERM\n";
5405                 kill 15, $pid;
5406             } else {
5407                 die "VM quit/powerdown failed - got timeout\n";
5408             }
5409         } else {
5410             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5411             return;
5412         }
5413     } else {
5414        if (!check_running($vmid, $nocheck)) {
5415             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5416             return;
5417        }
5418        if ($force) {
5419             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5420             kill 15, $pid;
5421         } else {
5422             die "VM quit/powerdown failed\n";
5423         }
5424     }
5425
5426     # wait again
5427     $timeout = 10;
5428
5429     my $count = 0;
5430     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5431         $count++;
5432         sleep 1;
5433     }
5434
5435     if ($count >= $timeout) {
5436         warn "VM still running - terminating now with SIGKILL\n";
5437         kill 9, $pid;
5438         sleep 1;
5439     }
5440
5441     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5442 }
5443
5444 # Note: use $nocheck to skip tests if VM configuration file exists.
5445 # We need that when migration VMs to other nodes (files already moved)
5446 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5447 sub vm_stop {
5448     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5449
5450     $force = 1 if !defined($force) && !$shutdown;
5451
5452     if ($migratedfrom){
5453         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5454         kill 15, $pid if $pid;
5455         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5456         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5457         return;
5458     }
5459
5460     PVE::QemuConfig->lock_config($vmid, sub {
5461         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5462    });
5463 }
5464
5465 sub vm_reboot {
5466     my ($vmid, $timeout) = @_;
5467
5468     PVE::QemuConfig->lock_config($vmid, sub {
5469         eval {
5470
5471             # only reboot if running, as qmeventd starts it again on a stop event
5472             return if !check_running($vmid);
5473
5474             create_reboot_request($vmid);
5475
5476             my $storecfg = PVE::Storage::config();
5477             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5478
5479         };
5480         if (my $err = $@) {
5481             # avoid that the next normal shutdown will be confused for a reboot
5482             clear_reboot_request($vmid);
5483             die $err;
5484         }
5485    });
5486 }
5487
5488 # note: if using the statestorage parameter, the caller has to check privileges
5489 sub vm_suspend {
5490     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5491
5492     my $conf;
5493     my $path;
5494     my $storecfg;
5495     my $vmstate;
5496
5497     PVE::QemuConfig->lock_config($vmid, sub {
5498
5499         $conf = PVE::QemuConfig->load_config($vmid);
5500
5501         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5502         PVE::QemuConfig->check_lock($conf)
5503             if !($skiplock || $is_backing_up);
5504
5505         die "cannot suspend to disk during backup\n"
5506             if $is_backing_up && $includestate;
5507
5508         if ($includestate) {
5509             $conf->{lock} = 'suspending';
5510             my $date = strftime("%Y-%m-%d", localtime(time()));
5511             $storecfg = PVE::Storage::config();
5512             if (!$statestorage) {
5513                 $statestorage = find_vmstate_storage($conf, $storecfg);
5514                 # check permissions for the storage
5515                 my $rpcenv = PVE::RPCEnvironment::get();
5516                 if ($rpcenv->{type} ne 'cli') {
5517                     my $authuser = $rpcenv->get_user();
5518                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5519                 }
5520             }
5521
5522
5523             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5524             $path = PVE::Storage::path($storecfg, $vmstate);
5525             PVE::QemuConfig->write_config($vmid, $conf);
5526         } else {
5527             mon_cmd($vmid, "stop");
5528         }
5529     });
5530
5531     if ($includestate) {
5532         # save vm state
5533         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5534
5535         eval {
5536             mon_cmd($vmid, "savevm-start", statefile => $path);
5537             for(;;) {
5538                 my $state = mon_cmd($vmid, "query-savevm");
5539                 if (!$state->{status}) {
5540                     die "savevm not active\n";
5541                 } elsif ($state->{status} eq 'active') {
5542                     sleep(1);
5543                     next;
5544                 } elsif ($state->{status} eq 'completed') {
5545                     print "State saved, quitting\n";
5546                     last;
5547                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5548                     die "query-savevm failed with error '$state->{error}'\n"
5549                 } else {
5550                     die "query-savevm returned status '$state->{status}'\n";
5551                 }
5552             }
5553         };
5554         my $err = $@;
5555
5556         PVE::QemuConfig->lock_config($vmid, sub {
5557             $conf = PVE::QemuConfig->load_config($vmid);
5558             if ($err) {
5559                 # cleanup, but leave suspending lock, to indicate something went wrong
5560                 eval {
5561                     mon_cmd($vmid, "savevm-end");
5562                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5563                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5564                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5565                     PVE::QemuConfig->write_config($vmid, $conf);
5566                 };
5567                 warn $@ if $@;
5568                 die $err;
5569             }
5570
5571             die "lock changed unexpectedly\n"
5572                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5573
5574             mon_cmd($vmid, "quit");
5575             $conf->{lock} = 'suspended';
5576             PVE::QemuConfig->write_config($vmid, $conf);
5577         });
5578     }
5579 }
5580
5581 sub vm_resume {
5582     my ($vmid, $skiplock, $nocheck) = @_;
5583
5584     PVE::QemuConfig->lock_config($vmid, sub {
5585         my $res = mon_cmd($vmid, 'query-status');
5586         my $resume_cmd = 'cont';
5587
5588         if ($res->{status} && $res->{status} eq 'suspended') {
5589             $resume_cmd = 'system_wakeup';
5590         }
5591
5592         if (!$nocheck) {
5593
5594             my $conf = PVE::QemuConfig->load_config($vmid);
5595
5596             PVE::QemuConfig->check_lock($conf)
5597                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5598         }
5599
5600         mon_cmd($vmid, $resume_cmd);
5601     });
5602 }
5603
5604 sub vm_sendkey {
5605     my ($vmid, $skiplock, $key) = @_;
5606
5607     PVE::QemuConfig->lock_config($vmid, sub {
5608
5609         my $conf = PVE::QemuConfig->load_config($vmid);
5610
5611         # there is no qmp command, so we use the human monitor command
5612         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5613         die $res if $res ne '';
5614     });
5615 }
5616
5617 # vzdump restore implementaion
5618
5619 sub tar_archive_read_firstfile {
5620     my $archive = shift;
5621
5622     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5623
5624     # try to detect archive type first
5625     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5626         die "unable to open file '$archive'\n";
5627     my $firstfile = <$fh>;
5628     kill 15, $pid;
5629     close $fh;
5630
5631     die "ERROR: archive contaions no data\n" if !$firstfile;
5632     chomp $firstfile;
5633
5634     return $firstfile;
5635 }
5636
5637 sub tar_restore_cleanup {
5638     my ($storecfg, $statfile) = @_;
5639
5640     print STDERR "starting cleanup\n";
5641
5642     if (my $fd = IO::File->new($statfile, "r")) {
5643         while (defined(my $line = <$fd>)) {
5644             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5645                 my $volid = $2;
5646                 eval {
5647                     if ($volid =~ m|^/|) {
5648                         unlink $volid || die 'unlink failed\n';
5649                     } else {
5650                         PVE::Storage::vdisk_free($storecfg, $volid);
5651                     }
5652                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5653                 };
5654                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5655             } else {
5656                 print STDERR "unable to parse line in statfile - $line";
5657             }
5658         }
5659         $fd->close();
5660     }
5661 }
5662
5663 sub restore_file_archive {
5664     my ($archive, $vmid, $user, $opts) = @_;
5665
5666     return restore_vma_archive($archive, $vmid, $user, $opts)
5667         if $archive eq '-';
5668
5669     my $info = PVE::Storage::archive_info($archive);
5670     my $format = $opts->{format} // $info->{format};
5671     my $comp = $info->{compression};
5672
5673     # try to detect archive format
5674     if ($format eq 'tar') {
5675         return restore_tar_archive($archive, $vmid, $user, $opts);
5676     } else {
5677         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5678     }
5679 }
5680
5681 # hepler to remove disks that will not be used after restore
5682 my $restore_cleanup_oldconf = sub {
5683     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5684
5685     PVE::QemuConfig->foreach_volume($oldconf, sub {
5686         my ($ds, $drive) = @_;
5687
5688         return if drive_is_cdrom($drive, 1);
5689
5690         my $volid = $drive->{file};
5691         return if !$volid || $volid =~ m|^/|;
5692
5693         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5694         return if !$path || !$owner || ($owner != $vmid);
5695
5696         # Note: only delete disk we want to restore
5697         # other volumes will become unused
5698         if ($virtdev_hash->{$ds}) {
5699             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5700             if (my $err = $@) {
5701                 warn $err;
5702             }
5703         }
5704     });
5705
5706     # delete vmstate files, after the restore we have no snapshots anymore
5707     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5708         my $snap = $oldconf->{snapshots}->{$snapname};
5709         if ($snap->{vmstate}) {
5710             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5711             if (my $err = $@) {
5712                 warn $err;
5713             }
5714         }
5715     }
5716 };
5717
5718 # Helper to parse vzdump backup device hints
5719 #
5720 # $rpcenv: Environment, used to ckeck storage permissions
5721 # $user: User ID, to check storage permissions
5722 # $storecfg: Storage configuration
5723 # $fh: the file handle for reading the configuration
5724 # $devinfo: should contain device sizes for all backu-up'ed devices
5725 # $options: backup options (pool, default storage)
5726 #
5727 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5728 my $parse_backup_hints = sub {
5729     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5730
5731     my $virtdev_hash = {};
5732
5733     while (defined(my $line = <$fh>)) {
5734         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5735             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5736             die "archive does not contain data for drive '$virtdev'\n"
5737                 if !$devinfo->{$devname};
5738
5739             if (defined($options->{storage})) {
5740                 $storeid = $options->{storage} || 'local';
5741             } elsif (!$storeid) {
5742                 $storeid = 'local';
5743             }
5744             $format = 'raw' if !$format;
5745             $devinfo->{$devname}->{devname} = $devname;
5746             $devinfo->{$devname}->{virtdev} = $virtdev;
5747             $devinfo->{$devname}->{format} = $format;
5748             $devinfo->{$devname}->{storeid} = $storeid;
5749
5750             # check permission on storage
5751             my $pool = $options->{pool}; # todo: do we need that?
5752             if ($user ne 'root@pam') {
5753                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5754             }
5755
5756             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5757         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5758             my $virtdev = $1;
5759             my $drive = parse_drive($virtdev, $2);
5760             if (drive_is_cloudinit($drive)) {
5761                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5762                 $storeid = $options->{storage} if defined ($options->{storage});
5763                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5764                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5765
5766                 $virtdev_hash->{$virtdev} = {
5767                     format => $format,
5768                     storeid => $storeid,
5769                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5770                     is_cloudinit => 1,
5771                 };
5772             }
5773         }
5774     }
5775
5776     return $virtdev_hash;
5777 };
5778
5779 # Helper to allocate and activate all volumes required for a restore
5780 #
5781 # $storecfg: Storage configuration
5782 # $virtdev_hash: as returned by parse_backup_hints()
5783 #
5784 # Returns: { $virtdev => $volid }
5785 my $restore_allocate_devices = sub {
5786     my ($storecfg, $virtdev_hash, $vmid) = @_;
5787
5788     my $map = {};
5789     foreach my $virtdev (sort keys %$virtdev_hash) {
5790         my $d = $virtdev_hash->{$virtdev};
5791         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5792         my $storeid = $d->{storeid};
5793         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5794
5795         # test if requested format is supported
5796         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5797         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5798         $d->{format} = $defFormat if !$supported;
5799
5800         my $name;
5801         if ($d->{is_cloudinit}) {
5802             $name = "vm-$vmid-cloudinit";
5803             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5804         }
5805
5806         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5807
5808         print STDERR "new volume ID is '$volid'\n";
5809         $d->{volid} = $volid;
5810
5811         PVE::Storage::activate_volumes($storecfg, [$volid]);
5812
5813         $map->{$virtdev} = $volid;
5814     }
5815
5816     return $map;
5817 };
5818
5819 my $restore_update_config_line = sub {
5820     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5821
5822     return if $line =~ m/^\#qmdump\#/;
5823     return if $line =~ m/^\#vzdump\#/;
5824     return if $line =~ m/^lock:/;
5825     return if $line =~ m/^unused\d+:/;
5826     return if $line =~ m/^parent:/;
5827
5828     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5829     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5830         # try to convert old 1.X settings
5831         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5832         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5833             my ($model, $macaddr) = split(/\=/, $devconfig);
5834             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5835             my $net = {
5836                 model => $model,
5837                 bridge => "vmbr$ind",
5838                 macaddr => $macaddr,
5839             };
5840             my $netstr = print_net($net);
5841
5842             print $outfd "net$cookie->{netcount}: $netstr\n";
5843             $cookie->{netcount}++;
5844         }
5845     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5846         my ($id, $netstr) = ($1, $2);
5847         my $net = parse_net($netstr);
5848         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5849         $netstr = print_net($net);
5850         print $outfd "$id: $netstr\n";
5851     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5852         my $virtdev = $1;
5853         my $value = $3;
5854         my $di = parse_drive($virtdev, $value);
5855         if (defined($di->{backup}) && !$di->{backup}) {
5856             print $outfd "#$line";
5857         } elsif ($map->{$virtdev}) {
5858             delete $di->{format}; # format can change on restore
5859             $di->{file} = $map->{$virtdev};
5860             $value = print_drive($di);
5861             print $outfd "$virtdev: $value\n";
5862         } else {
5863             print $outfd $line;
5864         }
5865     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5866         my $vmgenid = $1;
5867         if ($vmgenid ne '0') {
5868             # always generate a new vmgenid if there was a valid one setup
5869             $vmgenid = generate_uuid();
5870         }
5871         print $outfd "vmgenid: $vmgenid\n";
5872     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5873         my ($uuid, $uuid_str);
5874         UUID::generate($uuid);
5875         UUID::unparse($uuid, $uuid_str);
5876         my $smbios1 = parse_smbios1($2);
5877         $smbios1->{uuid} = $uuid_str;
5878         print $outfd $1.print_smbios1($smbios1)."\n";
5879     } else {
5880         print $outfd $line;
5881     }
5882 };
5883
5884 my $restore_deactivate_volumes = sub {
5885     my ($storecfg, $devinfo) = @_;
5886
5887     my $vollist = [];
5888     foreach my $devname (keys %$devinfo) {
5889         my $volid = $devinfo->{$devname}->{volid};
5890         push @$vollist, $volid if $volid;
5891     }
5892
5893     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5894 };
5895
5896 my $restore_destroy_volumes = sub {
5897     my ($storecfg, $devinfo) = @_;
5898
5899     foreach my $devname (keys %$devinfo) {
5900         my $volid = $devinfo->{$devname}->{volid};
5901         next if !$volid;
5902         eval {
5903             if ($volid =~ m|^/|) {
5904                 unlink $volid || die 'unlink failed\n';
5905             } else {
5906                 PVE::Storage::vdisk_free($storecfg, $volid);
5907             }
5908             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5909         };
5910         print STDERR "unable to cleanup '$volid' - $@" if $@;
5911     }
5912 };
5913
5914 sub scan_volids {
5915     my ($cfg, $vmid) = @_;
5916
5917     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5918
5919     my $volid_hash = {};
5920     foreach my $storeid (keys %$info) {
5921         foreach my $item (@{$info->{$storeid}}) {
5922             next if !($item->{volid} && $item->{size});
5923             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5924             $volid_hash->{$item->{volid}} = $item;
5925         }
5926     }
5927
5928     return $volid_hash;
5929 }
5930
5931 sub update_disk_config {
5932     my ($vmid, $conf, $volid_hash) = @_;
5933
5934     my $changes;
5935     my $prefix = "VM $vmid:";
5936
5937     # used and unused disks
5938     my $referenced = {};
5939
5940     # Note: it is allowed to define multiple storages with same path (alias), so
5941     # we need to check both 'volid' and real 'path' (two different volid can point
5942     # to the same path).
5943
5944     my $referencedpath = {};
5945
5946     # update size info
5947     PVE::QemuConfig->foreach_volume($conf, sub {
5948         my ($opt, $drive) = @_;
5949
5950         my $volid = $drive->{file};
5951         return if !$volid;
5952
5953         # mark volid as "in-use" for next step
5954         $referenced->{$volid} = 1;
5955         if ($volid_hash->{$volid} &&
5956             (my $path = $volid_hash->{$volid}->{path})) {
5957             $referencedpath->{$path} = 1;
5958         }
5959
5960         return if drive_is_cdrom($drive);
5961         return if !$volid_hash->{$volid};
5962
5963         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5964         if (defined($updated)) {
5965             $changes = 1;
5966             $conf->{$opt} = print_drive($updated);
5967             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5968         }
5969     });
5970
5971     # remove 'unusedX' entry if volume is used
5972     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5973         my ($opt, $drive) = @_;
5974
5975         my $volid = $drive->{file};
5976         return if !$volid;
5977
5978         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5979         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5980             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5981             $changes = 1;
5982             delete $conf->{$opt};
5983         }
5984
5985         $referenced->{$volid} = 1;
5986         $referencedpath->{$path} = 1 if $path;
5987     });
5988
5989     foreach my $volid (sort keys %$volid_hash) {
5990         next if $volid =~ m/vm-$vmid-state-/;
5991         next if $referenced->{$volid};
5992         my $path = $volid_hash->{$volid}->{path};
5993         next if !$path; # just to be sure
5994         next if $referencedpath->{$path};
5995         $changes = 1;
5996         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5997         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5998         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5999     }
6000
6001     return $changes;
6002 }
6003
6004 sub rescan {
6005     my ($vmid, $nolock, $dryrun) = @_;
6006
6007     my $cfg = PVE::Storage::config();
6008
6009     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6010     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6011     foreach my $stor (keys %{$cfg->{ids}}) {
6012         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6013     }
6014
6015     print "rescan volumes...\n";
6016     my $volid_hash = scan_volids($cfg, $vmid);
6017
6018     my $updatefn =  sub {
6019         my ($vmid) = @_;
6020
6021         my $conf = PVE::QemuConfig->load_config($vmid);
6022
6023         PVE::QemuConfig->check_lock($conf);
6024
6025         my $vm_volids = {};
6026         foreach my $volid (keys %$volid_hash) {
6027             my $info = $volid_hash->{$volid};
6028             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6029         }
6030
6031         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6032
6033         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6034     };
6035
6036     if (defined($vmid)) {
6037         if ($nolock) {
6038             &$updatefn($vmid);
6039         } else {
6040             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6041         }
6042     } else {
6043         my $vmlist = config_list();
6044         foreach my $vmid (keys %$vmlist) {
6045             if ($nolock) {
6046                 &$updatefn($vmid);
6047             } else {
6048                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6049             }
6050         }
6051     }
6052 }
6053
6054 sub restore_proxmox_backup_archive {
6055     my ($archive, $vmid, $user, $options) = @_;
6056
6057     my $storecfg = PVE::Storage::config();
6058
6059     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6060     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6061
6062     my $server = $scfg->{server};
6063     my $datastore = $scfg->{datastore};
6064     my $username = $scfg->{username} // 'root@pam';
6065     my $fingerprint = $scfg->{fingerprint};
6066
6067     my $repo = "$username\@$server:$datastore";
6068     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6069     local $ENV{PBS_PASSWORD} = $password;
6070     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6071
6072     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6073         PVE::Storage::parse_volname($storecfg, $archive);
6074
6075     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6076
6077     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6078
6079     my $tmpdir = "/var/tmp/vzdumptmp$$";
6080     rmtree $tmpdir;
6081     mkpath $tmpdir;
6082
6083     my $conffile = PVE::QemuConfig->config_file($vmid);
6084     my $tmpfn = "$conffile.$$.tmp";
6085      # disable interrupts (always do cleanups)
6086     local $SIG{INT} =
6087         local $SIG{TERM} =
6088         local $SIG{QUIT} =
6089         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6090
6091     # Note: $oldconf is undef if VM does not exists
6092     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6093     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6094
6095     my $rpcenv = PVE::RPCEnvironment::get();
6096     my $devinfo = {};
6097
6098     eval {
6099         # enable interrupts
6100         local $SIG{INT} =
6101             local $SIG{TERM} =
6102             local $SIG{QUIT} =
6103             local $SIG{HUP} =
6104             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6105
6106         my $cfgfn = "$tmpdir/qemu-server.conf";
6107         my $firewall_config_fn = "$tmpdir/fw.conf";
6108         my $index_fn = "$tmpdir/index.json";
6109
6110         my $cmd = "restore";
6111
6112         my $param = [$pbs_backup_name, "index.json", $index_fn];
6113         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6114         my $index = PVE::Tools::file_get_contents($index_fn);
6115         $index = decode_json($index);
6116
6117         # print Dumper($index);
6118         foreach my $info (@{$index->{files}}) {
6119             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6120                 my $devname = $1;
6121                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6122                     $devinfo->{$devname}->{size} = $1;
6123                 } else {
6124                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6125                 }
6126             }
6127         }
6128
6129         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6130         if (!$is_qemu_server_backup) {
6131             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6132         }
6133         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6134
6135         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6136         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6137
6138         if ($has_firewall_config) {
6139             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6140             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6141
6142             my $pve_firewall_dir = '/etc/pve/firewall';
6143             mkdir $pve_firewall_dir; # make sure the dir exists
6144             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6145         }
6146
6147         my $fh = IO::File->new($cfgfn, "r") ||
6148             "unable to read qemu-server.conf - $!\n";
6149
6150         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6151
6152         # fixme: rate limit?
6153
6154         # create empty/temp config
6155         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6156
6157         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6158
6159         # allocate volumes
6160         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6161
6162         foreach my $virtdev (sort keys %$virtdev_hash) {
6163             my $d = $virtdev_hash->{$virtdev};
6164             next if $d->{is_cloudinit}; # no need to restore cloudinit
6165
6166             my $volid = $d->{volid};
6167
6168             my $path = PVE::Storage::path($storecfg, $volid);
6169
6170             my $pbs_restore_cmd = [
6171                 '/usr/bin/pbs-restore',
6172                 '--repository', $repo,
6173                 $pbs_backup_name,
6174                 "$d->{devname}.img.fidx",
6175                 $path,
6176                 '--verbose',
6177                 ];
6178
6179             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6180
6181             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6182                 push @$pbs_restore_cmd, '--skip-zero';
6183             }
6184
6185             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6186             print "restore proxmox backup image: $dbg_cmdstring\n";
6187             run_command($pbs_restore_cmd);
6188         }
6189
6190         $fh->seek(0, 0) || die "seek failed - $!\n";
6191
6192         my $outfd = new IO::File ($tmpfn, "w") ||
6193             die "unable to write config for VM $vmid\n";
6194
6195         my $cookie = { netcount => 0 };
6196         while (defined(my $line = <$fh>)) {
6197             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6198         }
6199
6200         $fh->close();
6201         $outfd->close();
6202     };
6203     my $err = $@;
6204
6205     $restore_deactivate_volumes->($storecfg, $devinfo);
6206
6207     rmtree $tmpdir;
6208
6209     if ($err) {
6210         unlink $tmpfn;
6211         $restore_destroy_volumes->($storecfg, $devinfo);
6212         die $err;
6213     }
6214
6215     rename($tmpfn, $conffile) ||
6216         die "unable to commit configuration file '$conffile'\n";
6217
6218     PVE::Cluster::cfs_update(); # make sure we read new file
6219
6220     eval { rescan($vmid, 1); };
6221     warn $@ if $@;
6222 }
6223
6224 sub restore_vma_archive {
6225     my ($archive, $vmid, $user, $opts, $comp) = @_;
6226
6227     my $readfrom = $archive;
6228
6229     my $cfg = PVE::Storage::config();
6230     my $commands = [];
6231     my $bwlimit = $opts->{bwlimit};
6232
6233     my $dbg_cmdstring = '';
6234     my $add_pipe = sub {
6235         my ($cmd) = @_;
6236         push @$commands, $cmd;
6237         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6238         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6239         $readfrom = '-';
6240     };
6241
6242     my $input = undef;
6243     if ($archive eq '-') {
6244         $input = '<&STDIN';
6245     } else {
6246         # If we use a backup from a PVE defined storage we also consider that
6247         # storage's rate limit:
6248         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6249         if (defined($volid)) {
6250             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6251             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6252             if ($readlimit) {
6253                 print STDERR "applying read rate limit: $readlimit\n";
6254                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6255                 $add_pipe->($cstream);
6256             }
6257         }
6258     }
6259
6260     if ($comp) {
6261         my $info = PVE::Storage::decompressor_info('vma', $comp);
6262         my $cmd = $info->{decompressor};
6263         push @$cmd, $readfrom;
6264         $add_pipe->($cmd);
6265     }
6266
6267     my $tmpdir = "/var/tmp/vzdumptmp$$";
6268     rmtree $tmpdir;
6269
6270     # disable interrupts (always do cleanups)
6271     local $SIG{INT} =
6272         local $SIG{TERM} =
6273         local $SIG{QUIT} =
6274         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6275
6276     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6277     POSIX::mkfifo($mapfifo, 0600);
6278     my $fifofh;
6279
6280     my $openfifo = sub {
6281         open($fifofh, '>', $mapfifo) || die $!;
6282     };
6283
6284     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6285
6286     my $oldtimeout;
6287     my $timeout = 5;
6288
6289     my $devinfo = {};
6290
6291     my $rpcenv = PVE::RPCEnvironment::get();
6292
6293     my $conffile = PVE::QemuConfig->config_file($vmid);
6294     my $tmpfn = "$conffile.$$.tmp";
6295
6296     # Note: $oldconf is undef if VM does not exist
6297     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6298     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6299
6300     my %storage_limits;
6301
6302     my $print_devmap = sub {
6303         my $cfgfn = "$tmpdir/qemu-server.conf";
6304
6305         # we can read the config - that is already extracted
6306         my $fh = IO::File->new($cfgfn, "r") ||
6307             "unable to read qemu-server.conf - $!\n";
6308
6309         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6310         if (-f $fwcfgfn) {
6311             my $pve_firewall_dir = '/etc/pve/firewall';
6312             mkdir $pve_firewall_dir; # make sure the dir exists
6313             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6314         }
6315
6316         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6317
6318         foreach my $key (keys %storage_limits) {
6319             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6320             next if !$limit;
6321             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6322             $storage_limits{$key} = $limit * 1024;
6323         }
6324
6325         foreach my $devname (keys %$devinfo) {
6326             die "found no device mapping information for device '$devname'\n"
6327                 if !$devinfo->{$devname}->{virtdev};
6328         }
6329
6330         # create empty/temp config
6331         if ($oldconf) {
6332             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6333             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6334         }
6335
6336         # allocate volumes
6337         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6338
6339         # print restore information to $fifofh
6340         foreach my $virtdev (sort keys %$virtdev_hash) {
6341             my $d = $virtdev_hash->{$virtdev};
6342             next if $d->{is_cloudinit}; # no need to restore cloudinit
6343
6344             my $storeid = $d->{storeid};
6345             my $volid = $d->{volid};
6346
6347             my $map_opts = '';
6348             if (my $limit = $storage_limits{$storeid}) {
6349                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6350             }
6351
6352             my $write_zeros = 1;
6353             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6354                 $write_zeros = 0;
6355             }
6356
6357             my $path = PVE::Storage::path($cfg, $volid);
6358
6359             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6360
6361             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6362         }
6363
6364         $fh->seek(0, 0) || die "seek failed - $!\n";
6365
6366         my $outfd = new IO::File ($tmpfn, "w") ||
6367             die "unable to write config for VM $vmid\n";
6368
6369         my $cookie = { netcount => 0 };
6370         while (defined(my $line = <$fh>)) {
6371             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6372         }
6373
6374         $fh->close();
6375         $outfd->close();
6376     };
6377
6378     eval {
6379         # enable interrupts
6380         local $SIG{INT} =
6381             local $SIG{TERM} =
6382             local $SIG{QUIT} =
6383             local $SIG{HUP} =
6384             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6385         local $SIG{ALRM} = sub { die "got timeout\n"; };
6386
6387         $oldtimeout = alarm($timeout);
6388
6389         my $parser = sub {
6390             my $line = shift;
6391
6392             print "$line\n";
6393
6394             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6395                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6396                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6397             } elsif ($line =~ m/^CTIME: /) {
6398                 # we correctly received the vma config, so we can disable
6399                 # the timeout now for disk allocation (set to 10 minutes, so
6400                 # that we always timeout if something goes wrong)
6401                 alarm(600);
6402                 &$print_devmap();
6403                 print $fifofh "done\n";
6404                 my $tmp = $oldtimeout || 0;
6405                 $oldtimeout = undef;
6406                 alarm($tmp);
6407                 close($fifofh);
6408             }
6409         };
6410
6411         print "restore vma archive: $dbg_cmdstring\n";
6412         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6413     };
6414     my $err = $@;
6415
6416     alarm($oldtimeout) if $oldtimeout;
6417
6418     $restore_deactivate_volumes->($cfg, $devinfo);
6419
6420     unlink $mapfifo;
6421     rmtree $tmpdir;
6422
6423     if ($err) {
6424         unlink $tmpfn;
6425         $restore_destroy_volumes->($cfg, $devinfo);
6426         die $err;
6427     }
6428
6429     rename($tmpfn, $conffile) ||
6430         die "unable to commit configuration file '$conffile'\n";
6431
6432     PVE::Cluster::cfs_update(); # make sure we read new file
6433
6434     eval { rescan($vmid, 1); };
6435     warn $@ if $@;
6436 }
6437
6438 sub restore_tar_archive {
6439     my ($archive, $vmid, $user, $opts) = @_;
6440
6441     if ($archive ne '-') {
6442         my $firstfile = tar_archive_read_firstfile($archive);
6443         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6444             if $firstfile ne 'qemu-server.conf';
6445     }
6446
6447     my $storecfg = PVE::Storage::config();
6448
6449     # avoid zombie disks when restoring over an existing VM -> cleanup first
6450     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6451     # skiplock=1 because qmrestore has set the 'create' lock itself already
6452     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6453     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6454
6455     my $tocmd = "/usr/lib/qemu-server/qmextract";
6456
6457     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6458     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6459     $tocmd .= ' --prealloc' if $opts->{prealloc};
6460     $tocmd .= ' --info' if $opts->{info};
6461
6462     # tar option "xf" does not autodetect compression when read from STDIN,
6463     # so we pipe to zcat
6464     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6465         PVE::Tools::shellquote("--to-command=$tocmd");
6466
6467     my $tmpdir = "/var/tmp/vzdumptmp$$";
6468     mkpath $tmpdir;
6469
6470     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6471     local $ENV{VZDUMP_VMID} = $vmid;
6472     local $ENV{VZDUMP_USER} = $user;
6473
6474     my $conffile = PVE::QemuConfig->config_file($vmid);
6475     my $tmpfn = "$conffile.$$.tmp";
6476
6477     # disable interrupts (always do cleanups)
6478     local $SIG{INT} =
6479         local $SIG{TERM} =
6480         local $SIG{QUIT} =
6481         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6482
6483     eval {
6484         # enable interrupts
6485         local $SIG{INT} =
6486             local $SIG{TERM} =
6487             local $SIG{QUIT} =
6488             local $SIG{HUP} =
6489             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6490
6491         if ($archive eq '-') {
6492             print "extracting archive from STDIN\n";
6493             run_command($cmd, input => "<&STDIN");
6494         } else {
6495             print "extracting archive '$archive'\n";
6496             run_command($cmd);
6497         }
6498
6499         return if $opts->{info};
6500
6501         # read new mapping
6502         my $map = {};
6503         my $statfile = "$tmpdir/qmrestore.stat";
6504         if (my $fd = IO::File->new($statfile, "r")) {
6505             while (defined (my $line = <$fd>)) {
6506                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6507                     $map->{$1} = $2 if $1;
6508                 } else {
6509                     print STDERR "unable to parse line in statfile - $line\n";
6510                 }
6511             }
6512             $fd->close();
6513         }
6514
6515         my $confsrc = "$tmpdir/qemu-server.conf";
6516
6517         my $srcfd = new IO::File($confsrc, "r") ||
6518             die "unable to open file '$confsrc'\n";
6519
6520         my $outfd = new IO::File ($tmpfn, "w") ||
6521             die "unable to write config for VM $vmid\n";
6522
6523         my $cookie = { netcount => 0 };
6524         while (defined (my $line = <$srcfd>)) {
6525             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6526         }
6527
6528         $srcfd->close();
6529         $outfd->close();
6530     };
6531     if (my $err = $@) {
6532         unlink $tmpfn;
6533         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6534         die $err;
6535     }
6536
6537     rmtree $tmpdir;
6538
6539     rename $tmpfn, $conffile ||
6540         die "unable to commit configuration file '$conffile'\n";
6541
6542     PVE::Cluster::cfs_update(); # make sure we read new file
6543
6544     eval { rescan($vmid, 1); };
6545     warn $@ if $@;
6546 };
6547
6548 sub foreach_storage_used_by_vm {
6549     my ($conf, $func) = @_;
6550
6551     my $sidhash = {};
6552
6553     PVE::QemuConfig->foreach_volume($conf, sub {
6554         my ($ds, $drive) = @_;
6555         return if drive_is_cdrom($drive);
6556
6557         my $volid = $drive->{file};
6558
6559         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6560         $sidhash->{$sid} = $sid if $sid;
6561     });
6562
6563     foreach my $sid (sort keys %$sidhash) {
6564         &$func($sid);
6565     }
6566 }
6567
6568 my $qemu_snap_storage = {
6569     rbd => 1,
6570 };
6571 sub do_snapshots_with_qemu {
6572     my ($storecfg, $volid) = @_;
6573
6574     my $storage_name = PVE::Storage::parse_volume_id($volid);
6575     my $scfg = $storecfg->{ids}->{$storage_name};
6576
6577     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6578         return 1;
6579     }
6580
6581     if ($volid =~ m/\.(qcow2|qed)$/){
6582         return 1;
6583     }
6584
6585     return undef;
6586 }
6587
6588 sub qga_check_running {
6589     my ($vmid, $nowarn) = @_;
6590
6591     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6592     if ($@) {
6593         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6594         return 0;
6595     }
6596     return 1;
6597 }
6598
6599 sub template_create {
6600     my ($vmid, $conf, $disk) = @_;
6601
6602     my $storecfg = PVE::Storage::config();
6603
6604     PVE::QemuConfig->foreach_volume($conf, sub {
6605         my ($ds, $drive) = @_;
6606
6607         return if drive_is_cdrom($drive);
6608         return if $disk && $ds ne $disk;
6609
6610         my $volid = $drive->{file};
6611         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6612
6613         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6614         $drive->{file} = $voliddst;
6615         $conf->{$ds} = print_drive($drive);
6616         PVE::QemuConfig->write_config($vmid, $conf);
6617     });
6618 }
6619
6620 sub convert_iscsi_path {
6621     my ($path) = @_;
6622
6623     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6624         my $portal = $1;
6625         my $target = $2;
6626         my $lun = $3;
6627
6628         my $initiator_name = get_initiator_name();
6629
6630         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6631                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6632     }
6633
6634     die "cannot convert iscsi path '$path', unkown format\n";
6635 }
6636
6637 sub qemu_img_convert {
6638     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6639
6640     my $storecfg = PVE::Storage::config();
6641     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6642     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6643
6644     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6645
6646     my $cachemode;
6647     my $src_path;
6648     my $src_is_iscsi = 0;
6649     my $src_format;
6650
6651     if ($src_storeid) {
6652         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6653         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6654         $src_format = qemu_img_format($src_scfg, $src_volname);
6655         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6656         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6657         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6658     } elsif (-f $src_volid) {
6659         $src_path = $src_volid;
6660         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6661             $src_format = $1;
6662         }
6663     }
6664
6665     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6666
6667     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6668     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6669     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6670     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6671
6672     my $cmd = [];
6673     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6674     push @$cmd, '-l', "snapshot.name=$snapname"
6675         if $snapname && $src_format && $src_format eq "qcow2";
6676     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6677     push @$cmd, '-T', $cachemode if defined($cachemode);
6678
6679     if ($src_is_iscsi) {
6680         push @$cmd, '--image-opts';
6681         $src_path = convert_iscsi_path($src_path);
6682     } elsif ($src_format) {
6683         push @$cmd, '-f', $src_format;
6684     }
6685
6686     if ($dst_is_iscsi) {
6687         push @$cmd, '--target-image-opts';
6688         $dst_path = convert_iscsi_path($dst_path);
6689     } else {
6690         push @$cmd, '-O', $dst_format;
6691     }
6692
6693     push @$cmd, $src_path;
6694
6695     if (!$dst_is_iscsi && $is_zero_initialized) {
6696         push @$cmd, "zeroinit:$dst_path";
6697     } else {
6698         push @$cmd, $dst_path;
6699     }
6700
6701     my $parser = sub {
6702         my $line = shift;
6703         if($line =~ m/\((\S+)\/100\%\)/){
6704             my $percent = $1;
6705             my $transferred = int($size * $percent / 100);
6706             my $remaining = $size - $transferred;
6707
6708             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6709         }
6710
6711     };
6712
6713     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6714     my $err = $@;
6715     die "copy failed: $err" if $err;
6716 }
6717
6718 sub qemu_img_format {
6719     my ($scfg, $volname) = @_;
6720
6721     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6722         return $1;
6723     } else {
6724         return "raw";
6725     }
6726 }
6727
6728 sub qemu_drive_mirror {
6729     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6730
6731     $jobs = {} if !$jobs;
6732
6733     my $qemu_target;
6734     my $format;
6735     $jobs->{"drive-$drive"} = {};
6736
6737     if ($dst_volid =~ /^nbd:/) {
6738         $qemu_target = $dst_volid;
6739         $format = "nbd";
6740     } else {
6741         my $storecfg = PVE::Storage::config();
6742         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6743
6744         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6745
6746         $format = qemu_img_format($dst_scfg, $dst_volname);
6747
6748         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6749
6750         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6751     }
6752
6753     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6754     $opts->{format} = $format if $format;
6755
6756     if (defined($src_bitmap)) {
6757         $opts->{sync} = 'incremental';
6758         $opts->{bitmap} = $src_bitmap;
6759         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6760     }
6761
6762     if (defined($bwlimit)) {
6763         $opts->{speed} = $bwlimit * 1024;
6764         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6765     } else {
6766         print "drive mirror is starting for drive-$drive\n";
6767     }
6768
6769     # if a job already runs for this device we get an error, catch it for cleanup
6770     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6771     if (my $err = $@) {
6772         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6773         warn "$@\n" if $@;
6774         die "mirroring error: $err\n";
6775     }
6776
6777     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6778 }
6779
6780 # $completion can be either
6781 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6782 # 'cancel': wait until all jobs are ready, block-job-cancel them
6783 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6784 sub qemu_drive_mirror_monitor {
6785     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6786
6787     $completion //= 'complete';
6788
6789     eval {
6790         my $err_complete = 0;
6791
6792         while (1) {
6793             die "storage migration timed out\n" if $err_complete > 300;
6794
6795             my $stats = mon_cmd($vmid, "query-block-jobs");
6796
6797             my $running_mirror_jobs = {};
6798             foreach my $stat (@$stats) {
6799                 next if $stat->{type} ne 'mirror';
6800                 $running_mirror_jobs->{$stat->{device}} = $stat;
6801             }
6802
6803             my $readycounter = 0;
6804
6805             foreach my $job (keys %$jobs) {
6806
6807                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6808                     print "$job : finished\n";
6809                     delete $jobs->{$job};
6810                     next;
6811                 }
6812
6813                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6814
6815                 my $busy = $running_mirror_jobs->{$job}->{busy};
6816                 my $ready = $running_mirror_jobs->{$job}->{ready};
6817                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6818                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6819                     my $remaining = $total - $transferred;
6820                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6821
6822                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6823                 }
6824
6825                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6826             }
6827
6828             last if scalar(keys %$jobs) == 0;
6829
6830             if ($readycounter == scalar(keys %$jobs)) {
6831                 print "all mirroring jobs are ready \n";
6832                 last if $completion eq 'skip'; #do the complete later
6833
6834                 if ($vmiddst && $vmiddst != $vmid) {
6835                     my $agent_running = $qga && qga_check_running($vmid);
6836                     if ($agent_running) {
6837                         print "freeze filesystem\n";
6838                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6839                     } else {
6840                         print "suspend vm\n";
6841                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6842                     }
6843
6844                     # if we clone a disk for a new target vm, we don't switch the disk
6845                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6846
6847                     if ($agent_running) {
6848                         print "unfreeze filesystem\n";
6849                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6850                     } else {
6851                         print "resume vm\n";
6852                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6853                     }
6854
6855                     last;
6856                 } else {
6857
6858                     foreach my $job (keys %$jobs) {
6859                         # try to switch the disk if source and destination are on the same guest
6860                         print "$job: Completing block job...\n";
6861
6862                         my $op;
6863                         if ($completion eq 'complete') {
6864                             $op = 'block-job-complete';
6865                         } elsif ($completion eq 'cancel') {
6866                             $op = 'block-job-cancel';
6867                         } else {
6868                             die "invalid completion value: $completion\n";
6869                         }
6870                         eval { mon_cmd($vmid, $op, device => $job) };
6871                         if ($@ =~ m/cannot be completed/) {
6872                             print "$job: Block job cannot be completed, try again.\n";
6873                             $err_complete++;
6874                         }else {
6875                             print "$job: Completed successfully.\n";
6876                             $jobs->{$job}->{complete} = 1;
6877                         }
6878                     }
6879                 }
6880             }
6881             sleep 1;
6882         }
6883     };
6884     my $err = $@;
6885
6886     if ($err) {
6887         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6888         die "mirroring error: $err";
6889     }
6890
6891 }
6892
6893 sub qemu_blockjobs_cancel {
6894     my ($vmid, $jobs) = @_;
6895
6896     foreach my $job (keys %$jobs) {
6897         print "$job: Cancelling block job\n";
6898         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6899         $jobs->{$job}->{cancel} = 1;
6900     }
6901
6902     while (1) {
6903         my $stats = mon_cmd($vmid, "query-block-jobs");
6904
6905         my $running_jobs = {};
6906         foreach my $stat (@$stats) {
6907             $running_jobs->{$stat->{device}} = $stat;
6908         }
6909
6910         foreach my $job (keys %$jobs) {
6911
6912             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6913                 print "$job: Done.\n";
6914                 delete $jobs->{$job};
6915             }
6916         }
6917
6918         last if scalar(keys %$jobs) == 0;
6919
6920         sleep 1;
6921     }
6922 }
6923
6924 sub clone_disk {
6925     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6926         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6927
6928     my $newvolid;
6929
6930     if (!$full) {
6931         print "create linked clone of drive $drivename ($drive->{file})\n";
6932         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6933         push @$newvollist, $newvolid;
6934     } else {
6935
6936         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6937         $storeid = $storage if $storage;
6938
6939         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6940         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6941
6942         print "create full clone of drive $drivename ($drive->{file})\n";
6943         my $name = undef;
6944         if (drive_is_cloudinit($drive)) {
6945             $name = "vm-$newvmid-cloudinit";
6946             $name .= ".$dst_format" if $dst_format ne 'raw';
6947             $snapname = undef;
6948             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6949         } elsif ($drivename eq 'efidisk0') {
6950             $size = get_efivars_size($conf);
6951         }
6952         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6953         push @$newvollist, $newvolid;
6954
6955         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6956
6957         if (drive_is_cloudinit($drive)) {
6958             goto no_data_clone;
6959         }
6960
6961         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6962         if (!$running || $snapname) {
6963             # TODO: handle bwlimits
6964             if ($drivename eq 'efidisk0') {
6965                 # the relevant data on the efidisk may be smaller than the source
6966                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6967                 # that is given by the OVMF_VARS.fd
6968                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6969                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6970                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6971             } else {
6972                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6973             }
6974         } else {
6975
6976             my $kvmver = get_running_qemu_version ($vmid);
6977             if (!min_version($kvmver, 2, 7)) {
6978                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6979                     if $drive->{iothread};
6980             }
6981
6982             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6983         }
6984     }
6985
6986 no_data_clone:
6987     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6988
6989     my $disk = $drive;
6990     $disk->{format} = undef;
6991     $disk->{file} = $newvolid;
6992     $disk->{size} = $size;
6993
6994     return $disk;
6995 }
6996
6997 sub get_running_qemu_version {
6998     my ($vmid) = @_;
6999     my $res = mon_cmd($vmid, "query-version");
7000     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7001 }
7002
7003 sub qemu_use_old_bios_files {
7004     my ($machine_type) = @_;
7005
7006     return if !$machine_type;
7007
7008     my $use_old_bios_files = undef;
7009
7010     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7011         $machine_type = $1;
7012         $use_old_bios_files = 1;
7013     } else {
7014         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7015         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7016         # load new efi bios files on migration. So this hack is required to allow
7017         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7018         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7019         $use_old_bios_files = !min_version($version, 2, 4);
7020     }
7021
7022     return ($use_old_bios_files, $machine_type);
7023 }
7024
7025 sub get_efivars_size {
7026     my ($conf) = @_;
7027     my $arch = get_vm_arch($conf);
7028     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7029     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7030     return -s $ovmf_vars;
7031 }
7032
7033 sub update_efidisk_size {
7034     my ($conf) = @_;
7035
7036     return if !defined($conf->{efidisk0});
7037
7038     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7039     $disk->{size} = get_efivars_size($conf);
7040     $conf->{efidisk0} = print_drive($disk);
7041
7042     return;
7043 }
7044
7045 sub create_efidisk($$$$$) {
7046     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7047
7048     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7049     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7050
7051     my $vars_size_b = -s $ovmf_vars;
7052     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7053     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7054     PVE::Storage::activate_volumes($storecfg, [$volid]);
7055
7056     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7057     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7058
7059     return ($volid, $size/1024);
7060 }
7061
7062 sub vm_iothreads_list {
7063     my ($vmid) = @_;
7064
7065     my $res = mon_cmd($vmid, 'query-iothreads');
7066
7067     my $iothreads = {};
7068     foreach my $iothread (@$res) {
7069         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7070     }
7071
7072     return $iothreads;
7073 }
7074
7075 sub scsihw_infos {
7076     my ($conf, $drive) = @_;
7077
7078     my $maxdev = 0;
7079
7080     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7081         $maxdev = 7;
7082     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7083         $maxdev = 1;
7084     } else {
7085         $maxdev = 256;
7086     }
7087
7088     my $controller = int($drive->{index} / $maxdev);
7089     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7090
7091     return ($maxdev, $controller, $controller_prefix);
7092 }
7093
7094 sub windows_version {
7095     my ($ostype) = @_;
7096
7097     return 0 if !$ostype;
7098
7099     my $winversion = 0;
7100
7101     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7102         $winversion = 5;
7103     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7104         $winversion = 6;
7105     } elsif ($ostype =~ m/^win(\d+)$/) {
7106         $winversion = $1;
7107     }
7108
7109     return $winversion;
7110 }
7111
7112 sub resolve_dst_disk_format {
7113         my ($storecfg, $storeid, $src_volname, $format) = @_;
7114         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7115
7116         if (!$format) {
7117             # if no target format is specified, use the source disk format as hint
7118             if ($src_volname) {
7119                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7120                 $format = qemu_img_format($scfg, $src_volname);
7121             } else {
7122                 return $defFormat;
7123             }
7124         }
7125
7126         # test if requested format is supported - else use default
7127         my $supported = grep { $_ eq $format } @$validFormats;
7128         $format = $defFormat if !$supported;
7129         return $format;
7130 }
7131
7132 # NOTE: if this logic changes, please update docs & possibly gui logic
7133 sub find_vmstate_storage {
7134     my ($conf, $storecfg) = @_;
7135
7136     # first, return storage from conf if set
7137     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7138
7139     my ($target, $shared, $local);
7140
7141     foreach_storage_used_by_vm($conf, sub {
7142         my ($sid) = @_;
7143         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7144         my $dst = $scfg->{shared} ? \$shared : \$local;
7145         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7146     });
7147
7148     # second, use shared storage where VM has at least one disk
7149     # third, use local storage where VM has at least one disk
7150     # fall back to local storage
7151     $target = $shared // $local // 'local';
7152
7153     return $target;
7154 }
7155
7156 sub generate_uuid {
7157     my ($uuid, $uuid_str);
7158     UUID::generate($uuid);
7159     UUID::unparse($uuid, $uuid_str);
7160     return $uuid_str;
7161 }
7162
7163 sub generate_smbios1_uuid {
7164     return "uuid=".generate_uuid();
7165 }
7166
7167 sub nbd_stop {
7168     my ($vmid) = @_;
7169
7170     mon_cmd($vmid, 'nbd-server-stop');
7171 }
7172
7173 sub create_reboot_request {
7174     my ($vmid) = @_;
7175     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7176         or die "failed to create reboot trigger file: $!\n";
7177     close($fh);
7178 }
7179
7180 sub clear_reboot_request {
7181     my ($vmid) = @_;
7182     my $path = "/run/qemu-server/$vmid.reboot";
7183     my $res = 0;
7184
7185     $res = unlink($path);
7186     die "could not remove reboot request for $vmid: $!"
7187         if !$res && $! != POSIX::ENOENT;
7188
7189     return $res;
7190 }
7191
7192 # bash completion helper
7193
7194 sub complete_backup_archives {
7195     my ($cmdname, $pname, $cvalue) = @_;
7196
7197     my $cfg = PVE::Storage::config();
7198
7199     my $storeid;
7200
7201     if ($cvalue =~ m/^([^:]+):/) {
7202         $storeid = $1;
7203     }
7204
7205     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7206
7207     my $res = [];
7208     foreach my $id (keys %$data) {
7209         foreach my $item (@{$data->{$id}}) {
7210             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7211             push @$res, $item->{volid} if defined($item->{volid});
7212         }
7213     }
7214
7215     return $res;
7216 }
7217
7218 my $complete_vmid_full = sub {
7219     my ($running) = @_;
7220
7221     my $idlist = vmstatus();
7222
7223     my $res = [];
7224
7225     foreach my $id (keys %$idlist) {
7226         my $d = $idlist->{$id};
7227         if (defined($running)) {
7228             next if $d->{template};
7229             next if $running && $d->{status} ne 'running';
7230             next if !$running && $d->{status} eq 'running';
7231         }
7232         push @$res, $id;
7233
7234     }
7235     return $res;
7236 };
7237
7238 sub complete_vmid {
7239     return &$complete_vmid_full();
7240 }
7241
7242 sub complete_vmid_stopped {
7243     return &$complete_vmid_full(0);
7244 }
7245
7246 sub complete_vmid_running {
7247     return &$complete_vmid_full(1);
7248 }
7249
7250 sub complete_storage {
7251
7252     my $cfg = PVE::Storage::config();
7253     my $ids = $cfg->{ids};
7254
7255     my $res = [];
7256     foreach my $sid (keys %$ids) {
7257         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7258         next if !$ids->{$sid}->{content}->{images};
7259         push @$res, $sid;
7260     }
7261
7262     return $res;
7263 }
7264
7265 sub complete_migration_storage {
7266     my ($cmd, $param, $current_value, $all_args) = @_;
7267
7268     my $targetnode = @$all_args[1];
7269
7270     my $cfg = PVE::Storage::config();
7271     my $ids = $cfg->{ids};
7272
7273     my $res = [];
7274     foreach my $sid (keys %$ids) {
7275         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7276         next if !$ids->{$sid}->{content}->{images};
7277         push @$res, $sid;
7278     }
7279
7280     return $res;
7281 }
7282
7283 1;