PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename_cache;
 111 sub nodename {
 112     $nodename_cache //= PVE::INotify::nodename();
 113     return $nodename_cache;
 114 }
 115
 116 my $watchdog_fmt = {
 117     model => {
 118         default_key => 1,
 119         type => 'string',
 120         enum => [qw(i6300esb ib700)],
 121         description => "Watchdog type to emulate.",
 122         default => 'i6300esb',
 123         optional => 1,
 124     },
 125     action => {
 126         type => 'string',
 127         enum => [qw(reset shutdown poweroff pause debug none)],
 128         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 129         optional => 1,
 130     },
 131 };
 132 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 133
 134 my $agent_fmt = {
 135     enabled => {
 136         description => "Enable/disable Qemu GuestAgent.",
 137         type => 'boolean',
 138         default => 0,
 139         default_key => 1,
 140     },
 141     fstrim_cloned_disks => {
 142         description => "Run fstrim after cloning/moving a disk.",
 143         type => 'boolean',
 144         optional => 1,
 145         default => 0
 146     },
 147     type => {
 148         description => "Select the agent type",
 149         type => 'string',
 150         default => 'virtio',
 151         optional => 1,
 152         enum => [qw(virtio isa)],
 153     },
 154 };
 155
 156 my $vga_fmt = {
 157     type => {
 158         description => "Select the VGA type.",
 159         type => 'string',
 160         default => 'std',
 161         optional => 1,
 162         default_key => 1,
 163         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 164     },
 165     memory => {
 166         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 167         type => 'integer',
 168         optional => 1,
 169         minimum => 4,
 170         maximum => 512,
 171     },
 172 };
 173
 174 my $ivshmem_fmt = {
 175     size => {
 176         type => 'integer',
 177         minimum => 1,
 178         description => "The size of the file in MB.",
 179     },
 180     name => {
 181         type => 'string',
 182         pattern => '[a-zA-Z0-9\-]+',
 183         optional => 1,
 184         format_description => 'string',
 185         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 186     },
 187 };
 188
 189 my $audio_fmt = {
 190     device => {
 191         type => 'string',
 192         enum => [qw(ich9-intel-hda intel-hda AC97)],
 193         description =>  "Configure an audio device."
 194     },
 195     driver =>  {
 196         type => 'string',
 197         enum => ['spice'],
 198         default => 'spice',
 199         optional => 1,
 200         description => "Driver backend for the audio device."
 201     },
 202 };
 203
 204 my $spice_enhancements_fmt = {
 205     foldersharing => {
 206         type => 'boolean',
 207         optional => 1,
 208         default => '0',
 209         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 210     },
 211     videostreaming =>  {
 212         type => 'string',
 213         enum => ['off', 'all', 'filter'],
 214         default => 'off',
 215         optional => 1,
 216         description => "Enable video streaming. Uses compression for detected video streams."
 217     },
 218 };
 219
 220 my $rng_fmt = {
 221     source => {
 222         type => 'string',
 223         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 224         default_key => 1,
 225         description => "The file on the host to gather entropy from. In most"
 226                      . " cases /dev/urandom should be preferred over /dev/random"
 227                      . " to avoid entropy-starvation issues on the host. Using"
 228                      . " urandom does *not* decrease security in any meaningful"
 229                      . " way, as it's still seeded from real entropy, and the"
 230                      . " bytes provided will most likely be mixed with real"
 231                      . " entropy on the guest as well. /dev/hwrng can be used"
 232                      . " to pass through a hardware RNG from the host.",
 233     },
 234     max_bytes => {
 235         type => 'integer',
 236         description => "Maximum bytes of entropy injected into the guest every"
 237                      . " 'period' milliseconds. Prefer a lower value when using"
 238                      . " /dev/random as source. Use 0 to disable limiting"
 239                      . " (potentially dangerous!).",
 240         optional => 1,
 241
 242         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 243         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 244         # of overwhelming the host, provided we're reading from /dev/urandom
 245         default => 1024,
 246     },
 247     period => {
 248         type => 'integer',
 249         description => "Every 'period' milliseconds the entropy-injection quota"
 250                      . " is reset, allowing the guest to retrieve another"
 251                      . " 'max_bytes' of entropy.",
 252         optional => 1,
 253         default => 1000,
 254     },
 255 };
 256
 257 my $confdesc = {
 258     onboot => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Specifies whether a VM will be started during system bootup.",
 262         default => 0,
 263     },
 264     autostart => {
 265         optional => 1,
 266         type => 'boolean',
 267         description => "Automatic restart after crash (currently ignored).",
 268         default => 0,
 269     },
 270     hotplug => {
 271         optional => 1,
 272         type => 'string', format => 'pve-hotplug-features',
 273         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 274         default => 'network,disk,usb',
 275     },
 276     reboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 280         default => 1,
 281     },
 282     lock => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Lock/unlock the VM.",
 286         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 287     },
 288     cpulimit => {
 289         optional => 1,
 290         type => 'number',
 291         description => "Limit of CPU usage.",
 292         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 293         minimum => 0,
 294         maximum => 128,
 295         default => 0,
 296     },
 297     cpuunits => {
 298         optional => 1,
 299         type => 'integer',
 300         description => "CPU weight for a VM.",
 301         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 302         minimum => 2,
 303         maximum => 262144,
 304         default => 1024,
 305     },
 306     memory => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 310         minimum => 16,
 311         default => 512,
 312     },
 313     balloon => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 317         minimum => 0,
 318     },
 319     shares => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 323         minimum => 0,
 324         maximum => 50000,
 325         default => 1000,
 326     },
 327     keyboard => {
 328         optional => 1,
 329         type => 'string',
 330         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 331                        "It should not be necessary to set it.",
 332         enum => PVE::Tools::kvmkeymaplist(),
 333         default => undef,
 334     },
 335     name => {
 336         optional => 1,
 337         type => 'string', format => 'dns-name',
 338         description => "Set a name for the VM. Only used on the configuration web interface.",
 339     },
 340     scsihw => {
 341         optional => 1,
 342         type => 'string',
 343         description => "SCSI controller model",
 344         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 345         default => 'lsi',
 346     },
 347     description => {
 348         optional => 1,
 349         type => 'string',
 350         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 351     },
 352     ostype => {
 353         optional => 1,
 354         type => 'string',
 355         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 356         description => "Specify guest operating system.",
 357         verbose_description => <<EODESC,
 358 Specify guest operating system. This is used to enable special
 359 optimization/features for specific operating systems:
 360
 361 [horizontal]
 362 other;; unspecified OS
 363 wxp;; Microsoft Windows XP
 364 w2k;; Microsoft Windows 2000
 365 w2k3;; Microsoft Windows 2003
 366 w2k8;; Microsoft Windows 2008
 367 wvista;; Microsoft Windows Vista
 368 win7;; Microsoft Windows 7
 369 win8;; Microsoft Windows 8/2012/2012r2
 370 win10;; Microsoft Windows 10/2016
 371 l24;; Linux 2.4 Kernel
 372 l26;; Linux 2.6 - 5.X Kernel
 373 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 374 EODESC
 375     },
 376     boot => {
 377         optional => 1,
 378         type => 'string',
 379         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 380         pattern => '[acdn]{1,4}',
 381         default => 'cdn',
 382     },
 383     bootdisk => {
 384         optional => 1,
 385         type => 'string', format => 'pve-qm-bootdisk',
 386         description => "Enable booting from specified disk.",
 387         pattern => '(ide|sata|scsi|virtio)\d+',
 388     },
 389     smp => {
 390         optional => 1,
 391         type => 'integer',
 392         description => "The number of CPUs. Please use option -sockets instead.",
 393         minimum => 1,
 394         default => 1,
 395     },
 396     sockets => {
 397         optional => 1,
 398         type => 'integer',
 399         description => "The number of CPU sockets.",
 400         minimum => 1,
 401         default => 1,
 402     },
 403     cores => {
 404         optional => 1,
 405         type => 'integer',
 406         description => "The number of cores per socket.",
 407         minimum => 1,
 408         default => 1,
 409     },
 410     numa => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable NUMA.",
 414         default => 0,
 415     },
 416     hugepages => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Enable/disable hugepages memory.",
 420         enum => [qw(any 2 1024)],
 421     },
 422     vcpus => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "Number of hotplugged vcpus.",
 426         minimum => 1,
 427         default => 0,
 428     },
 429     acpi => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable ACPI.",
 433         default => 1,
 434     },
 435     agent => {
 436         optional => 1,
 437         description => "Enable/disable Qemu GuestAgent and its properties.",
 438         type => 'string',
 439         format => $agent_fmt,
 440     },
 441     kvm => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable KVM hardware virtualization.",
 445         default => 1,
 446     },
 447     tdf => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Enable/disable time drift fix.",
 451         default => 0,
 452     },
 453     localtime => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 457     },
 458     freeze => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 462     },
 463     vga => {
 464         optional => 1,
 465         type => 'string', format => $vga_fmt,
 466         description => "Configure the VGA hardware.",
 467         verbose_description => "Configure the VGA Hardware. If you want to use ".
 468             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 469             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 470             "is 'std' for all OS types besides some Windows versions (XP and " .
 471             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 472             "display server. For win* OS you can select how many independent " .
 473             "displays you want, Linux guests can add displays them self.\n".
 474             "You can also run without any graphic card, using a serial device as terminal.",
 475     },
 476     watchdog => {
 477         optional => 1,
 478         type => 'string', format => 'pve-qm-watchdog',
 479         description => "Create a virtual hardware watchdog device.",
 480         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 481             " (by a guest action), the watchdog must be periodically polled " .
 482             "by an agent inside the guest or else the watchdog will reset " .
 483             "the guest (or execute the respective action specified)",
 484     },
 485     startdate => {
 486         optional => 1,
 487         type => 'string',
 488         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 489         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 490         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 491         default => 'now',
 492     },
 493     startup =>  get_standard_option('pve-startup-order'),
 494     template => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable Template.",
 498         default => 0,
 499     },
 500     args => {
 501         optional => 1,
 502         type => 'string',
 503         description => "Arbitrary arguments passed to kvm.",
 504         verbose_description => <<EODESCR,
 505 Arbitrary arguments passed to kvm, for example:
 506
 507 args: -no-reboot -no-hpet
 508
 509 NOTE: this option is for experts only.
 510 EODESCR
 511     },
 512     tablet => {
 513         optional => 1,
 514         type => 'boolean',
 515         default => 1,
 516         description => "Enable/disable the USB tablet device.",
 517         verbose_description => "Enable/disable the USB tablet device. This device is " .
 518             "usually needed to allow absolute mouse positioning with VNC. " .
 519             "Else the mouse runs out of sync with normal VNC clients. " .
 520             "If you're running lots of console-only guests on one host, " .
 521             "you may consider disabling this to save some context switches. " .
 522             "This is turned off by default if you use spice (-vga=qxl).",
 523     },
 524     migrate_speed => {
 525         optional => 1,
 526         type => 'integer',
 527         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 528         minimum => 0,
 529         default => 0,
 530     },
 531     migrate_downtime => {
 532         optional => 1,
 533         type => 'number',
 534         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 535         minimum => 0,
 536         default => 0.1,
 537     },
 538     cdrom => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-ide',
 541         typetext => '<volume>',
 542         description => "This is an alias for option -ide2",
 543     },
 544     cpu => {
 545         optional => 1,
 546         description => "Emulated CPU type.",
 547         type => 'string',
 548         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 549     },
 550     parent => get_standard_option('pve-snapshot-name', {
 551         optional => 1,
 552         description => "Parent snapshot name. This is used internally, and should not be modified.",
 553     }),
 554     snaptime => {
 555         optional => 1,
 556         description => "Timestamp for snapshots.",
 557         type => 'integer',
 558         minimum => 0,
 559     },
 560     vmstate => {
 561         optional => 1,
 562         type => 'string', format => 'pve-volume-id',
 563         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 564     },
 565     vmstatestorage => get_standard_option('pve-storage-id', {
 566         description => "Default storage for VM state volumes/files.",
 567         optional => 1,
 568     }),
 569     runningmachine => get_standard_option('pve-qemu-machine', {
 570         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 571     }),
 572     machine => get_standard_option('pve-qemu-machine'),
 573     arch => {
 574         description => "Virtual processor architecture. Defaults to the host.",
 575         optional => 1,
 576         type => 'string',
 577         enum => [qw(x86_64 aarch64)],
 578     },
 579     smbios1 => {
 580         description => "Specify SMBIOS type 1 fields.",
 581         type => 'string', format => 'pve-qm-smbios1',
 582         maxLength => 512,
 583         optional => 1,
 584     },
 585     protection => {
 586         optional => 1,
 587         type => 'boolean',
 588         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 589         default => 0,
 590     },
 591     bios => {
 592         optional => 1,
 593         type => 'string',
 594         enum => [ qw(seabios ovmf) ],
 595         description => "Select BIOS implementation.",
 596         default => 'seabios',
 597     },
 598     vmgenid => {
 599         type => 'string',
 600         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 601         format_description => 'UUID',
 602         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 603         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 604             " 128-bit integer value identifier to the guest OS. This allows to".
 605             " notify the guest operating system when the virtual machine is".
 606             " executed with a different configuration (e.g. snapshot execution".
 607             " or creation from a template). The guest operating system notices".
 608             " the change, and is then able to react as appropriate by marking".
 609             " its copies of distributed databases as dirty, re-initializing its".
 610             " random number generator, etc.\n".
 611             "Note that auto-creation only works when done throug API/CLI create".
 612             " or update methods, but not when manually editing the config file.",
 613         default => "1 (autogenerated)",
 614         optional => 1,
 615     },
 616     hookscript => {
 617         type => 'string',
 618         format => 'pve-volume-id',
 619         optional => 1,
 620         description => "Script that will be executed during various steps in the vms lifetime.",
 621     },
 622     ivshmem => {
 623         type => 'string',
 624         format => $ivshmem_fmt,
 625         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 626         optional => 1,
 627     },
 628     audio0 => {
 629         type => 'string',
 630         format => $audio_fmt,
 631         description => "Configure a audio device, useful in combination with QXL/Spice.",
 632         optional => 1
 633     },
 634     spice_enhancements => {
 635         type => 'string',
 636         format => $spice_enhancements_fmt,
 637         description => "Configure additional enhancements for SPICE.",
 638         optional => 1
 639     },
 640     tags => {
 641         type => 'string', format => 'pve-tag-list',
 642         description => 'Tags of the VM. This is only meta information.',
 643         optional => 1,
 644     },
 645     rng0 => {
 646         type => 'string',
 647         format => $rng_fmt,
 648         description => "Configure a VirtIO-based Random Number Generator.",
 649         optional => 1,
 650     },
 651 };
 652
 653 my $cicustom_fmt = {
 654     meta => {
 655         type => 'string',
 656         optional => 1,
 657         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 658         format => 'pve-volume-id',
 659         format_description => 'volume',
 660     },
 661     network => {
 662         type => 'string',
 663         optional => 1,
 664         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 665         format => 'pve-volume-id',
 666         format_description => 'volume',
 667     },
 668     user => {
 669         type => 'string',
 670         optional => 1,
 671         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 672         format => 'pve-volume-id',
 673         format_description => 'volume',
 674     },
 675 };
 676 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 677
 678 my $confdesc_cloudinit = {
 679     citype => {
 680         optional => 1,
 681         type => 'string',
 682         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 683         enum => ['configdrive2', 'nocloud'],
 684     },
 685     ciuser => {
 686         optional => 1,
 687         type => 'string',
 688         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 689     },
 690     cipassword => {
 691         optional => 1,
 692         type => 'string',
 693         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 694     },
 695     cicustom => {
 696         optional => 1,
 697         type => 'string',
 698         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 699         format => 'pve-qm-cicustom',
 700     },
 701     searchdomain => {
 702         optional => 1,
 703         type => 'string',
 704         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 705     },
 706     nameserver => {
 707         optional => 1,
 708         type => 'string', format => 'address-list',
 709         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 710     },
 711     sshkeys => {
 712         optional => 1,
 713         type => 'string',
 714         format => 'urlencoded',
 715         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 716     },
 717 };
 718
 719 # what about other qemu settings ?
 720 #cpu => 'string',
 721 #machine => 'string',
 722 #fda => 'file',
 723 #fdb => 'file',
 724 #mtdblock => 'file',
 725 #sd => 'file',
 726 #pflash => 'file',
 727 #snapshot => 'bool',
 728 #bootp => 'file',
 729 ##tftp => 'dir',
 730 ##smb => 'dir',
 731 #kernel => 'file',
 732 #append => 'string',
 733 #initrd => 'file',
 734 ##soundhw => 'string',
 735
 736 while (my ($k, $v) = each %$confdesc) {
 737     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 738 }
 739
 740 my $MAX_USB_DEVICES = 5;
 741 my $MAX_NETS = 32;
 742 my $MAX_HOSTPCI_DEVICES = 16;
 743 my $MAX_SERIAL_PORTS = 4;
 744 my $MAX_PARALLEL_PORTS = 3;
 745 my $MAX_NUMA = 8;
 746
 747 my $numa_fmt = {
 748     cpus => {
 749         type => "string",
 750         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 751         description => "CPUs accessing this NUMA node.",
 752         format_description => "id[-id];...",
 753     },
 754     memory => {
 755         type => "number",
 756         description => "Amount of memory this NUMA node provides.",
 757         optional => 1,
 758     },
 759     hostnodes => {
 760         type => "string",
 761         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 762         description => "Host NUMA nodes to use.",
 763         format_description => "id[-id];...",
 764         optional => 1,
 765     },
 766     policy => {
 767         type => 'string',
 768         enum => [qw(preferred bind interleave)],
 769         description => "NUMA allocation policy.",
 770         optional => 1,
 771     },
 772 };
 773 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 774 my $numadesc = {
 775     optional => 1,
 776     type => 'string', format => $numa_fmt,
 777     description => "NUMA topology.",
 778 };
 779 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 780
 781 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 782     $confdesc->{"numa$i"} = $numadesc;
 783 }
 784
 785 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 786                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 787                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 788 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 789
 790 my $net_fmt_bridge_descr = <<__EOD__;
 791 Bridge to attach the network device to. The Proxmox VE standard bridge
 792 is called 'vmbr0'.
 793
 794 If you do not specify a bridge, we create a kvm user (NATed) network
 795 device, which provides DHCP and DNS services. The following addresses
 796 are used:
 797
 798  10.0.2.2   Gateway
 799  10.0.2.3   DNS Server
 800  10.0.2.4   SMB Server
 801
 802 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 803 __EOD__
 804
 805 my $net_fmt = {
 806     macaddr  => get_standard_option('mac-addr', {
 807         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 808     }),
 809     model => {
 810         type => 'string',
 811         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 812         enum => $nic_model_list,
 813         default_key => 1,
 814     },
 815     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 816     bridge => {
 817         type => 'string',
 818         description => $net_fmt_bridge_descr,
 819         format_description => 'bridge',
 820         optional => 1,
 821     },
 822     queues => {
 823         type => 'integer',
 824         minimum => 0, maximum => 16,
 825         description => 'Number of packet queues to be used on the device.',
 826         optional => 1,
 827     },
 828     rate => {
 829         type => 'number',
 830         minimum => 0,
 831         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 832         optional => 1,
 833     },
 834     tag => {
 835         type => 'integer',
 836         minimum => 1, maximum => 4094,
 837         description => 'VLAN tag to apply to packets on this interface.',
 838         optional => 1,
 839     },
 840     trunks => {
 841         type => 'string',
 842         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 843         description => 'VLAN trunks to pass through this interface.',
 844         format_description => 'vlanid[;vlanid...]',
 845         optional => 1,
 846     },
 847     firewall => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be protected by the firewall.',
 850         optional => 1,
 851     },
 852     link_down => {
 853         type => 'boolean',
 854         description => 'Whether this interface should be disconnected (like pulling the plug).',
 855         optional => 1,
 856     },
 857 };
 858
 859 my $netdesc = {
 860     optional => 1,
 861     type => 'string', format => $net_fmt,
 862     description => "Specify network devices.",
 863 };
 864
 865 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 866
 867 my $ipconfig_fmt = {
 868     ip => {
 869         type => 'string',
 870         format => 'pve-ipv4-config',
 871         format_description => 'IPv4Format/CIDR',
 872         description => 'IPv4 address in CIDR format.',
 873         optional => 1,
 874         default => 'dhcp',
 875     },
 876     gw => {
 877         type => 'string',
 878         format => 'ipv4',
 879         format_description => 'GatewayIPv4',
 880         description => 'Default gateway for IPv4 traffic.',
 881         optional => 1,
 882         requires => 'ip',
 883     },
 884     ip6 => {
 885         type => 'string',
 886         format => 'pve-ipv6-config',
 887         format_description => 'IPv6Format/CIDR',
 888         description => 'IPv6 address in CIDR format.',
 889         optional => 1,
 890         default => 'dhcp',
 891     },
 892     gw6 => {
 893         type => 'string',
 894         format => 'ipv6',
 895         format_description => 'GatewayIPv6',
 896         description => 'Default gateway for IPv6 traffic.',
 897         optional => 1,
 898         requires => 'ip6',
 899     },
 900 };
 901 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 902 my $ipconfigdesc = {
 903     optional => 1,
 904     type => 'string', format => 'pve-qm-ipconfig',
 905     description => <<'EODESCR',
 906 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 907
 908 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 909
 910 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 911 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 912
 913 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 914 EODESCR
 915 };
 916 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 917
 918 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 919     $confdesc->{"net$i"} = $netdesc;
 920     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 921 }
 922
 923 foreach my $key (keys %$confdesc_cloudinit) {
 924     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 925 }
 926
 927 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 928 sub verify_volume_id_or_qm_path {
 929     my ($volid, $noerr) = @_;
 930
 931     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 932         return $volid;
 933     }
 934
 935     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 936     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 937     if ($@) {
 938         return undef if $noerr;
 939         die $@;
 940     }
 941     return $volid;
 942 }
 943
 944 my $usb_fmt = {
 945     host => {
 946         default_key => 1,
 947         type => 'string', format => 'pve-qm-usb-device',
 948         format_description => 'HOSTUSBDEVICE|spice',
 949         description => <<EODESCR,
 950 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 951
 952  'bus-port(.port)*' (decimal numbers) or
 953  'vendor_id:product_id' (hexadeciaml numbers) or
 954  'spice'
 955
 956 You can use the 'lsusb -t' command to list existing usb devices.
 957
 958 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 959
 960 The value 'spice' can be used to add a usb redirection devices for spice.
 961 EODESCR
 962     },
 963     usb3 => {
 964         optional => 1,
 965         type => 'boolean',
 966         description => "Specifies whether if given host option is a USB3 device or port.",
 967         default => 0,
 968     },
 969 };
 970
 971 my $usbdesc = {
 972     optional => 1,
 973     type => 'string', format => $usb_fmt,
 974     description => "Configure an USB device (n is 0 to 4).",
 975 };
 976 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 977
 978 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 979 my $hostpci_fmt = {
 980     host => {
 981         default_key => 1,
 982         type => 'string',
 983         pattern => qr/$PCIRE(;$PCIRE)*/,
 984         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 985         description => <<EODESCR,
 986 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 987 of PCI virtual functions of the host. HOSTPCIID syntax is:
 988
 989 'bus:dev.func' (hexadecimal numbers)
 990
 991 You can us the 'lspci' command to list existing PCI devices.
 992 EODESCR
 993     },
 994     rombar => {
 995         type => 'boolean',
 996         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 997         optional => 1,
 998         default => 1,
 999     },
1000     romfile => {
1001         type => 'string',
1002         pattern => '[^,;]+',
1003         format_description => 'string',
1004         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1005         optional => 1,
1006     },
1007     pcie => {
1008         type => 'boolean',
1009         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1010         optional => 1,
1011         default => 0,
1012     },
1013     'x-vga' => {
1014         type => 'boolean',
1015         description =>  "Enable vfio-vga device support.",
1016         optional => 1,
1017         default => 0,
1018     },
1019     'mdev' => {
1020         type => 'string',
1021         format_description => 'string',
1022         pattern => '[^/\.:]+',
1023         optional => 1,
1024         description => <<EODESCR
1025 The type of mediated device to use.
1026 An instance of this type will be created on startup of the VM and
1027 will be cleaned up when the VM stops.
1028 EODESCR
1029     }
1030 };
1031 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1032
1033 my $hostpcidesc = {
1034         optional => 1,
1035         type => 'string', format => 'pve-qm-hostpci',
1036         description => "Map host PCI devices into guest.",
1037         verbose_description =>  <<EODESCR,
1038 Map host PCI devices into guest.
1039
1040 NOTE: This option allows direct access to host hardware. So it is no longer
1041 possible to migrate such machines - use with special care.
1042
1043 CAUTION: Experimental! User reported problems with this option.
1044 EODESCR
1045 };
1046 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1047
1048 my $serialdesc = {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '(/dev/.+|socket)',
1052         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1053         verbose_description =>  <<EODESCR,
1054 Create a serial device inside the VM (n is 0 to 3), and pass through a
1055 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1056 host side (use 'qm terminal' to open a terminal connection).
1057
1058 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1059
1060 CAUTION: Experimental! User reported problems with this option.
1061 EODESCR
1062 };
1063
1064 my $paralleldesc= {
1065         optional => 1,
1066         type => 'string',
1067         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1068         description =>  "Map host parallel devices (n is 0 to 2).",
1069         verbose_description =>  <<EODESCR,
1070 Map host parallel devices (n is 0 to 2).
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1073
1074 CAUTION: Experimental! User reported problems with this option.
1075 EODESCR
1076 };
1077
1078 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1079     $confdesc->{"parallel$i"} = $paralleldesc;
1080 }
1081
1082 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1083     $confdesc->{"serial$i"} = $serialdesc;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1087     $confdesc->{"hostpci$i"} = $hostpcidesc;
1088 }
1089
1090 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1091     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1092 }
1093
1094 for (my $i = 0; $i < $PVE::QemuServer::Drive::MAX_UNUSED_DISKS; $i++) {
1095     $confdesc->{"unused$i"} = $PVE::QemuServer::Drive::unuseddesc;
1096 }
1097
1098 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1099     $confdesc->{"usb$i"} = $usbdesc;
1100 }
1101
1102 my $kvm_api_version = 0;
1103
1104 sub kvm_version {
1105     return $kvm_api_version if $kvm_api_version;
1106
1107     open my $fh, '<', '/dev/kvm'
1108         or return undef;
1109
1110     # 0xae00 => KVM_GET_API_VERSION
1111     $kvm_api_version = ioctl($fh, 0xae00, 0);
1112
1113     return $kvm_api_version;
1114 }
1115
1116 my $kvm_user_version = {};
1117 my $kvm_mtime = {};
1118
1119 sub kvm_user_version {
1120     my ($binary) = @_;
1121
1122     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1123     my $st = stat($binary);
1124
1125     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1126     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1127         $cachedmtime == $st->mtime;
1128
1129     $kvm_user_version->{$binary} = 'unknown';
1130     $kvm_mtime->{$binary} = $st->mtime;
1131
1132     my $code = sub {
1133         my $line = shift;
1134         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1135             $kvm_user_version->{$binary} = $2;
1136         }
1137     };
1138
1139     eval { run_command([$binary, '--version'], outfunc => $code); };
1140     warn $@ if $@;
1141
1142     return $kvm_user_version->{$binary};
1143
1144 }
1145
1146 sub kernel_has_vhost_net {
1147     return -c '/dev/vhost-net';
1148 }
1149
1150 sub option_exists {
1151     my $key = shift;
1152     return defined($confdesc->{$key});
1153 }
1154
1155 my $cdrom_path;
1156 sub get_cdrom_path {
1157
1158     return  $cdrom_path if $cdrom_path;
1159
1160     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1161     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1162     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1163 }
1164
1165 sub get_iso_path {
1166     my ($storecfg, $vmid, $cdrom) = @_;
1167
1168     if ($cdrom eq 'cdrom') {
1169         return get_cdrom_path();
1170     } elsif ($cdrom eq 'none') {
1171         return '';
1172     } elsif ($cdrom =~ m|^/|) {
1173         return $cdrom;
1174     } else {
1175         return PVE::Storage::path($storecfg, $cdrom);
1176     }
1177 }
1178
1179 # try to convert old style file names to volume IDs
1180 sub filename_to_volume_id {
1181     my ($vmid, $file, $media) = @_;
1182
1183      if (!($file eq 'none' || $file eq 'cdrom' ||
1184           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1185
1186         return undef if $file =~ m|/|;
1187
1188         if ($media && $media eq 'cdrom') {
1189             $file = "local:iso/$file";
1190         } else {
1191             $file = "local:$vmid/$file";
1192         }
1193     }
1194
1195     return $file;
1196 }
1197
1198 sub verify_media_type {
1199     my ($opt, $vtype, $media) = @_;
1200
1201     return if !$media;
1202
1203     my $etype;
1204     if ($media eq 'disk') {
1205         $etype = 'images';
1206     } elsif ($media eq 'cdrom') {
1207         $etype = 'iso';
1208     } else {
1209         die "internal error";
1210     }
1211
1212     return if ($vtype eq $etype);
1213
1214     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1215 }
1216
1217 sub cleanup_drive_path {
1218     my ($opt, $storecfg, $drive) = @_;
1219
1220     # try to convert filesystem paths to volume IDs
1221
1222     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1223         ($drive->{file} !~ m|^/dev/.+|) &&
1224         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1225         ($drive->{file} !~ m/^\d+$/)) {
1226         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1227         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1228         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1229         verify_media_type($opt, $vtype, $drive->{media});
1230         $drive->{file} = $volid;
1231     }
1232
1233     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1234 }
1235
1236 sub parse_hotplug_features {
1237     my ($data) = @_;
1238
1239     my $res = {};
1240
1241     return $res if $data eq '0';
1242
1243     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1244
1245     foreach my $feature (PVE::Tools::split_list($data)) {
1246         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1247             $res->{$1} = 1;
1248         } else {
1249             die "invalid hotplug feature '$feature'\n";
1250         }
1251     }
1252     return $res;
1253 }
1254
1255 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1256 sub pve_verify_hotplug_features {
1257     my ($value, $noerr) = @_;
1258
1259     return $value if parse_hotplug_features($value);
1260
1261     return undef if $noerr;
1262
1263     die "unable to parse hotplug option\n";
1264 }
1265
1266 sub scsi_inquiry {
1267     my($fh, $noerr) = @_;
1268
1269     my $SG_IO = 0x2285;
1270     my $SG_GET_VERSION_NUM = 0x2282;
1271
1272     my $versionbuf = "\x00" x 8;
1273     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1274     if (!$ret) {
1275         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1276         return undef;
1277     }
1278     my $version = unpack("I", $versionbuf);
1279     if ($version < 30000) {
1280         die "scsi generic interface too old\n"  if !$noerr;
1281         return undef;
1282     }
1283
1284     my $buf = "\x00" x 36;
1285     my $sensebuf = "\x00" x 8;
1286     my $cmd = pack("C x3 C x1", 0x12, 36);
1287
1288     # see /usr/include/scsi/sg.h
1289     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1290
1291     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1292                       length($sensebuf), 0, length($buf), $buf,
1293                       $cmd, $sensebuf, 6000);
1294
1295     $ret = ioctl($fh, $SG_IO, $packet);
1296     if (!$ret) {
1297         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1298         return undef;
1299     }
1300
1301     my @res = unpack($sg_io_hdr_t, $packet);
1302     if ($res[17] || $res[18]) {
1303         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1304         return undef;
1305     }
1306
1307     my $res = {};
1308     (my $byte0, my $byte1, $res->{vendor},
1309      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1310
1311     $res->{removable} = $byte1 & 128 ? 1 : 0;
1312     $res->{type} = $byte0 & 31;
1313
1314     return $res;
1315 }
1316
1317 sub path_is_scsi {
1318     my ($path) = @_;
1319
1320     my $fh = IO::File->new("+<$path") || return undef;
1321     my $res = scsi_inquiry($fh, 1);
1322     close($fh);
1323
1324     return $res;
1325 }
1326
1327 sub print_tabletdevice_full {
1328     my ($conf, $arch) = @_;
1329
1330     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1331
1332     # we use uhci for old VMs because tablet driver was buggy in older qemu
1333     my $usbbus;
1334     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1335         $usbbus = 'ehci';
1336     } else {
1337         $usbbus = 'uhci';
1338     }
1339
1340     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1341 }
1342
1343 sub print_keyboarddevice_full {
1344     my ($conf, $arch, $machine) = @_;
1345
1346     return undef if $arch ne 'aarch64';
1347
1348     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1349 }
1350
1351 sub print_drivedevice_full {
1352     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1353
1354     my $device = '';
1355     my $maxdev = 0;
1356
1357     if ($drive->{interface} eq 'virtio') {
1358         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1359         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1360         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1361     } elsif ($drive->{interface} eq 'scsi') {
1362
1363         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1364         my $unit = $drive->{index} % $maxdev;
1365         my $devicetype = 'hd';
1366         my $path = '';
1367         if (drive_is_cdrom($drive)) {
1368             $devicetype = 'cd';
1369         } else {
1370             if ($drive->{file} =~ m|^/|) {
1371                 $path = $drive->{file};
1372                 if (my $info = path_is_scsi($path)) {
1373                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1374                         $devicetype = 'block';
1375                     } elsif ($info->{type} == 1) { # tape
1376                         $devicetype = 'generic';
1377                     }
1378                 }
1379             } else {
1380                  $path = PVE::Storage::path($storecfg, $drive->{file});
1381             }
1382
1383             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1384             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1385             if ($path =~ m/^iscsi\:\/\// &&
1386                !min_version($version, 4, 1)) {
1387                 $devicetype = 'generic';
1388             }
1389         }
1390
1391         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1392             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1393         } else {
1394             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1395         }
1396
1397         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1398             $device .= ",rotation_rate=1";
1399         }
1400         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1401
1402     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1403         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1404         my $controller = int($drive->{index} / $maxdev);
1405         my $unit = $drive->{index} % $maxdev;
1406         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1407
1408         $device = "ide-$devicetype";
1409         if ($drive->{interface} eq 'ide') {
1410             $device .= ",bus=ide.$controller,unit=$unit";
1411         } else {
1412             $device .= ",bus=ahci$controller.$unit";
1413         }
1414         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1415
1416         if ($devicetype eq 'hd') {
1417             if (my $model = $drive->{model}) {
1418                 $model = URI::Escape::uri_unescape($model);
1419                 $device .= ",model=$model";
1420             }
1421             if ($drive->{ssd}) {
1422                 $device .= ",rotation_rate=1";
1423             }
1424         }
1425         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1426     } elsif ($drive->{interface} eq 'usb') {
1427         die "implement me";
1428         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1429     } else {
1430         die "unsupported interface type";
1431     }
1432
1433     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1434
1435     if (my $serial = $drive->{serial}) {
1436         $serial = URI::Escape::uri_unescape($serial);
1437         $device .= ",serial=$serial";
1438     }
1439
1440
1441     return $device;
1442 }
1443
1444 sub get_initiator_name {
1445     my $initiator;
1446
1447     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1448     while (defined(my $line = <$fh>)) {
1449         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1450         $initiator = $1;
1451         last;
1452     }
1453     $fh->close();
1454
1455     return $initiator;
1456 }
1457
1458 sub print_drive_commandline_full {
1459     my ($storecfg, $vmid, $drive) = @_;
1460
1461     my $path;
1462     my $volid = $drive->{file};
1463     my $format;
1464
1465     if (drive_is_cdrom($drive)) {
1466         $path = get_iso_path($storecfg, $vmid, $volid);
1467     } else {
1468         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1469         if ($storeid) {
1470             $path = PVE::Storage::path($storecfg, $volid);
1471             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1472             $format = qemu_img_format($scfg, $volname);
1473         } else {
1474             $path = $volid;
1475             $format = "raw";
1476         }
1477    }
1478
1479     my $opts = '';
1480     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1481     foreach my $o (@qemu_drive_options) {
1482         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1483     }
1484
1485     # snapshot only accepts on|off
1486     if (defined($drive->{snapshot})) {
1487         my $v = $drive->{snapshot} ? 'on' : 'off';
1488         $opts .= ",snapshot=$v";
1489     }
1490
1491     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1492         my ($dir, $qmpname) = @$type;
1493         if (my $v = $drive->{"mbps$dir"}) {
1494             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1495         }
1496         if (my $v = $drive->{"mbps${dir}_max"}) {
1497             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1498         }
1499         if (my $v = $drive->{"bps${dir}_max_length"}) {
1500             $opts .= ",throttling.bps$qmpname-max-length=$v";
1501         }
1502         if (my $v = $drive->{"iops${dir}"}) {
1503             $opts .= ",throttling.iops$qmpname=$v";
1504         }
1505         if (my $v = $drive->{"iops${dir}_max"}) {
1506             $opts .= ",throttling.iops$qmpname-max=$v";
1507         }
1508         if (my $v = $drive->{"iops${dir}_max_length"}) {
1509             $opts .= ",throttling.iops$qmpname-max-length=$v";
1510         }
1511     }
1512
1513     $opts .= ",format=$format" if $format && !$drive->{format};
1514
1515     my $cache_direct = 0;
1516
1517     if (my $cache = $drive->{cache}) {
1518         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1519     } elsif (!drive_is_cdrom($drive)) {
1520         $opts .= ",cache=none";
1521         $cache_direct = 1;
1522     }
1523
1524     # aio native works only with O_DIRECT
1525     if (!$drive->{aio}) {
1526         if($cache_direct) {
1527             $opts .= ",aio=native";
1528         } else {
1529             $opts .= ",aio=threads";
1530         }
1531     }
1532
1533     if (!drive_is_cdrom($drive)) {
1534         my $detectzeroes;
1535         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1536             $detectzeroes = 'off';
1537         } elsif ($drive->{discard}) {
1538             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1539         } else {
1540             # This used to be our default with discard not being specified:
1541             $detectzeroes = 'on';
1542         }
1543         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1544     }
1545
1546     my $pathinfo = $path ? "file=$path," : '';
1547
1548     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1549 }
1550
1551 sub print_netdevice_full {
1552     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1553
1554     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1555
1556     my $device = $net->{model};
1557     if ($net->{model} eq 'virtio') {
1558          $device = 'virtio-net-pci';
1559      };
1560
1561     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1562     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1563     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1564         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1565         my $vectors = $net->{queues} * 2 + 2;
1566         $tmpstr .= ",vectors=$vectors,mq=on";
1567     }
1568     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1569
1570     if ($use_old_bios_files) {
1571         my $romfile;
1572         if ($device eq 'virtio-net-pci') {
1573             $romfile = 'pxe-virtio.rom';
1574         } elsif ($device eq 'e1000') {
1575             $romfile = 'pxe-e1000.rom';
1576         } elsif ($device eq 'ne2k') {
1577             $romfile = 'pxe-ne2k_pci.rom';
1578         } elsif ($device eq 'pcnet') {
1579             $romfile = 'pxe-pcnet.rom';
1580         } elsif ($device eq 'rtl8139') {
1581             $romfile = 'pxe-rtl8139.rom';
1582         }
1583         $tmpstr .= ",romfile=$romfile" if $romfile;
1584     }
1585
1586     return $tmpstr;
1587 }
1588
1589 sub print_netdev_full {
1590     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1591
1592     my $i = '';
1593     if ($netid =~ m/^net(\d+)$/) {
1594         $i = int($1);
1595     }
1596
1597     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1598
1599     my $ifname = "tap${vmid}i$i";
1600
1601     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1602     die "interface name '$ifname' is too long (max 15 character)\n"
1603         if length($ifname) >= 16;
1604
1605     my $vhostparam = '';
1606     if (is_native($arch)) {
1607         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1608     }
1609
1610     my $vmname = $conf->{name} || "vm$vmid";
1611
1612     my $netdev = "";
1613     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1614
1615     if ($net->{bridge}) {
1616         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1617     } else {
1618         $netdev = "type=user,id=$netid,hostname=$vmname";
1619     }
1620
1621     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1622
1623     return $netdev;
1624 }
1625
1626 my $vga_map = {
1627     'cirrus' => 'cirrus-vga',
1628     'std' => 'VGA',
1629     'vmware' => 'vmware-svga',
1630     'virtio' => 'virtio-vga',
1631 };
1632
1633 sub print_vga_device {
1634     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1635
1636     my $type = $vga_map->{$vga->{type}};
1637     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1638         $type = 'virtio-gpu';
1639     }
1640     my $vgamem_mb = $vga->{memory};
1641
1642     my $max_outputs = '';
1643     if ($qxlnum) {
1644         $type = $id ? 'qxl' : 'qxl-vga';
1645
1646         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1647             # set max outputs so linux can have up to 4 qxl displays with one device
1648             if (min_version($machine_version, 4, 1)) {
1649                 $max_outputs = ",max_outputs=4";
1650             }
1651         }
1652     }
1653
1654     die "no devicetype for $vga->{type}\n" if !$type;
1655
1656     my $memory = "";
1657     if ($vgamem_mb) {
1658         if ($vga->{type} eq 'virtio') {
1659             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1660             $memory = ",max_hostmem=$bytes";
1661         } elsif ($qxlnum) {
1662             # from https://www.spice-space.org/multiple-monitors.html
1663             $memory = ",vgamem_mb=$vga->{memory}";
1664             my $ram = $vgamem_mb * 4;
1665             my $vram = $vgamem_mb * 2;
1666             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1667         } else {
1668             $memory = ",vgamem_mb=$vga->{memory}";
1669         }
1670     } elsif ($qxlnum && $id) {
1671         $memory = ",ram_size=67108864,vram_size=33554432";
1672     }
1673
1674     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1675     my $vgaid = "vga" . ($id // '');
1676     my $pciaddr;
1677
1678     if ($q35 && $vgaid eq 'vga') {
1679         # the first display uses pcie.0 bus on q35 machines
1680         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1681     } else {
1682         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1683     }
1684
1685     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1686 }
1687
1688 sub parse_number_sets {
1689     my ($set) = @_;
1690     my $res = [];
1691     foreach my $part (split(/;/, $set)) {
1692         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1693             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1694             push @$res, [ $1, $2 ];
1695         } else {
1696             die "invalid range: $part\n";
1697         }
1698     }
1699     return $res;
1700 }
1701
1702 sub parse_numa {
1703     my ($data) = @_;
1704
1705     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1706     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1707     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1708     return $res;
1709 }
1710
1711 sub parse_hostpci {
1712     my ($value) = @_;
1713
1714     return undef if !$value;
1715
1716     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1717
1718     my @idlist = split(/;/, $res->{host});
1719     delete $res->{host};
1720     foreach my $id (@idlist) {
1721         my $devs = PVE::SysFSTools::lspci($id);
1722         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1723         push @{$res->{pciid}}, @$devs;
1724     }
1725     return $res;
1726 }
1727
1728 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1729 sub parse_net {
1730     my ($data) = @_;
1731
1732     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1733     if ($@) {
1734         warn $@;
1735         return undef;
1736     }
1737     if (!defined($res->{macaddr})) {
1738         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1739         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1740     }
1741     return $res;
1742 }
1743
1744 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1745 sub parse_ipconfig {
1746     my ($data) = @_;
1747
1748     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1749     if ($@) {
1750         warn $@;
1751         return undef;
1752     }
1753
1754     if ($res->{gw} && !$res->{ip}) {
1755         warn 'gateway specified without specifying an IP address';
1756         return undef;
1757     }
1758     if ($res->{gw6} && !$res->{ip6}) {
1759         warn 'IPv6 gateway specified without specifying an IPv6 address';
1760         return undef;
1761     }
1762     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1763         warn 'gateway specified together with DHCP';
1764         return undef;
1765     }
1766     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1767         # gw6 + auto/dhcp
1768         warn "IPv6 gateway specified together with $res->{ip6} address";
1769         return undef;
1770     }
1771
1772     if (!$res->{ip} && !$res->{ip6}) {
1773         return { ip => 'dhcp', ip6 => 'dhcp' };
1774     }
1775
1776     return $res;
1777 }
1778
1779 sub print_net {
1780     my $net = shift;
1781
1782     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1783 }
1784
1785 sub add_random_macs {
1786     my ($settings) = @_;
1787
1788     foreach my $opt (keys %$settings) {
1789         next if $opt !~ m/^net(\d+)$/;
1790         my $net = parse_net($settings->{$opt});
1791         next if !$net;
1792         $settings->{$opt} = print_net($net);
1793     }
1794 }
1795
1796 sub vm_is_volid_owner {
1797     my ($storecfg, $vmid, $volid) = @_;
1798
1799     if ($volid !~  m|^/|) {
1800         my ($path, $owner);
1801         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1802         if ($owner && ($owner == $vmid)) {
1803             return 1;
1804         }
1805     }
1806
1807     return undef;
1808 }
1809
1810 sub vmconfig_register_unused_drive {
1811     my ($storecfg, $vmid, $conf, $drive) = @_;
1812
1813     if (drive_is_cloudinit($drive)) {
1814         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1815         warn $@ if $@;
1816     } elsif (!drive_is_cdrom($drive)) {
1817         my $volid = $drive->{file};
1818         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1819             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1820         }
1821     }
1822 }
1823
1824 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1825 my $smbios1_fmt = {
1826     uuid => {
1827         type => 'string',
1828         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1829         format_description => 'UUID',
1830         description => "Set SMBIOS1 UUID.",
1831         optional => 1,
1832     },
1833     version => {
1834         type => 'string',
1835         pattern => '[A-Za-z0-9+\/]+={0,2}',
1836         format_description => 'Base64 encoded string',
1837         description => "Set SMBIOS1 version.",
1838         optional => 1,
1839     },
1840     serial => {
1841         type => 'string',
1842         pattern => '[A-Za-z0-9+\/]+={0,2}',
1843         format_description => 'Base64 encoded string',
1844         description => "Set SMBIOS1 serial number.",
1845         optional => 1,
1846     },
1847     manufacturer => {
1848         type => 'string',
1849         pattern => '[A-Za-z0-9+\/]+={0,2}',
1850         format_description => 'Base64 encoded string',
1851         description => "Set SMBIOS1 manufacturer.",
1852         optional => 1,
1853     },
1854     product => {
1855         type => 'string',
1856         pattern => '[A-Za-z0-9+\/]+={0,2}',
1857         format_description => 'Base64 encoded string',
1858         description => "Set SMBIOS1 product ID.",
1859         optional => 1,
1860     },
1861     sku => {
1862         type => 'string',
1863         pattern => '[A-Za-z0-9+\/]+={0,2}',
1864         format_description => 'Base64 encoded string',
1865         description => "Set SMBIOS1 SKU string.",
1866         optional => 1,
1867     },
1868     family => {
1869         type => 'string',
1870         pattern => '[A-Za-z0-9+\/]+={0,2}',
1871         format_description => 'Base64 encoded string',
1872         description => "Set SMBIOS1 family string.",
1873         optional => 1,
1874     },
1875     base64 => {
1876         type => 'boolean',
1877         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1878         optional => 1,
1879     },
1880 };
1881
1882 sub parse_smbios1 {
1883     my ($data) = @_;
1884
1885     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1886     warn $@ if $@;
1887     return $res;
1888 }
1889
1890 sub print_smbios1 {
1891     my ($smbios1) = @_;
1892     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1893 }
1894
1895 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1896
1897 sub parse_watchdog {
1898     my ($value) = @_;
1899
1900     return undef if !$value;
1901
1902     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1903     warn $@ if $@;
1904     return $res;
1905 }
1906
1907 sub parse_guest_agent {
1908     my ($value) = @_;
1909
1910     return {} if !defined($value->{agent});
1911
1912     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1913     warn $@ if $@;
1914
1915     # if the agent is disabled ignore the other potentially set properties
1916     return {} if !$res->{enabled};
1917     return $res;
1918 }
1919
1920 sub parse_vga {
1921     my ($value) = @_;
1922
1923     return {} if !$value;
1924     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1925     warn $@ if $@;
1926     return $res;
1927 }
1928
1929 sub parse_rng {
1930     my ($value) = @_;
1931
1932     return undef if !$value;
1933
1934     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1935     warn $@ if $@;
1936     return $res;
1937 }
1938
1939 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1940 sub verify_usb_device {
1941     my ($value, $noerr) = @_;
1942
1943     return $value if parse_usb_device($value);
1944
1945     return undef if $noerr;
1946
1947     die "unable to parse usb device\n";
1948 }
1949
1950 # add JSON properties for create and set function
1951 sub json_config_properties {
1952     my $prop = shift;
1953
1954     foreach my $opt (keys %$confdesc) {
1955         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1956         $prop->{$opt} = $confdesc->{$opt};
1957     }
1958
1959     return $prop;
1960 }
1961
1962 # return copy of $confdesc_cloudinit to generate documentation
1963 sub cloudinit_config_properties {
1964
1965     return dclone($confdesc_cloudinit);
1966 }
1967
1968 sub check_type {
1969     my ($key, $value) = @_;
1970
1971     die "unknown setting '$key'\n" if !$confdesc->{$key};
1972
1973     my $type = $confdesc->{$key}->{type};
1974
1975     if (!defined($value)) {
1976         die "got undefined value\n";
1977     }
1978
1979     if ($value =~ m/[\n\r]/) {
1980         die "property contains a line feed\n";
1981     }
1982
1983     if ($type eq 'boolean') {
1984         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1985         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1986         die "type check ('boolean') failed - got '$value'\n";
1987     } elsif ($type eq 'integer') {
1988         return int($1) if $value =~ m/^(\d+)$/;
1989         die "type check ('integer') failed - got '$value'\n";
1990     } elsif ($type eq 'number') {
1991         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1992         die "type check ('number') failed - got '$value'\n";
1993     } elsif ($type eq 'string') {
1994         if (my $fmt = $confdesc->{$key}->{format}) {
1995             PVE::JSONSchema::check_format($fmt, $value);
1996             return $value;
1997         }
1998         $value =~ s/^\"(.*)\"$/$1/;
1999         return $value;
2000     } else {
2001         die "internal error"
2002     }
2003 }
2004
2005 sub destroy_vm {
2006     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2007
2008     my $conf = PVE::QemuConfig->load_config($vmid);
2009
2010     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2011
2012     if ($conf->{template}) {
2013         # check if any base image is still used by a linked clone
2014         foreach_drive($conf, sub {
2015                 my ($ds, $drive) = @_;
2016                 return if drive_is_cdrom($drive);
2017
2018                 my $volid = $drive->{file};
2019                 return if !$volid || $volid =~ m|^/|;
2020
2021                 die "base volume '$volid' is still in use by linked cloned\n"
2022                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2023
2024         });
2025     }
2026
2027     # only remove disks owned by this VM
2028     foreach_drive($conf, sub {
2029         my ($ds, $drive) = @_;
2030         return if drive_is_cdrom($drive, 1);
2031
2032         my $volid = $drive->{file};
2033         return if !$volid || $volid =~ m|^/|;
2034
2035         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2036         return if !$path || !$owner || ($owner != $vmid);
2037
2038         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2039         warn "Could not remove disk '$volid', check manually: $@" if $@;
2040     });
2041
2042     # also remove unused disk
2043     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2044     PVE::Storage::foreach_volid($vmdisks, sub {
2045         my ($volid, $sid, $volname, $d) = @_;
2046         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2047         warn $@ if $@;
2048     });
2049
2050     if (defined $replacement_conf) {
2051         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2052     } else {
2053         PVE::QemuConfig->destroy_config($vmid);
2054     }
2055 }
2056
2057 sub parse_vm_config {
2058     my ($filename, $raw) = @_;
2059
2060     return undef if !defined($raw);
2061
2062     my $res = {
2063         digest => Digest::SHA::sha1_hex($raw),
2064         snapshots => {},
2065         pending => {},
2066     };
2067
2068     $filename =~ m|/qemu-server/(\d+)\.conf$|
2069         || die "got strange filename '$filename'";
2070
2071     my $vmid = $1;
2072
2073     my $conf = $res;
2074     my $descr;
2075     my $section = '';
2076
2077     my @lines = split(/\n/, $raw);
2078     foreach my $line (@lines) {
2079         next if $line =~ m/^\s*$/;
2080
2081         if ($line =~ m/^\[PENDING\]\s*$/i) {
2082             $section = 'pending';
2083             if (defined($descr)) {
2084                 $descr =~ s/\s+$//;
2085                 $conf->{description} = $descr;
2086             }
2087             $descr = undef;
2088             $conf = $res->{$section} = {};
2089             next;
2090
2091         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2092             $section = $1;
2093             if (defined($descr)) {
2094                 $descr =~ s/\s+$//;
2095                 $conf->{description} = $descr;
2096             }
2097             $descr = undef;
2098             $conf = $res->{snapshots}->{$section} = {};
2099             next;
2100         }
2101
2102         if ($line =~ m/^\#(.*)\s*$/) {
2103             $descr = '' if !defined($descr);
2104             $descr .= PVE::Tools::decode_text($1) . "\n";
2105             next;
2106         }
2107
2108         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2109             $descr = '' if !defined($descr);
2110             $descr .= PVE::Tools::decode_text($2);
2111         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2112             $conf->{snapstate} = $1;
2113         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2114             my $key = $1;
2115             my $value = $2;
2116             $conf->{$key} = $value;
2117         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2118             my $value = $1;
2119             if ($section eq 'pending') {
2120                 $conf->{delete} = $value; # we parse this later
2121             } else {
2122                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2123             }
2124         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2125             my $key = $1;
2126             my $value = $2;
2127             eval { $value = check_type($key, $value); };
2128             if ($@) {
2129                 warn "vm $vmid - unable to parse value of '$key' - $@";
2130             } else {
2131                 $key = 'ide2' if $key eq 'cdrom';
2132                 my $fmt = $confdesc->{$key}->{format};
2133                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2134                     my $v = parse_drive($key, $value);
2135                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2136                         $v->{file} = $volid;
2137                         $value = print_drive($v);
2138                     } else {
2139                         warn "vm $vmid - unable to parse value of '$key'\n";
2140                         next;
2141                     }
2142                 }
2143
2144                 $conf->{$key} = $value;
2145             }
2146         }
2147     }
2148
2149     if (defined($descr)) {
2150         $descr =~ s/\s+$//;
2151         $conf->{description} = $descr;
2152     }
2153     delete $res->{snapstate}; # just to be sure
2154
2155     return $res;
2156 }
2157
2158 sub write_vm_config {
2159     my ($filename, $conf) = @_;
2160
2161     delete $conf->{snapstate}; # just to be sure
2162
2163     if ($conf->{cdrom}) {
2164         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2165         $conf->{ide2} = $conf->{cdrom};
2166         delete $conf->{cdrom};
2167     }
2168
2169     # we do not use 'smp' any longer
2170     if ($conf->{sockets}) {
2171         delete $conf->{smp};
2172     } elsif ($conf->{smp}) {
2173         $conf->{sockets} = $conf->{smp};
2174         delete $conf->{cores};
2175         delete $conf->{smp};
2176     }
2177
2178     my $used_volids = {};
2179
2180     my $cleanup_config = sub {
2181         my ($cref, $pending, $snapname) = @_;
2182
2183         foreach my $key (keys %$cref) {
2184             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2185                 $key eq 'snapstate' || $key eq 'pending';
2186             my $value = $cref->{$key};
2187             if ($key eq 'delete') {
2188                 die "propertry 'delete' is only allowed in [PENDING]\n"
2189                     if !$pending;
2190                 # fixme: check syntax?
2191                 next;
2192             }
2193             eval { $value = check_type($key, $value); };
2194             die "unable to parse value of '$key' - $@" if $@;
2195
2196             $cref->{$key} = $value;
2197
2198             if (!$snapname && is_valid_drivename($key)) {
2199                 my $drive = parse_drive($key, $value);
2200                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2201             }
2202         }
2203     };
2204
2205     &$cleanup_config($conf);
2206
2207     &$cleanup_config($conf->{pending}, 1);
2208
2209     foreach my $snapname (keys %{$conf->{snapshots}}) {
2210         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2211         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2212     }
2213
2214     # remove 'unusedX' settings if we re-add a volume
2215     foreach my $key (keys %$conf) {
2216         my $value = $conf->{$key};
2217         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2218             delete $conf->{$key};
2219         }
2220     }
2221
2222     my $generate_raw_config = sub {
2223         my ($conf, $pending) = @_;
2224
2225         my $raw = '';
2226
2227         # add description as comment to top of file
2228         if (defined(my $descr = $conf->{description})) {
2229             if ($descr) {
2230                 foreach my $cl (split(/\n/, $descr)) {
2231                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2232                 }
2233             } else {
2234                 $raw .= "#\n" if $pending;
2235             }
2236         }
2237
2238         foreach my $key (sort keys %$conf) {
2239             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2240             $raw .= "$key: $conf->{$key}\n";
2241         }
2242         return $raw;
2243     };
2244
2245     my $raw = &$generate_raw_config($conf);
2246
2247     if (scalar(keys %{$conf->{pending}})){
2248         $raw .= "\n[PENDING]\n";
2249         $raw .= &$generate_raw_config($conf->{pending}, 1);
2250     }
2251
2252     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2253         $raw .= "\n[$snapname]\n";
2254         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2255     }
2256
2257     return $raw;
2258 }
2259
2260 sub load_defaults {
2261
2262     my $res = {};
2263
2264     # we use static defaults from our JSON schema configuration
2265     foreach my $key (keys %$confdesc) {
2266         if (defined(my $default = $confdesc->{$key}->{default})) {
2267             $res->{$key} = $default;
2268         }
2269     }
2270
2271     return $res;
2272 }
2273
2274 sub config_list {
2275     my $vmlist = PVE::Cluster::get_vmlist();
2276     my $res = {};
2277     return $res if !$vmlist || !$vmlist->{ids};
2278     my $ids = $vmlist->{ids};
2279     my $nodename = nodename();
2280
2281     foreach my $vmid (keys %$ids) {
2282         my $d = $ids->{$vmid};
2283         next if !$d->{node} || $d->{node} ne $nodename;
2284         next if !$d->{type} || $d->{type} ne 'qemu';
2285         $res->{$vmid}->{exists} = 1;
2286     }
2287     return $res;
2288 }
2289
2290 # test if VM uses local resources (to prevent migration)
2291 sub check_local_resources {
2292     my ($conf, $noerr) = @_;
2293
2294     my @loc_res = ();
2295
2296     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2297     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2298
2299     push @loc_res, "ivshmem" if $conf->{ivshmem};
2300
2301     foreach my $k (keys %$conf) {
2302         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2303         # sockets are safe: they will recreated be on the target side post-migrate
2304         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2305         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2306     }
2307
2308     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2309
2310     return \@loc_res;
2311 }
2312
2313 # check if used storages are available on all nodes (use by migrate)
2314 sub check_storage_availability {
2315     my ($storecfg, $conf, $node) = @_;
2316
2317     foreach_drive($conf, sub {
2318         my ($ds, $drive) = @_;
2319
2320         my $volid = $drive->{file};
2321         return if !$volid;
2322
2323         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2324         return if !$sid;
2325
2326         # check if storage is available on both nodes
2327         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2328         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2329    });
2330 }
2331
2332 # list nodes where all VM images are available (used by has_feature API)
2333 sub shared_nodes {
2334     my ($conf, $storecfg) = @_;
2335
2336     my $nodelist = PVE::Cluster::get_nodelist();
2337     my $nodehash = { map { $_ => 1 } @$nodelist };
2338     my $nodename = nodename();
2339
2340     foreach_drive($conf, sub {
2341         my ($ds, $drive) = @_;
2342
2343         my $volid = $drive->{file};
2344         return if !$volid;
2345
2346         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2347         if ($storeid) {
2348             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2349             if ($scfg->{disable}) {
2350                 $nodehash = {};
2351             } elsif (my $avail = $scfg->{nodes}) {
2352                 foreach my $node (keys %$nodehash) {
2353                     delete $nodehash->{$node} if !$avail->{$node};
2354                 }
2355             } elsif (!$scfg->{shared}) {
2356                 foreach my $node (keys %$nodehash) {
2357                     delete $nodehash->{$node} if $node ne $nodename
2358                 }
2359             }
2360         }
2361     });
2362
2363     return $nodehash
2364 }
2365
2366 sub check_local_storage_availability {
2367     my ($conf, $storecfg) = @_;
2368
2369     my $nodelist = PVE::Cluster::get_nodelist();
2370     my $nodehash = { map { $_ => {} } @$nodelist };
2371
2372     foreach_drive($conf, sub {
2373         my ($ds, $drive) = @_;
2374
2375         my $volid = $drive->{file};
2376         return if !$volid;
2377
2378         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2379         if ($storeid) {
2380             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2381
2382             if ($scfg->{disable}) {
2383                 foreach my $node (keys %$nodehash) {
2384                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2385                 }
2386             } elsif (my $avail = $scfg->{nodes}) {
2387                 foreach my $node (keys %$nodehash) {
2388                     if (!$avail->{$node}) {
2389                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2390                     }
2391                 }
2392             }
2393         }
2394     });
2395
2396     foreach my $node (values %$nodehash) {
2397         if (my $unavail = $node->{unavailable_storages}) {
2398             $node->{unavailable_storages} = [ sort keys %$unavail ];
2399         }
2400     }
2401
2402     return $nodehash
2403 }
2404
2405 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2406 sub check_running {
2407     my ($vmid, $nocheck, $node) = @_;
2408
2409     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2410     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2411 }
2412
2413 sub vzlist {
2414
2415     my $vzlist = config_list();
2416
2417     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2418
2419     while (defined(my $de = $fd->read)) {
2420         next if $de !~ m/^(\d+)\.pid$/;
2421         my $vmid = $1;
2422         next if !defined($vzlist->{$vmid});
2423         if (my $pid = check_running($vmid)) {
2424             $vzlist->{$vmid}->{pid} = $pid;
2425         }
2426     }
2427
2428     return $vzlist;
2429 }
2430
2431 our $vmstatus_return_properties = {
2432     vmid => get_standard_option('pve-vmid'),
2433     status => {
2434         description => "Qemu process status.",
2435         type => 'string',
2436         enum => ['stopped', 'running'],
2437     },
2438     maxmem => {
2439         description => "Maximum memory in bytes.",
2440         type => 'integer',
2441         optional => 1,
2442         renderer => 'bytes',
2443     },
2444     maxdisk => {
2445         description => "Root disk size in bytes.",
2446         type => 'integer',
2447         optional => 1,
2448         renderer => 'bytes',
2449     },
2450     name => {
2451         description => "VM name.",
2452         type => 'string',
2453         optional => 1,
2454     },
2455     qmpstatus => {
2456         description => "Qemu QMP agent status.",
2457         type => 'string',
2458         optional => 1,
2459     },
2460     pid => {
2461         description => "PID of running qemu process.",
2462         type => 'integer',
2463         optional => 1,
2464     },
2465     uptime => {
2466         description => "Uptime.",
2467         type => 'integer',
2468         optional => 1,
2469         renderer => 'duration',
2470     },
2471     cpus => {
2472         description => "Maximum usable CPUs.",
2473         type => 'number',
2474         optional => 1,
2475     },
2476     lock => {
2477         description => "The current config lock, if any.",
2478         type => 'string',
2479         optional => 1,
2480     },
2481     tags => {
2482         description  => "The current configured tags, if any",
2483         type => 'string',
2484         optional => 1,
2485     },
2486 };
2487
2488 my $last_proc_pid_stat;
2489
2490 # get VM status information
2491 # This must be fast and should not block ($full == false)
2492 # We only query KVM using QMP if $full == true (this can be slow)
2493 sub vmstatus {
2494     my ($opt_vmid, $full) = @_;
2495
2496     my $res = {};
2497
2498     my $storecfg = PVE::Storage::config();
2499
2500     my $list = vzlist();
2501     my $defaults = load_defaults();
2502
2503     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2504
2505     my $cpucount = $cpuinfo->{cpus} || 1;
2506
2507     foreach my $vmid (keys %$list) {
2508         next if $opt_vmid && ($vmid ne $opt_vmid);
2509
2510         my $conf = PVE::QemuConfig->load_config($vmid);
2511
2512         my $d = { vmid => $vmid };
2513         $d->{pid} = $list->{$vmid}->{pid};
2514
2515         # fixme: better status?
2516         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2517
2518         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2519         if (defined($size)) {
2520             $d->{disk} = 0; # no info available
2521             $d->{maxdisk} = $size;
2522         } else {
2523             $d->{disk} = 0;
2524             $d->{maxdisk} = 0;
2525         }
2526
2527         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2528             * ($conf->{cores} || $defaults->{cores});
2529         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2530         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2531
2532         $d->{name} = $conf->{name} || "VM $vmid";
2533         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2534             : $defaults->{memory}*(1024*1024);
2535
2536         if ($conf->{balloon}) {
2537             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2538             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2539                 : $defaults->{shares};
2540         }
2541
2542         $d->{uptime} = 0;
2543         $d->{cpu} = 0;
2544         $d->{mem} = 0;
2545
2546         $d->{netout} = 0;
2547         $d->{netin} = 0;
2548
2549         $d->{diskread} = 0;
2550         $d->{diskwrite} = 0;
2551
2552         $d->{template} = PVE::QemuConfig->is_template($conf);
2553
2554         $d->{serial} = 1 if conf_has_serial($conf);
2555         $d->{lock} = $conf->{lock} if $conf->{lock};
2556         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2557
2558         $res->{$vmid} = $d;
2559     }
2560
2561     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2562     foreach my $dev (keys %$netdev) {
2563         next if $dev !~ m/^tap([1-9]\d*)i/;
2564         my $vmid = $1;
2565         my $d = $res->{$vmid};
2566         next if !$d;
2567
2568         $d->{netout} += $netdev->{$dev}->{receive};
2569         $d->{netin} += $netdev->{$dev}->{transmit};
2570
2571         if ($full) {
2572             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2573             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2574         }
2575
2576     }
2577
2578     my $ctime = gettimeofday;
2579
2580     foreach my $vmid (keys %$list) {
2581
2582         my $d = $res->{$vmid};
2583         my $pid = $d->{pid};
2584         next if !$pid;
2585
2586         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2587         next if !$pstat; # not running
2588
2589         my $used = $pstat->{utime} + $pstat->{stime};
2590
2591         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2592
2593         if ($pstat->{vsize}) {
2594             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2595         }
2596
2597         my $old = $last_proc_pid_stat->{$pid};
2598         if (!$old) {
2599             $last_proc_pid_stat->{$pid} = {
2600                 time => $ctime,
2601                 used => $used,
2602                 cpu => 0,
2603             };
2604             next;
2605         }
2606
2607         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2608
2609         if ($dtime > 1000) {
2610             my $dutime = $used -  $old->{used};
2611
2612             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2613             $last_proc_pid_stat->{$pid} = {
2614                 time => $ctime,
2615                 used => $used,
2616                 cpu => $d->{cpu},
2617             };
2618         } else {
2619             $d->{cpu} = $old->{cpu};
2620         }
2621     }
2622
2623     return $res if !$full;
2624
2625     my $qmpclient = PVE::QMPClient->new();
2626
2627     my $ballooncb = sub {
2628         my ($vmid, $resp) = @_;
2629
2630         my $info = $resp->{'return'};
2631         return if !$info->{max_mem};
2632
2633         my $d = $res->{$vmid};
2634
2635         # use memory assigned to VM
2636         $d->{maxmem} = $info->{max_mem};
2637         $d->{balloon} = $info->{actual};
2638
2639         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2640             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2641             $d->{freemem} = $info->{free_mem};
2642         }
2643
2644         $d->{ballooninfo} = $info;
2645     };
2646
2647     my $blockstatscb = sub {
2648         my ($vmid, $resp) = @_;
2649         my $data = $resp->{'return'} || [];
2650         my $totalrdbytes = 0;
2651         my $totalwrbytes = 0;
2652
2653         for my $blockstat (@$data) {
2654             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2655             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2656
2657             $blockstat->{device} =~ s/drive-//;
2658             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2659         }
2660         $res->{$vmid}->{diskread} = $totalrdbytes;
2661         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2662     };
2663
2664     my $statuscb = sub {
2665         my ($vmid, $resp) = @_;
2666
2667         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2668         # this fails if ballon driver is not loaded, so this must be
2669         # the last commnand (following command are aborted if this fails).
2670         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2671
2672         my $status = 'unknown';
2673         if (!defined($status = $resp->{'return'}->{status})) {
2674             warn "unable to get VM status\n";
2675             return;
2676         }
2677
2678         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2679     };
2680
2681     foreach my $vmid (keys %$list) {
2682         next if $opt_vmid && ($vmid ne $opt_vmid);
2683         next if !$res->{$vmid}->{pid}; # not running
2684         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2685     }
2686
2687     $qmpclient->queue_execute(undef, 2);
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2692     }
2693
2694     return $res;
2695 }
2696
2697 sub conf_has_serial {
2698     my ($conf) = @_;
2699
2700     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2701         if ($conf->{"serial$i"}) {
2702             return 1;
2703         }
2704     }
2705
2706     return 0;
2707 }
2708
2709 sub conf_has_audio {
2710     my ($conf, $id) = @_;
2711
2712     $id //= 0;
2713     my $audio = $conf->{"audio$id"};
2714     return undef if !defined($audio);
2715
2716     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2717     my $audiodriver = $audioproperties->{driver} // 'spice';
2718
2719     return {
2720         dev => $audioproperties->{device},
2721         dev_id => "audiodev$id",
2722         backend => $audiodriver,
2723         backend_id => "$audiodriver-backend${id}",
2724     };
2725 }
2726
2727 sub vga_conf_has_spice {
2728     my ($vga) = @_;
2729
2730     my $vgaconf = parse_vga($vga);
2731     my $vgatype = $vgaconf->{type};
2732     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2733
2734     return $1 || 1;
2735 }
2736
2737 sub is_native($) {
2738     my ($arch) = @_;
2739     return get_host_arch() eq $arch;
2740 }
2741
2742 sub get_vm_arch {
2743     my ($conf) = @_;
2744     return $conf->{arch} // get_host_arch();
2745 }
2746
2747 my $default_machines = {
2748     x86_64 => 'pc',
2749     aarch64 => 'virt',
2750 };
2751
2752 sub get_vm_machine {
2753     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2754
2755     my $machine = $forcemachine || $conf->{machine};
2756
2757     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2758         $arch //= 'x86_64';
2759         $machine ||= $default_machines->{$arch};
2760         if ($add_pve_version) {
2761             $kvmversion //= kvm_user_version();
2762             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2763             $machine .= "+pve$pvever";
2764         }
2765     }
2766
2767     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2768         # for version-pinned machines that do not include a pve-version (e.g.
2769         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2770         $machine .= '+pve0';
2771     }
2772
2773     return $machine;
2774 }
2775
2776 sub get_ovmf_files($) {
2777     my ($arch) = @_;
2778
2779     my $ovmf = $OVMF->{$arch}
2780         or die "no OVMF images known for architecture '$arch'\n";
2781
2782     return @$ovmf;
2783 }
2784
2785 my $Arch2Qemu = {
2786     aarch64 => '/usr/bin/qemu-system-aarch64',
2787     x86_64 => '/usr/bin/qemu-system-x86_64',
2788 };
2789 sub get_command_for_arch($) {
2790     my ($arch) = @_;
2791     return '/usr/bin/kvm' if is_native($arch);
2792
2793     my $cmd = $Arch2Qemu->{$arch}
2794         or die "don't know how to emulate architecture '$arch'\n";
2795     return $cmd;
2796 }
2797
2798 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2799 # to use in a QEMU command line (-cpu element), first array_intersect the result
2800 # of query_supported_ with query_understood_. This is necessary because:
2801 #
2802 # a) query_understood_ returns flags the host cannot use and
2803 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2804 #    flags, but CPU settings - with most of them being flags. Those settings
2805 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2806 #
2807 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2808 # expensive. If you need the value returned from this, you can get it much
2809 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2810 # $accel being 'kvm' or 'tcg'.
2811 #
2812 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2813 # changes, automatically populating pmxcfs.
2814 #
2815 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2816 # since kvm and tcg machines support different flags
2817 #
2818 sub query_supported_cpu_flags {
2819     my ($arch) = @_;
2820
2821     $arch //= get_host_arch();
2822     my $default_machine = $default_machines->{$arch};
2823
2824     my $flags = {};
2825
2826     # FIXME: Once this is merged, the code below should work for ARM as well:
2827     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2828     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2829         $arch eq "aarch64";
2830
2831     my $kvm_supported = defined(kvm_version());
2832     my $qemu_cmd = get_command_for_arch($arch);
2833     my $fakevmid = -1;
2834     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2835
2836     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2837     my $query_supported_run_qemu = sub {
2838         my ($kvm) = @_;
2839
2840         my $flags = {};
2841         my $cmd = [
2842             $qemu_cmd,
2843             '-machine', $default_machine,
2844             '-display', 'none',
2845             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2846             '-mon', 'chardev=qmp,mode=control',
2847             '-pidfile', $pidfile,
2848             '-S', '-daemonize'
2849         ];
2850
2851         if (!$kvm) {
2852             push @$cmd, '-accel', 'tcg';
2853         }
2854
2855         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2856         die "QEMU flag querying VM exited with code " . $rc if $rc;
2857
2858         eval {
2859             my $cmd_result = mon_cmd(
2860                 $fakevmid,
2861                 'query-cpu-model-expansion',
2862                 type => 'full',
2863                 model => { name => 'host' }
2864             );
2865
2866             my $props = $cmd_result->{model}->{props};
2867             foreach my $prop (keys %$props) {
2868                 next if $props->{$prop} ne '1';
2869                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2870                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2871                 # We only keep those with underscores, to match /proc/cpuinfo
2872                 $prop =~ s/\.|-/_/g;
2873                 $flags->{$prop} = 1;
2874             }
2875         };
2876         my $err = $@;
2877
2878         # force stop with 10 sec timeout and 'nocheck'
2879         # always stop, even if QMP failed
2880         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2881
2882         die $err if $err;
2883
2884         return [ sort keys %$flags ];
2885     };
2886
2887     # We need to query QEMU twice, since KVM and TCG have different supported flags
2888     PVE::QemuConfig->lock_config($fakevmid, sub {
2889         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2890         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2891
2892         if ($kvm_supported) {
2893             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2894             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2895         }
2896     });
2897
2898     return $flags;
2899 }
2900
2901 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2902 my $understood_cpu_flag_dir = "/usr/share/kvm";
2903 sub query_understood_cpu_flags {
2904     my $arch = get_host_arch();
2905     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2906
2907     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2908         if ! -e $filepath;
2909
2910     my $raw = file_get_contents($filepath);
2911     $raw =~ s/^\s+|\s+$//g;
2912     my @flags = split(/\s+/, $raw);
2913
2914     return \@flags;
2915 }
2916
2917 sub config_to_command {
2918     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2919
2920     my $cmd = [];
2921     my $globalFlags = [];
2922     my $machineFlags = [];
2923     my $rtcFlags = [];
2924     my $devices = [];
2925     my $pciaddr = '';
2926     my $bridges = {};
2927     my $ostype = $conf->{ostype};
2928     my $winversion = windows_version($ostype);
2929     my $kvm = $conf->{kvm};
2930     my $nodename = nodename();
2931
2932     my $arch = get_vm_arch($conf);
2933     my $kvm_binary = get_command_for_arch($arch);
2934     my $kvmver = kvm_user_version($kvm_binary);
2935
2936     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2937         $kvmver //= "undefined";
2938         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2939     }
2940
2941     my $add_pve_version = min_version($kvmver, 4, 1);
2942
2943     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2944     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2945     $kvm //= 1 if is_native($arch);
2946
2947     $machine_version =~ m/(\d+)\.(\d+)/;
2948     my ($machine_major, $machine_minor) = ($1, $2);
2949     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2950         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2951
2952     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2953         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2954         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2955           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2956     }
2957
2958     # if a specific +pve version is required for a feature, use $version_guard
2959     # instead of min_version to allow machines to be run with the minimum
2960     # required version
2961     my $required_pve_version = 0;
2962     my $version_guard = sub {
2963         my ($major, $minor, $pve) = @_;
2964         return 0 if !min_version($machine_version, $major, $minor, $pve);
2965         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2966         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2967         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2968         return 1;
2969     };
2970
2971     if ($kvm) {
2972         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2973             if !defined kvm_version();
2974     }
2975
2976     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2977     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2978     my $use_old_bios_files = undef;
2979     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2980
2981     my $cpuunits = defined($conf->{cpuunits}) ?
2982             $conf->{cpuunits} : $defaults->{cpuunits};
2983
2984     push @$cmd, $kvm_binary;
2985
2986     push @$cmd, '-id', $vmid;
2987
2988     my $vmname = $conf->{name} || "vm$vmid";
2989
2990     push @$cmd, '-name', $vmname;
2991
2992     my $use_virtio = 0;
2993
2994     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2995     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2996     push @$cmd, '-mon', "chardev=qmp,mode=control";
2997
2998     if (min_version($machine_version, 2, 12)) {
2999         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3000         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3001     }
3002
3003     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3004
3005     push @$cmd, '-daemonize';
3006
3007     if ($conf->{smbios1}) {
3008         my $smbios_conf = parse_smbios1($conf->{smbios1});
3009         if ($smbios_conf->{base64}) {
3010             # Do not pass base64 flag to qemu
3011             delete $smbios_conf->{base64};
3012             my $smbios_string = "";
3013             foreach my $key (keys %$smbios_conf) {
3014                 my $value;
3015                 if ($key eq "uuid") {
3016                     $value = $smbios_conf->{uuid}
3017                 } else {
3018                     $value = decode_base64($smbios_conf->{$key});
3019                 }
3020                 # qemu accepts any binary data, only commas need escaping by double comma
3021                 $value =~ s/,/,,/g;
3022                 $smbios_string .= "," . $key . "=" . $value if $value;
3023             }
3024             push @$cmd, '-smbios', "type=1" . $smbios_string;
3025         } else {
3026             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3027         }
3028     }
3029
3030     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3031     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3032         die "uefi base image not found\n" if ! -f $ovmf_code;
3033
3034         my $path;
3035         my $format;
3036         if (my $efidisk = $conf->{efidisk0}) {
3037             my $d = parse_drive('efidisk0', $efidisk);
3038             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3039             $format = $d->{format};
3040             if ($storeid) {
3041                 $path = PVE::Storage::path($storecfg, $d->{file});
3042                 if (!defined($format)) {
3043                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3044                     $format = qemu_img_format($scfg, $volname);
3045                 }
3046             } else {
3047                 $path = $d->{file};
3048                 die "efidisk format must be specified\n"
3049                     if !defined($format);
3050             }
3051         } else {
3052             warn "no efidisk configured! Using temporary efivars disk.\n";
3053             $path = "/tmp/$vmid-ovmf.fd";
3054             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3055             $format = 'raw';
3056         }
3057
3058         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3059         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3060     }
3061
3062     # load q35 config
3063     if ($q35) {
3064         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3065         if (min_version($machine_version, 4, 0)) {
3066             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3067         } else {
3068             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3069         }
3070     }
3071
3072     if ($conf->{vmgenid}) {
3073         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3074     }
3075
3076     # add usb controllers
3077     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3078     push @$devices, @usbcontrollers if @usbcontrollers;
3079     my $vga = parse_vga($conf->{vga});
3080
3081     my $qxlnum = vga_conf_has_spice($conf->{vga});
3082     $vga->{type} = 'qxl' if $qxlnum;
3083
3084     if (!$vga->{type}) {
3085         if ($arch eq 'aarch64') {
3086             $vga->{type} = 'virtio';
3087         } elsif (min_version($machine_version, 2, 9)) {
3088             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3089         } else {
3090             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3091         }
3092     }
3093
3094     # enable absolute mouse coordinates (needed by vnc)
3095     my $tablet;
3096     if (defined($conf->{tablet})) {
3097         $tablet = $conf->{tablet};
3098     } else {
3099         $tablet = $defaults->{tablet};
3100         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3101         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3102     }
3103
3104     if ($tablet) {
3105         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3106         my $kbd = print_keyboarddevice_full($conf, $arch);
3107         push @$devices, '-device', $kbd if defined($kbd);
3108     }
3109
3110     my $kvm_off = 0;
3111     my $gpu_passthrough;
3112
3113     # host pci devices
3114     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3115         my $id = "hostpci$i";
3116         my $d = parse_hostpci($conf->{$id});
3117         next if !$d;
3118
3119         if (my $pcie = $d->{pcie}) {
3120             die "q35 machine model is not enabled" if !$q35;
3121             # win7 wants to have the pcie devices directly on the pcie bus
3122             # instead of in the root port
3123             if ($winversion == 7) {
3124                 $pciaddr = print_pcie_addr("${id}bus0");
3125             } else {
3126                 # add more root ports if needed, 4 are present by default
3127                 # by pve-q35 cfgs, rest added here on demand.
3128                 if ($i > 3) {
3129                     push @$devices, '-device', print_pcie_root_port($i);
3130                 }
3131                 $pciaddr = print_pcie_addr($id);
3132             }
3133         } else {
3134             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3135         }
3136
3137         my $xvga = '';
3138         if ($d->{'x-vga'}) {
3139             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3140             $kvm_off = 1;
3141             $vga->{type} = 'none' if !defined($conf->{vga});
3142             $gpu_passthrough = 1;
3143         }
3144
3145         my $pcidevices = $d->{pciid};
3146         my $multifunction = 1 if @$pcidevices > 1;
3147
3148         my $sysfspath;
3149         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3150             my $pci_id = $pcidevices->[0]->{id};
3151             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3152             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3153         } elsif ($d->{mdev}) {
3154             warn "ignoring mediated device '$id' with multifunction device\n";
3155         }
3156
3157         my $j=0;
3158         foreach my $pcidevice (@$pcidevices) {
3159             my $devicestr = "vfio-pci";
3160
3161             if ($sysfspath) {
3162                 $devicestr .= ",sysfsdev=$sysfspath";
3163             } else {
3164                 $devicestr .= ",host=$pcidevice->{id}";
3165             }
3166
3167             my $mf_addr = $multifunction ? ".$j" : '';
3168             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3169
3170             if ($j == 0) {
3171                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3172                 $devicestr .= "$xvga";
3173                 $devicestr .= ",multifunction=on" if $multifunction;
3174                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3175             }
3176
3177             push @$devices, '-device', $devicestr;
3178             $j++;
3179         }
3180     }
3181
3182     # usb devices
3183     my $usb_dev_features = {};
3184     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3185
3186     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3187     push @$devices, @usbdevices if @usbdevices;
3188     # serial devices
3189     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3190         if (my $path = $conf->{"serial$i"}) {
3191             if ($path eq 'socket') {
3192                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3193                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3194                 # On aarch64, serial0 is the UART device. Qemu only allows
3195                 # connecting UART devices via the '-serial' command line, as
3196                 # the device has a fixed slot on the hardware...
3197                 if ($arch eq 'aarch64' && $i == 0) {
3198                     push @$devices, '-serial', "chardev:serial$i";
3199                 } else {
3200                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3201                 }
3202             } else {
3203                 die "no such serial device\n" if ! -c $path;
3204                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3205                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3206             }
3207         }
3208     }
3209
3210     # parallel devices
3211     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3212         if (my $path = $conf->{"parallel$i"}) {
3213             die "no such parallel device\n" if ! -c $path;
3214             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3215             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3216             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3217         }
3218     }
3219
3220     if (my $audio = conf_has_audio($conf)) {
3221
3222         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3223
3224         my $id = $audio->{dev_id};
3225         if ($audio->{dev} eq 'AC97') {
3226             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3227         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3228             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3229             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3230             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3231         } else {
3232             die "unkown audio device '$audio->{dev}', implement me!";
3233         }
3234
3235         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3236     }
3237
3238     my $sockets = 1;
3239     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3240     $sockets = $conf->{sockets} if  $conf->{sockets};
3241
3242     my $cores = $conf->{cores} || 1;
3243
3244     my $maxcpus = $sockets * $cores;
3245
3246     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3247
3248     my $allowed_vcpus = $cpuinfo->{cpus};
3249
3250     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3251         if ($allowed_vcpus < $maxcpus);
3252
3253     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3254
3255         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3256         for (my $i = 2; $i <= $vcpus; $i++)  {
3257             my $cpustr = print_cpu_device($conf,$i);
3258             push @$cmd, '-device', $cpustr;
3259         }
3260
3261     } else {
3262
3263         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3264     }
3265     push @$cmd, '-nodefaults';
3266
3267     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3268
3269     my $bootindex_hash = {};
3270     my $i = 1;
3271     foreach my $o (split(//, $bootorder)) {
3272         $bootindex_hash->{$o} = $i*100;
3273         $i++;
3274     }
3275
3276     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3277
3278     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3279
3280     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3281
3282     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3283         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3284         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3285         push @$cmd,  '-vnc', "unix:$socket,password";
3286     } else {
3287         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3288         push @$cmd, '-nographic';
3289     }
3290
3291     # time drift fix
3292     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3293
3294     my $useLocaltime = $conf->{localtime};
3295
3296     if ($winversion >= 5) { # windows
3297         $useLocaltime = 1 if !defined($conf->{localtime});
3298
3299         # use time drift fix when acpi is enabled
3300         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3301             $tdf = 1 if !defined($conf->{tdf});
3302         }
3303     }
3304
3305     if ($winversion >= 6) {
3306         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3307         push @$cmd, '-no-hpet';
3308     }
3309
3310     push @$rtcFlags, 'driftfix=slew' if $tdf;
3311
3312     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3313         push @$rtcFlags, "base=$conf->{startdate}";
3314     } elsif ($useLocaltime) {
3315         push @$rtcFlags, 'base=localtime';
3316     }
3317
3318     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3319
3320     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3321
3322     push @$cmd, '-S' if $conf->{freeze};
3323
3324     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3325
3326     my $guest_agent = parse_guest_agent($conf);
3327
3328     if ($guest_agent->{enabled}) {
3329         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3330         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3331
3332         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3333             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3334             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3335             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3336         } elsif ($guest_agent->{type} eq 'isa') {
3337             push @$devices, '-device', "isa-serial,chardev=qga0";
3338         }
3339     }
3340
3341     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3342     if ($rng && &$version_guard(4, 1, 2)) {
3343         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3344         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3345
3346         my $limiter_str = "";
3347         if ($max_bytes) {
3348             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3349         }
3350
3351         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3352         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3353             if ! -e $rng->{source};
3354
3355         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3356
3357         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3358         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3359     }
3360
3361     my $spice_port;
3362
3363     if ($qxlnum) {
3364         if ($qxlnum > 1) {
3365             if ($winversion){
3366                 for(my $i = 1; $i < $qxlnum; $i++){
3367                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3368                 }
3369             } else {
3370                 # assume other OS works like Linux
3371                 my ($ram, $vram) = ("134217728", "67108864");
3372                 if ($vga->{memory}) {
3373                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3374                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3375                 }
3376                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3377                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3378             }
3379         }
3380
3381         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3382
3383         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3384         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3385         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3386
3387         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3388         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3389         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3390
3391         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3392         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3393
3394         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3395         if ($spice_enhancement->{foldersharing}) {
3396             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3397             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3398         }
3399
3400         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3401         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3402         push @$devices, '-spice', "$spice_opts";
3403     }
3404
3405     # enable balloon by default, unless explicitly disabled
3406     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3407         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3408         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3409     }
3410
3411     if ($conf->{watchdog}) {
3412         my $wdopts = parse_watchdog($conf->{watchdog});
3413         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3414         my $watchdog = $wdopts->{model} || 'i6300esb';
3415         push @$devices, '-device', "$watchdog$pciaddr";
3416         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3417     }
3418
3419     my $vollist = [];
3420     my $scsicontroller = {};
3421     my $ahcicontroller = {};
3422     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3423
3424     # Add iscsi initiator name if available
3425     if (my $initiator = get_initiator_name()) {
3426         push @$devices, '-iscsi', "initiator-name=$initiator";
3427     }
3428
3429     foreach_drive($conf, sub {
3430         my ($ds, $drive) = @_;
3431
3432         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3433             push @$vollist, $drive->{file};
3434         }
3435
3436         # ignore efidisk here, already added in bios/fw handling code above
3437         return if $drive->{interface} eq 'efidisk';
3438
3439         $use_virtio = 1 if $ds =~ m/^virtio/;
3440
3441         if (drive_is_cdrom ($drive)) {
3442             if ($bootindex_hash->{d}) {
3443                 $drive->{bootindex} = $bootindex_hash->{d};
3444                 $bootindex_hash->{d} += 1;
3445             }
3446         } else {
3447             if ($bootindex_hash->{c}) {
3448                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3449                 $bootindex_hash->{c} += 1;
3450             }
3451         }
3452
3453         if($drive->{interface} eq 'virtio'){
3454            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3455         }
3456
3457         if ($drive->{interface} eq 'scsi') {
3458
3459             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3460
3461             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3462                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3463
3464             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3465             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3466
3467             my $iothread = '';
3468             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3469                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3470                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3471             } elsif ($drive->{iothread}) {
3472                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3473             }
3474
3475             my $queues = '';
3476             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3477                 $queues = ",num_queues=$drive->{queues}";
3478             }
3479
3480             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3481             $scsicontroller->{$controller}=1;
3482         }
3483
3484         if ($drive->{interface} eq 'sata') {
3485            my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3486            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3487            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3488            $ahcicontroller->{$controller}=1;
3489         }
3490
3491         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3492         push @$devices, '-drive',$drive_cmd;
3493         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3494     });
3495
3496     for (my $i = 0; $i < $MAX_NETS; $i++) {
3497          next if !$conf->{"net$i"};
3498          my $d = parse_net($conf->{"net$i"});
3499          next if !$d;
3500
3501          $use_virtio = 1 if $d->{model} eq 'virtio';
3502
3503          if ($bootindex_hash->{n}) {
3504             $d->{bootindex} = $bootindex_hash->{n};
3505             $bootindex_hash->{n} += 1;
3506          }
3507
3508          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3509          push @$devices, '-netdev', $netdevfull;
3510
3511          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3512          push @$devices, '-device', $netdevicefull;
3513     }
3514
3515     if ($conf->{ivshmem}) {
3516         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3517
3518         my $bus;
3519         if ($q35) {
3520             $bus = print_pcie_addr("ivshmem");
3521         } else {
3522             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3523         }
3524
3525         my $ivshmem_name = $ivshmem->{name} // $vmid;
3526         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3527
3528         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3529         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3530     }
3531
3532     # pci.4 is nested in pci.1
3533     $bridges->{1} = 1 if $bridges->{4};
3534
3535     if (!$q35) {
3536         # add pci bridges
3537         if (min_version($machine_version, 2, 3)) {
3538            $bridges->{1} = 1;
3539            $bridges->{2} = 1;
3540         }
3541
3542         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3543
3544     }
3545
3546     for my $k (sort {$b cmp $a} keys %$bridges) {
3547         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3548         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3549         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3550         if ($q35) {
3551             # add after -readconfig pve-q35.cfg
3552             splice @$devices, 2, 0, '-device', $devstr;
3553         } else {
3554             unshift @$devices, '-device', $devstr if $k > 0;
3555         }
3556     }
3557
3558     if (!$kvm) {
3559         push @$machineFlags, 'accel=tcg';
3560     }
3561
3562     my $machine_type_min = $machine_type;
3563     if ($add_pve_version) {
3564         $machine_type_min =~ s/\+pve\d+$//;
3565         $machine_type_min .= "+pve$required_pve_version";
3566     }
3567     push @$machineFlags, "type=${machine_type_min}";
3568
3569     push @$cmd, @$devices;
3570     push @$cmd, '-rtc', join(',', @$rtcFlags)
3571         if scalar(@$rtcFlags);
3572     push @$cmd, '-machine', join(',', @$machineFlags)
3573         if scalar(@$machineFlags);
3574     push @$cmd, '-global', join(',', @$globalFlags)
3575         if scalar(@$globalFlags);
3576
3577     if (my $vmstate = $conf->{vmstate}) {
3578         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3579         push @$vollist, $vmstate;
3580         push @$cmd, '-loadstate', $statepath;
3581         print "activating and using '$vmstate' as vmstate\n";
3582     }
3583
3584     # add custom args
3585     if ($conf->{args}) {
3586         my $aa = PVE::Tools::split_args($conf->{args});
3587         push @$cmd, @$aa;
3588     }
3589
3590     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3591 }
3592
3593 sub spice_port {
3594     my ($vmid) = @_;
3595
3596     my $res = mon_cmd($vmid, 'query-spice');
3597
3598     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3599 }
3600
3601 sub vm_devices_list {
3602     my ($vmid) = @_;
3603
3604     my $res = mon_cmd($vmid, 'query-pci');
3605     my $devices_to_check = [];
3606     my $devices = {};
3607     foreach my $pcibus (@$res) {
3608         push @$devices_to_check, @{$pcibus->{devices}},
3609     }
3610
3611     while (@$devices_to_check) {
3612         my $to_check = [];
3613         for my $d (@$devices_to_check) {
3614             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3615             next if !$d->{'pci_bridge'};
3616
3617             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3618             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3619         }
3620         $devices_to_check = $to_check;
3621     }
3622
3623     my $resblock = mon_cmd($vmid, 'query-block');
3624     foreach my $block (@$resblock) {
3625         if($block->{device} =~ m/^drive-(\S+)/){
3626                 $devices->{$1} = 1;
3627         }
3628     }
3629
3630     my $resmice = mon_cmd($vmid, 'query-mice');
3631     foreach my $mice (@$resmice) {
3632         if ($mice->{name} eq 'QEMU HID Tablet') {
3633             $devices->{tablet} = 1;
3634             last;
3635         }
3636     }
3637
3638     # for usb devices there is no query-usb
3639     # but we can iterate over the entries in
3640     # qom-list path=/machine/peripheral
3641     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3642     foreach my $per (@$resperipheral) {
3643         if ($per->{name} =~ m/^usb\d+$/) {
3644             $devices->{$per->{name}} = 1;
3645         }
3646     }
3647
3648     return $devices;
3649 }
3650
3651 sub vm_deviceplug {
3652     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3653
3654     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3655
3656     my $devices_list = vm_devices_list($vmid);
3657     return 1 if defined($devices_list->{$deviceid});
3658
3659     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3660
3661     if ($deviceid eq 'tablet') {
3662
3663         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3664
3665     } elsif ($deviceid eq 'keyboard') {
3666
3667         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3668
3669     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3670
3671         die "usb hotplug currently not reliable\n";
3672         # since we can't reliably hot unplug all added usb devices
3673         # and usb passthrough disables live migration
3674         # we disable usb hotplugging for now
3675         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3676
3677     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3678
3679         qemu_iothread_add($vmid, $deviceid, $device);
3680
3681         qemu_driveadd($storecfg, $vmid, $device);
3682         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3683
3684         qemu_deviceadd($vmid, $devicefull);
3685         eval { qemu_deviceaddverify($vmid, $deviceid); };
3686         if (my $err = $@) {
3687             eval { qemu_drivedel($vmid, $deviceid); };
3688             warn $@ if $@;
3689             die $err;
3690         }
3691
3692     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3693
3694
3695         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3696         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3697         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3698
3699         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3700
3701         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3702             qemu_iothread_add($vmid, $deviceid, $device);
3703             $devicefull .= ",iothread=iothread-$deviceid";
3704         }
3705
3706         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3707             $devicefull .= ",num_queues=$device->{queues}";
3708         }
3709
3710         qemu_deviceadd($vmid, $devicefull);
3711         qemu_deviceaddverify($vmid, $deviceid);
3712
3713     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3714
3715         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3716         qemu_driveadd($storecfg, $vmid, $device);
3717
3718         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3719         eval { qemu_deviceadd($vmid, $devicefull); };
3720         if (my $err = $@) {
3721             eval { qemu_drivedel($vmid, $deviceid); };
3722             warn $@ if $@;
3723             die $err;
3724         }
3725
3726     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3727
3728         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3729
3730         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3731         my $use_old_bios_files = undef;
3732         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3733
3734         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3735         qemu_deviceadd($vmid, $netdevicefull);
3736         eval {
3737             qemu_deviceaddverify($vmid, $deviceid);
3738             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3739         };
3740         if (my $err = $@) {
3741             eval { qemu_netdevdel($vmid, $deviceid); };
3742             warn $@ if $@;
3743             die $err;
3744         }
3745
3746     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3747
3748         my $bridgeid = $2;
3749         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3750         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3751
3752         qemu_deviceadd($vmid, $devicefull);
3753         qemu_deviceaddverify($vmid, $deviceid);
3754
3755     } else {
3756         die "can't hotplug device '$deviceid'\n";
3757     }
3758
3759     return 1;
3760 }
3761
3762 # fixme: this should raise exceptions on error!
3763 sub vm_deviceunplug {
3764     my ($vmid, $conf, $deviceid) = @_;
3765
3766     my $devices_list = vm_devices_list($vmid);
3767     return 1 if !defined($devices_list->{$deviceid});
3768
3769     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3770
3771     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3772
3773         qemu_devicedel($vmid, $deviceid);
3774
3775     } elsif ($deviceid =~ m/^usb\d+$/) {
3776
3777         die "usb hotplug currently not reliable\n";
3778         # when unplugging usb devices this way,
3779         # there may be remaining usb controllers/hubs
3780         # so we disable it for now
3781         qemu_devicedel($vmid, $deviceid);
3782         qemu_devicedelverify($vmid, $deviceid);
3783
3784     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3785
3786         qemu_devicedel($vmid, $deviceid);
3787         qemu_devicedelverify($vmid, $deviceid);
3788         qemu_drivedel($vmid, $deviceid);
3789         qemu_iothread_del($conf, $vmid, $deviceid);
3790
3791     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3792
3793         qemu_devicedel($vmid, $deviceid);
3794         qemu_devicedelverify($vmid, $deviceid);
3795         qemu_iothread_del($conf, $vmid, $deviceid);
3796
3797     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3798
3799         qemu_devicedel($vmid, $deviceid);
3800         qemu_drivedel($vmid, $deviceid);
3801         qemu_deletescsihw($conf, $vmid, $deviceid);
3802
3803     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3804
3805         qemu_devicedel($vmid, $deviceid);
3806         qemu_devicedelverify($vmid, $deviceid);
3807         qemu_netdevdel($vmid, $deviceid);
3808
3809     } else {
3810         die "can't unplug device '$deviceid'\n";
3811     }
3812
3813     return 1;
3814 }
3815
3816 sub qemu_deviceadd {
3817     my ($vmid, $devicefull) = @_;
3818
3819     $devicefull = "driver=".$devicefull;
3820     my %options =  split(/[=,]/, $devicefull);
3821
3822     mon_cmd($vmid, "device_add" , %options);
3823 }
3824
3825 sub qemu_devicedel {
3826     my ($vmid, $deviceid) = @_;
3827
3828     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3829 }
3830
3831 sub qemu_iothread_add {
3832     my($vmid, $deviceid, $device) = @_;
3833
3834     if ($device->{iothread}) {
3835         my $iothreads = vm_iothreads_list($vmid);
3836         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3837     }
3838 }
3839
3840 sub qemu_iothread_del {
3841     my($conf, $vmid, $deviceid) = @_;
3842
3843     my $confid = $deviceid;
3844     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3845         $confid = 'scsi' . $1;
3846     }
3847     my $device = parse_drive($confid, $conf->{$confid});
3848     if ($device->{iothread}) {
3849         my $iothreads = vm_iothreads_list($vmid);
3850         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3851     }
3852 }
3853
3854 sub qemu_objectadd {
3855     my($vmid, $objectid, $qomtype) = @_;
3856
3857     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3858
3859     return 1;
3860 }
3861
3862 sub qemu_objectdel {
3863     my($vmid, $objectid) = @_;
3864
3865     mon_cmd($vmid, "object-del", id => $objectid);
3866
3867     return 1;
3868 }
3869
3870 sub qemu_driveadd {
3871     my ($storecfg, $vmid, $device) = @_;
3872
3873     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3874     $drive =~ s/\\/\\\\/g;
3875     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3876
3877     # If the command succeeds qemu prints: "OK"
3878     return 1 if $ret =~ m/OK/s;
3879
3880     die "adding drive failed: $ret\n";
3881 }
3882
3883 sub qemu_drivedel {
3884     my($vmid, $deviceid) = @_;
3885
3886     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3887     $ret =~ s/^\s+//;
3888
3889     return 1 if $ret eq "";
3890
3891     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3892     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3893
3894     die "deleting drive $deviceid failed : $ret\n";
3895 }
3896
3897 sub qemu_deviceaddverify {
3898     my ($vmid, $deviceid) = @_;
3899
3900     for (my $i = 0; $i <= 5; $i++) {
3901          my $devices_list = vm_devices_list($vmid);
3902          return 1 if defined($devices_list->{$deviceid});
3903          sleep 1;
3904     }
3905
3906     die "error on hotplug device '$deviceid'\n";
3907 }
3908
3909
3910 sub qemu_devicedelverify {
3911     my ($vmid, $deviceid) = @_;
3912
3913     # need to verify that the device is correctly removed as device_del
3914     # is async and empty return is not reliable
3915
3916     for (my $i = 0; $i <= 5; $i++) {
3917          my $devices_list = vm_devices_list($vmid);
3918          return 1 if !defined($devices_list->{$deviceid});
3919          sleep 1;
3920     }
3921
3922     die "error on hot-unplugging device '$deviceid'\n";
3923 }
3924
3925 sub qemu_findorcreatescsihw {
3926     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3927
3928     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3929
3930     my $scsihwid="$controller_prefix$controller";
3931     my $devices_list = vm_devices_list($vmid);
3932
3933     if(!defined($devices_list->{$scsihwid})) {
3934         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3935     }
3936
3937     return 1;
3938 }
3939
3940 sub qemu_deletescsihw {
3941     my ($conf, $vmid, $opt) = @_;
3942
3943     my $device = parse_drive($opt, $conf->{$opt});
3944
3945     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3946         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3947         return 1;
3948     }
3949
3950     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3951
3952     my $devices_list = vm_devices_list($vmid);
3953     foreach my $opt (keys %{$devices_list}) {
3954         if (is_valid_drivename($opt)) {
3955             my $drive = parse_drive($opt, $conf->{$opt});
3956             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3957                 return 1;
3958             }
3959         }
3960     }
3961
3962     my $scsihwid="scsihw$controller";
3963
3964     vm_deviceunplug($vmid, $conf, $scsihwid);
3965
3966     return 1;
3967 }
3968
3969 sub qemu_add_pci_bridge {
3970     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3971
3972     my $bridges = {};
3973
3974     my $bridgeid;
3975
3976     print_pci_addr($device, $bridges, $arch, $machine_type);
3977
3978     while (my ($k, $v) = each %$bridges) {
3979         $bridgeid = $k;
3980     }
3981     return 1 if !defined($bridgeid) || $bridgeid < 1;
3982
3983     my $bridge = "pci.$bridgeid";
3984     my $devices_list = vm_devices_list($vmid);
3985
3986     if (!defined($devices_list->{$bridge})) {
3987         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3988     }
3989
3990     return 1;
3991 }
3992
3993 sub qemu_set_link_status {
3994     my ($vmid, $device, $up) = @_;
3995
3996     mon_cmd($vmid, "set_link", name => $device,
3997                up => $up ? JSON::true : JSON::false);
3998 }
3999
4000 sub qemu_netdevadd {
4001     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4002
4003     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4004     my %options =  split(/[=,]/, $netdev);
4005
4006     mon_cmd($vmid, "netdev_add",  %options);
4007     return 1;
4008 }
4009
4010 sub qemu_netdevdel {
4011     my ($vmid, $deviceid) = @_;
4012
4013     mon_cmd($vmid, "netdev_del", id => $deviceid);
4014 }
4015
4016 sub qemu_usb_hotplug {
4017     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4018
4019     return if !$device;
4020
4021     # remove the old one first
4022     vm_deviceunplug($vmid, $conf, $deviceid);
4023
4024     # check if xhci controller is necessary and available
4025     if ($device->{usb3}) {
4026
4027         my $devicelist = vm_devices_list($vmid);
4028
4029         if (!$devicelist->{xhci}) {
4030             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4031             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4032         }
4033     }
4034     my $d = parse_usb_device($device->{host});
4035     $d->{usb3} = $device->{usb3};
4036
4037     # add the new one
4038     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4039 }
4040
4041 sub qemu_cpu_hotplug {
4042     my ($vmid, $conf, $vcpus) = @_;
4043
4044     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4045
4046     my $sockets = 1;
4047     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4048     $sockets = $conf->{sockets} if  $conf->{sockets};
4049     my $cores = $conf->{cores} || 1;
4050     my $maxcpus = $sockets * $cores;
4051
4052     $vcpus = $maxcpus if !$vcpus;
4053
4054     die "you can't add more vcpus than maxcpus\n"
4055         if $vcpus > $maxcpus;
4056
4057     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4058
4059     if ($vcpus < $currentvcpus) {
4060
4061         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4062
4063             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4064                 qemu_devicedel($vmid, "cpu$i");
4065                 my $retry = 0;
4066                 my $currentrunningvcpus = undef;
4067                 while (1) {
4068                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4069                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4070                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4071                     $retry++;
4072                     sleep 1;
4073                 }
4074                 #update conf after each succesfull cpu unplug
4075                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4076                 PVE::QemuConfig->write_config($vmid, $conf);
4077             }
4078         } else {
4079             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4080         }
4081
4082         return;
4083     }
4084
4085     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4086     die "vcpus in running vm does not match its configuration\n"
4087         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4088
4089     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4090
4091         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4092             my $cpustr = print_cpu_device($conf, $i);
4093             qemu_deviceadd($vmid, $cpustr);
4094
4095             my $retry = 0;
4096             my $currentrunningvcpus = undef;
4097             while (1) {
4098                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4099                 last if scalar(@{$currentrunningvcpus}) == $i;
4100                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4101                 sleep 1;
4102                 $retry++;
4103             }
4104             #update conf after each succesfull cpu hotplug
4105             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4106             PVE::QemuConfig->write_config($vmid, $conf);
4107         }
4108     } else {
4109
4110         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4111             mon_cmd($vmid, "cpu-add", id => int($i));
4112         }
4113     }
4114 }
4115
4116 sub qemu_block_set_io_throttle {
4117     my ($vmid, $deviceid,
4118         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4119         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4120         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4121         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4122
4123     return if !check_running($vmid) ;
4124
4125     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4126         bps => int($bps),
4127         bps_rd => int($bps_rd),
4128         bps_wr => int($bps_wr),
4129         iops => int($iops),
4130         iops_rd => int($iops_rd),
4131         iops_wr => int($iops_wr),
4132         bps_max => int($bps_max),
4133         bps_rd_max => int($bps_rd_max),
4134         bps_wr_max => int($bps_wr_max),
4135         iops_max => int($iops_max),
4136         iops_rd_max => int($iops_rd_max),
4137         iops_wr_max => int($iops_wr_max),
4138         bps_max_length => int($bps_max_length),
4139         bps_rd_max_length => int($bps_rd_max_length),
4140         bps_wr_max_length => int($bps_wr_max_length),
4141         iops_max_length => int($iops_max_length),
4142         iops_rd_max_length => int($iops_rd_max_length),
4143         iops_wr_max_length => int($iops_wr_max_length),
4144     );
4145
4146 }
4147
4148 # old code, only used to shutdown old VM after update
4149 sub __read_avail {
4150     my ($fh, $timeout) = @_;
4151
4152     my $sel = new IO::Select;
4153     $sel->add($fh);
4154
4155     my $res = '';
4156     my $buf;
4157
4158     my @ready;
4159     while (scalar (@ready = $sel->can_read($timeout))) {
4160         my $count;
4161         if ($count = $fh->sysread($buf, 8192)) {
4162             if ($buf =~ /^(.*)\(qemu\) $/s) {
4163                 $res .= $1;
4164                 last;
4165             } else {
4166                 $res .= $buf;
4167             }
4168         } else {
4169             if (!defined($count)) {
4170                 die "$!\n";
4171             }
4172             last;
4173         }
4174     }
4175
4176     die "monitor read timeout\n" if !scalar(@ready);
4177
4178     return $res;
4179 }
4180
4181 sub qemu_block_resize {
4182     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4183
4184     my $running = check_running($vmid);
4185
4186     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4187
4188     return if !$running;
4189
4190     my $padding = (1024 - $size % 1024) % 1024;
4191     $size = $size + $padding;
4192
4193     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4194
4195 }
4196
4197 sub qemu_volume_snapshot {
4198     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4199
4200     my $running = check_running($vmid);
4201
4202     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4203         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4204     } else {
4205         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4206     }
4207 }
4208
4209 sub qemu_volume_snapshot_delete {
4210     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4211
4212     my $running = check_running($vmid);
4213
4214     if($running) {
4215
4216         $running = undef;
4217         my $conf = PVE::QemuConfig->load_config($vmid);
4218         foreach_drive($conf, sub {
4219             my ($ds, $drive) = @_;
4220             $running = 1 if $drive->{file} eq $volid;
4221         });
4222     }
4223
4224     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4225         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4226     } else {
4227         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4228     }
4229 }
4230
4231 sub set_migration_caps {
4232     my ($vmid) = @_;
4233
4234     my $cap_ref = [];
4235
4236     my $enabled_cap = {
4237         "auto-converge" => 1,
4238         "xbzrle" => 1,
4239         "x-rdma-pin-all" => 0,
4240         "zero-blocks" => 0,
4241         "compress" => 0
4242     };
4243
4244     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4245
4246     for my $supported_capability (@$supported_capabilities) {
4247         push @$cap_ref, {
4248             capability => $supported_capability->{capability},
4249             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4250         };
4251     }
4252
4253     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4254 }
4255
4256 my $fast_plug_option = {
4257     'lock' => 1,
4258     'name' => 1,
4259     'onboot' => 1,
4260     'shares' => 1,
4261     'startup' => 1,
4262     'description' => 1,
4263     'protection' => 1,
4264     'vmstatestorage' => 1,
4265     'hookscript' => 1,
4266     'tags' => 1,
4267 };
4268
4269 # hotplug changes in [PENDING]
4270 # $selection hash can be used to only apply specified options, for
4271 # example: { cores => 1 } (only apply changed 'cores')
4272 # $errors ref is used to return error messages
4273 sub vmconfig_hotplug_pending {
4274     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4275
4276     my $defaults = load_defaults();
4277     my $arch = get_vm_arch($conf);
4278     my $machine_type = get_vm_machine($conf, undef, $arch);
4279
4280     # commit values which do not have any impact on running VM first
4281     # Note: those option cannot raise errors, we we do not care about
4282     # $selection and always apply them.
4283
4284     my $add_error = sub {
4285         my ($opt, $msg) = @_;
4286         $errors->{$opt} = "hotplug problem - $msg";
4287     };
4288
4289     my $changes = 0;
4290     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4291         if ($fast_plug_option->{$opt}) {
4292             $conf->{$opt} = $conf->{pending}->{$opt};
4293             delete $conf->{pending}->{$opt};
4294             $changes = 1;
4295         }
4296     }
4297
4298     if ($changes) {
4299         PVE::QemuConfig->write_config($vmid, $conf);
4300     }
4301
4302     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4303
4304     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4305     foreach my $opt (sort keys %$pending_delete_hash) {
4306         next if $selection && !$selection->{$opt};
4307         my $force = $pending_delete_hash->{$opt}->{force};
4308         eval {
4309             if ($opt eq 'hotplug') {
4310                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4311             } elsif ($opt eq 'tablet') {
4312                 die "skip\n" if !$hotplug_features->{usb};
4313                 if ($defaults->{tablet}) {
4314                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4315                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4316                         if $arch eq 'aarch64';
4317                 } else {
4318                     vm_deviceunplug($vmid, $conf, 'tablet');
4319                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4320                 }
4321             } elsif ($opt =~ m/^usb\d+/) {
4322                 die "skip\n";
4323                 # since we cannot reliably hot unplug usb devices
4324                 # we are disabling it
4325                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4326                 vm_deviceunplug($vmid, $conf, $opt);
4327             } elsif ($opt eq 'vcpus') {
4328                 die "skip\n" if !$hotplug_features->{cpu};
4329                 qemu_cpu_hotplug($vmid, $conf, undef);
4330             } elsif ($opt eq 'balloon') {
4331                 # enable balloon device is not hotpluggable
4332                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4333                 # here we reset the ballooning value to memory
4334                 my $balloon = $conf->{memory} || $defaults->{memory};
4335                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4336             } elsif ($fast_plug_option->{$opt}) {
4337                 # do nothing
4338             } elsif ($opt =~ m/^net(\d+)$/) {
4339                 die "skip\n" if !$hotplug_features->{network};
4340                 vm_deviceunplug($vmid, $conf, $opt);
4341             } elsif (is_valid_drivename($opt)) {
4342                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4343                 vm_deviceunplug($vmid, $conf, $opt);
4344                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4345             } elsif ($opt =~ m/^memory$/) {
4346                 die "skip\n" if !$hotplug_features->{memory};
4347                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4348             } elsif ($opt eq 'cpuunits') {
4349                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4350             } elsif ($opt eq 'cpulimit') {
4351                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4352             } else {
4353                 die "skip\n";
4354             }
4355         };
4356         if (my $err = $@) {
4357             &$add_error($opt, $err) if $err ne "skip\n";
4358         } else {
4359             delete $conf->{$opt};
4360             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4361         }
4362     }
4363
4364     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4365     $apply_pending_cloudinit = sub {
4366         return if $apply_pending_cloudinit_done; # once is enough
4367         $apply_pending_cloudinit_done = 1; # once is enough
4368
4369         my ($key, $value) = @_;
4370
4371         my @cloudinit_opts = keys %$confdesc_cloudinit;
4372         foreach my $opt (keys %{$conf->{pending}}) {
4373             next if !grep { $_ eq $opt } @cloudinit_opts;
4374             $conf->{$opt} = delete $conf->{pending}->{$opt};
4375         }
4376
4377         my $new_conf = { %$conf };
4378         $new_conf->{$key} = $value;
4379         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4380     };
4381
4382     foreach my $opt (keys %{$conf->{pending}}) {
4383         next if $selection && !$selection->{$opt};
4384         my $value = $conf->{pending}->{$opt};
4385         eval {
4386             if ($opt eq 'hotplug') {
4387                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4388             } elsif ($opt eq 'tablet') {
4389                 die "skip\n" if !$hotplug_features->{usb};
4390                 if ($value == 1) {
4391                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4392                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4393                         if $arch eq 'aarch64';
4394                 } elsif ($value == 0) {
4395                     vm_deviceunplug($vmid, $conf, 'tablet');
4396                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4397                 }
4398             } elsif ($opt =~ m/^usb\d+$/) {
4399                 die "skip\n";
4400                 # since we cannot reliably hot unplug usb devices
4401                 # we are disabling it
4402                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4403                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4404                 die "skip\n" if !$d;
4405                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4406             } elsif ($opt eq 'vcpus') {
4407                 die "skip\n" if !$hotplug_features->{cpu};
4408                 qemu_cpu_hotplug($vmid, $conf, $value);
4409             } elsif ($opt eq 'balloon') {
4410                 # enable/disable balloning device is not hotpluggable
4411                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4412                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4413                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4414
4415                 # allow manual ballooning if shares is set to zero
4416                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4417                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4418                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4419                 }
4420             } elsif ($opt =~ m/^net(\d+)$/) {
4421                 # some changes can be done without hotplug
4422                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4423                                     $vmid, $opt, $value, $arch, $machine_type);
4424             } elsif (is_valid_drivename($opt)) {
4425                 die "skip\n" if $opt eq 'efidisk0';
4426                 # some changes can be done without hotplug
4427                 my $drive = parse_drive($opt, $value);
4428                 if (drive_is_cloudinit($drive)) {
4429                     &$apply_pending_cloudinit($opt, $value);
4430                 }
4431                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4432                                      $vmid, $opt, $value, $arch, $machine_type);
4433             } elsif ($opt =~ m/^memory$/) { #dimms
4434                 die "skip\n" if !$hotplug_features->{memory};
4435                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4436             } elsif ($opt eq 'cpuunits') {
4437                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4438             } elsif ($opt eq 'cpulimit') {
4439                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4440                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4441             } else {
4442                 die "skip\n";  # skip non-hot-pluggable options
4443             }
4444         };
4445         if (my $err = $@) {
4446             &$add_error($opt, $err) if $err ne "skip\n";
4447         } else {
4448             $conf->{$opt} = $value;
4449             delete $conf->{pending}->{$opt};
4450         }
4451     }
4452
4453     PVE::QemuConfig->write_config($vmid, $conf);
4454 }
4455
4456 sub try_deallocate_drive {
4457     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4458
4459     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4460         my $volid = $drive->{file};
4461         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4462             my $sid = PVE::Storage::parse_volume_id($volid);
4463             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4464
4465             # check if the disk is really unused
4466             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4467                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4468             PVE::Storage::vdisk_free($storecfg, $volid);
4469             return 1;
4470         } else {
4471             # If vm is not owner of this disk remove from config
4472             return 1;
4473         }
4474     }
4475
4476     return undef;
4477 }
4478
4479 sub vmconfig_delete_or_detach_drive {
4480     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4481
4482     my $drive = parse_drive($opt, $conf->{$opt});
4483
4484     my $rpcenv = PVE::RPCEnvironment::get();
4485     my $authuser = $rpcenv->get_user();
4486
4487     if ($force) {
4488         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4489         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4490     } else {
4491         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4492     }
4493 }
4494
4495
4496
4497 sub vmconfig_apply_pending {
4498     my ($vmid, $conf, $storecfg, $errors) = @_;
4499
4500     my $add_apply_error = sub {
4501         my ($opt, $msg) = @_;
4502         my $err_msg = "unable to apply pending change $opt : $msg";
4503         $errors->{$opt} = $err_msg;
4504         warn $err_msg;
4505     };
4506
4507     # cold plug
4508
4509     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4510     foreach my $opt (sort keys %$pending_delete_hash) {
4511         my $force = $pending_delete_hash->{$opt}->{force};
4512         eval {
4513             if ($opt =~ m/^unused/) {
4514                 die "internal error";
4515             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4516                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4517             }
4518         };
4519         if (my $err = $@) {
4520             $add_apply_error->($opt, $err);
4521         } else {
4522             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4523             delete $conf->{$opt};
4524         }
4525     }
4526
4527     PVE::QemuConfig->cleanup_pending($conf);
4528
4529     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4530         next if $opt eq 'delete'; # just to be sure
4531         eval {
4532             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4533                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4534             }
4535         };
4536         if (my $err = $@) {
4537             $add_apply_error->($opt, $err);
4538         } else {
4539             $conf->{$opt} = delete $conf->{pending}->{$opt};
4540         }
4541     }
4542
4543     # write all changes at once to avoid unnecessary i/o
4544     PVE::QemuConfig->write_config($vmid, $conf);
4545 }
4546
4547 sub vmconfig_update_net {
4548     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4549
4550     my $newnet = parse_net($value);
4551
4552     if ($conf->{$opt}) {
4553         my $oldnet = parse_net($conf->{$opt});
4554
4555         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4556             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4557             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4558             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4559
4560             # for non online change, we try to hot-unplug
4561             die "skip\n" if !$hotplug;
4562             vm_deviceunplug($vmid, $conf, $opt);
4563         } else {
4564
4565             die "internal error" if $opt !~ m/net(\d+)/;
4566             my $iface = "tap${vmid}i$1";
4567
4568             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4569                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4570                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4571                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4572                 PVE::Network::tap_unplug($iface);
4573
4574                 if ($have_sdn) {
4575                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4576                 } else {
4577                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4578                 }
4579             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4580                 # Rate can be applied on its own but any change above needs to
4581                 # include the rate in tap_plug since OVS resets everything.
4582                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4583             }
4584
4585             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4586                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4587             }
4588
4589             return 1;
4590         }
4591     }
4592
4593     if ($hotplug) {
4594         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4595     } else {
4596         die "skip\n";
4597     }
4598 }
4599
4600 sub vmconfig_update_disk {
4601     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4602
4603     my $drive = parse_drive($opt, $value);
4604
4605     if ($conf->{$opt}) {
4606
4607         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4608
4609             my $media = $drive->{media} || 'disk';
4610             my $oldmedia = $old_drive->{media} || 'disk';
4611             die "unable to change media type\n" if $media ne $oldmedia;
4612
4613             if (!drive_is_cdrom($old_drive)) {
4614
4615                 if ($drive->{file} ne $old_drive->{file}) {
4616
4617                     die "skip\n" if !$hotplug;
4618
4619                     # unplug and register as unused
4620                     vm_deviceunplug($vmid, $conf, $opt);
4621                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4622
4623                 } else {
4624                     # update existing disk
4625
4626                     # skip non hotpluggable value
4627                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4628                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4629                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4630                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4631                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4632                         die "skip\n";
4633                     }
4634
4635                     # apply throttle
4636                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4637                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4638                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4639                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4640                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4641                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4642                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4643                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4644                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4645                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4646                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4647                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4648                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4649                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4650                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4651                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4652                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4653                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4654
4655                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4656                                                    ($drive->{mbps} || 0)*1024*1024,
4657                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4658                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4659                                                    $drive->{iops} || 0,
4660                                                    $drive->{iops_rd} || 0,
4661                                                    $drive->{iops_wr} || 0,
4662                                                    ($drive->{mbps_max} || 0)*1024*1024,
4663                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4664                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4665                                                    $drive->{iops_max} || 0,
4666                                                    $drive->{iops_rd_max} || 0,
4667                                                    $drive->{iops_wr_max} || 0,
4668                                                    $drive->{bps_max_length} || 1,
4669                                                    $drive->{bps_rd_max_length} || 1,
4670                                                    $drive->{bps_wr_max_length} || 1,
4671                                                    $drive->{iops_max_length} || 1,
4672                                                    $drive->{iops_rd_max_length} || 1,
4673                                                    $drive->{iops_wr_max_length} || 1);
4674
4675                     }
4676
4677                     return 1;
4678                 }
4679
4680             } else { # cdrom
4681
4682                 if ($drive->{file} eq 'none') {
4683                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4684                     if (drive_is_cloudinit($old_drive)) {
4685                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4686                     }
4687                 } else {
4688                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4689
4690                     # force eject if locked
4691                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4692
4693                     if ($path) {
4694                         mon_cmd($vmid, "blockdev-change-medium",
4695                             id => "$opt", filename => "$path");
4696                     }
4697                 }
4698
4699                 return 1;
4700             }
4701         }
4702     }
4703
4704     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4705     # hotplug new disks
4706     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4707     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4708 }
4709
4710 sub vm_start {
4711     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4712         $forcemachine, $spice_ticket, $migration_network, $migration_type,
4713         $targetstorage, $timeout, $nbd_protocol_version) = @_;
4714
4715     PVE::QemuConfig->lock_config($vmid, sub {
4716         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4717
4718         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4719
4720         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
4721
4722         PVE::QemuConfig->check_lock($conf)
4723             if !($skiplock || $is_suspended);
4724
4725         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4726
4727         # clean up leftover reboot request files
4728         eval { clear_reboot_request($vmid); };
4729         warn $@ if $@;
4730
4731         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4732             vmconfig_apply_pending($vmid, $conf, $storecfg);
4733             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4734         }
4735
4736         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4737
4738         my $defaults = load_defaults();
4739
4740         # set environment variable useful inside network script
4741         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4742
4743         my $local_volumes = {};
4744
4745         if ($targetstorage) {
4746             foreach_drive($conf, sub {
4747                 my ($ds, $drive) = @_;
4748
4749                 return if drive_is_cdrom($drive);
4750
4751                 my $volid = $drive->{file};
4752
4753                 return if !$volid;
4754
4755                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4756
4757                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4758                 return if $scfg->{shared};
4759                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4760             });
4761
4762             my $format = undef;
4763
4764             foreach my $opt (sort keys %$local_volumes) {
4765
4766                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4767                 my $drive = parse_drive($opt, $conf->{$opt});
4768
4769                 # If a remote storage is specified and the format of the original
4770                 # volume is not available there, fall back to the default format.
4771                 # Otherwise use the same format as the original.
4772                 if ($targetstorage && $targetstorage ne "1") {
4773                     $storeid = $targetstorage;
4774                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4775                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4776                     my $fileFormat = qemu_img_format($scfg, $volname);
4777                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4778                 } else {
4779                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4780                     $format = qemu_img_format($scfg, $volname);
4781                 }
4782
4783                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4784                 my $newdrive = $drive;
4785                 $newdrive->{format} = $format;
4786                 $newdrive->{file} = $newvolid;
4787                 my $drivestr = print_drive($newdrive);
4788                 $local_volumes->{$opt} = $drivestr;
4789                 #pass drive to conf for command line
4790                 $conf->{$opt} = $drivestr;
4791             }
4792         }
4793
4794         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4795
4796         if ($is_suspended) {
4797             # enforce machine type on suspended vm to ensure HW compatibility
4798             $forcemachine = $conf->{runningmachine};
4799             print "Resuming suspended VM\n";
4800         }
4801
4802         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4803
4804         my $migration_ip;
4805         my $get_migration_ip = sub {
4806             my ($cidr, $nodename) = @_;
4807
4808             return $migration_ip if defined($migration_ip);
4809
4810             if (!defined($cidr)) {
4811                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4812                 $cidr = $dc_conf->{migration}->{network};
4813             }
4814
4815             if (defined($cidr)) {
4816                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4817
4818                 die "could not get IP: no address configured on local " .
4819                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
4820
4821                 die "could not get IP: multiple addresses configured on local " .
4822                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
4823
4824                 $migration_ip = @$ips[0];
4825             }
4826
4827             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4828                 if !defined($migration_ip);
4829
4830             return $migration_ip;
4831         };
4832
4833         my $migrate_uri;
4834         if ($statefile) {
4835             if ($statefile eq 'tcp') {
4836                 my $localip = "localhost";
4837                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4838                 my $nodename = nodename();
4839
4840                 if (!defined($migration_type)) {
4841                     if (defined($datacenterconf->{migration}->{type})) {
4842                         $migration_type = $datacenterconf->{migration}->{type};
4843                     } else {
4844                         $migration_type = 'secure';
4845                     }
4846                 }
4847
4848                 if ($migration_type eq 'insecure') {
4849                     $localip = $get_migration_ip->($migration_network, $nodename);
4850                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4851                 }
4852
4853                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4854                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4855                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4856                 push @$cmd, '-incoming', $migrate_uri;
4857                 push @$cmd, '-S';
4858
4859             } elsif ($statefile eq 'unix') {
4860                 # should be default for secure migrations as a ssh TCP forward
4861                 # tunnel is not deterministic reliable ready and fails regurarly
4862                 # to set up in time, so use UNIX socket forwards
4863                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4864                 unlink $socket_addr;
4865
4866                 $migrate_uri = "unix:$socket_addr";
4867
4868                 push @$cmd, '-incoming', $migrate_uri;
4869                 push @$cmd, '-S';
4870
4871             } elsif (-e $statefile) {
4872                 push @$cmd, '-loadstate', $statefile;
4873             } else {
4874                 my $statepath = PVE::Storage::path($storecfg, $statefile);
4875                 push @$vollist, $statefile;
4876                 push @$cmd, '-loadstate', $statepath;
4877             }
4878         } elsif ($paused) {
4879             push @$cmd, '-S';
4880         }
4881
4882         # host pci devices
4883         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4884           my $d = parse_hostpci($conf->{"hostpci$i"});
4885           next if !$d;
4886           my $pcidevices = $d->{pciid};
4887           foreach my $pcidevice (@$pcidevices) {
4888                 my $pciid = $pcidevice->{id};
4889
4890                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4891                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4892                 die "no pci device info for device '$pciid'\n" if !$info;
4893
4894                 if ($d->{mdev}) {
4895                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4896                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4897                 } else {
4898                     die "can't unbind/bind pci group to vfio '$pciid'\n"
4899                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4900                     die "can't reset pci device '$pciid'\n"
4901                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4902                 }
4903           }
4904         }
4905
4906         PVE::Storage::activate_volumes($storecfg, $vollist);
4907
4908         eval {
4909             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4910                 outfunc => sub {}, errfunc => sub {});
4911         };
4912         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4913         # timeout should be more than enough here...
4914         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4915
4916         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4917                                                   : $defaults->{cpuunits};
4918
4919         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
4920         my %run_params = (
4921             timeout => $statefile ? undef : $start_timeout,
4922             umask => 0077,
4923             noerr => 1,
4924         );
4925
4926         # when migrating, prefix QEMU output so other side can pick up any
4927         # errors that might occur and show the user
4928         if ($migratedfrom) {
4929             $run_params{quiet} = 1;
4930             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4931         }
4932
4933         my %properties = (
4934             Slice => 'qemu.slice',
4935             KillMode => 'none',
4936             CPUShares => $cpuunits
4937         );
4938
4939         if (my $cpulimit = $conf->{cpulimit}) {
4940             $properties{CPUQuota} = int($cpulimit * 100);
4941         }
4942         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4943
4944         my $run_qemu = sub {
4945             PVE::Tools::run_fork sub {
4946                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4947
4948                 my $exitcode = run_command($cmd, %run_params);
4949                 die "QEMU exited with code $exitcode\n" if $exitcode;
4950             };
4951         };
4952
4953         if ($conf->{hugepages}) {
4954
4955             my $code = sub {
4956                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4957                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4958
4959                 PVE::QemuServer::Memory::hugepages_mount();
4960                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4961
4962                 eval { $run_qemu->() };
4963                 if (my $err = $@) {
4964                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4965                     die $err;
4966                 }
4967
4968                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4969             };
4970             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4971
4972         } else {
4973             eval { $run_qemu->() };
4974         }
4975
4976         if (my $err = $@) {
4977             # deactivate volumes if start fails
4978             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4979             die "start failed: $err";
4980         }
4981
4982         print "migration listens on $migrate_uri\n" if $migrate_uri;
4983
4984         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4985             eval { mon_cmd($vmid, "cont"); };
4986             warn $@ if $@;
4987         }
4988
4989         #start nbd server for storage migration
4990         if ($targetstorage) {
4991             $nbd_protocol_version //= 0;
4992
4993             my $migrate_storage_uri;
4994             # nbd_protocol_version > 0 for unix socket support
4995             if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
4996                 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
4997                 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
4998                 $migrate_storage_uri = "nbd:unix:$socket_path";
4999             } else {
5000                 my $nodename = nodename();
5001                 my $localip = $get_migration_ip->($migration_network, $nodename);
5002                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5003                 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5004
5005                 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5006                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5007                 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5008             }
5009
5010             foreach my $opt (sort keys %$local_volumes) {
5011                 my $drivestr = $local_volumes->{$opt};
5012                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5013                 print "storage migration listens on $migrate_storage_uri:exportname=drive-$opt volume:$drivestr\n";
5014             }
5015         }
5016
5017         if ($migratedfrom) {
5018             eval {
5019                 set_migration_caps($vmid);
5020             };
5021             warn $@ if $@;
5022
5023             if ($spice_port) {
5024                 print "spice listens on port $spice_port\n";
5025                 if ($spice_ticket) {
5026                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5027                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5028                 }
5029             }
5030
5031         } else {
5032             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5033                 if !$statefile && $conf->{balloon};
5034
5035             foreach my $opt (keys %$conf) {
5036                 next if $opt !~  m/^net\d+$/;
5037                 my $nicconf = parse_net($conf->{$opt});
5038                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5039             }
5040         }
5041
5042         mon_cmd($vmid, 'qom-set',
5043                     path => "machine/peripheral/balloon0",
5044                     property => "guest-stats-polling-interval",
5045                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5046
5047         if ($is_suspended) {
5048             print "Resumed VM, removing state\n";
5049             if (my $vmstate = $conf->{vmstate}) {
5050                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5051                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5052             }
5053             delete $conf->@{qw(lock vmstate runningmachine)};
5054             PVE::QemuConfig->write_config($vmid, $conf);
5055         }
5056
5057         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5058     });
5059 }
5060
5061 sub vm_commandline {
5062     my ($storecfg, $vmid, $snapname) = @_;
5063
5064     my $conf = PVE::QemuConfig->load_config($vmid);
5065     my $forcemachine;
5066
5067     if ($snapname) {
5068         my $snapshot = $conf->{snapshots}->{$snapname};
5069         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5070
5071         # check for a 'runningmachine' in snapshot
5072         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5073
5074         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5075
5076         $conf = $snapshot;
5077     }
5078
5079     my $defaults = load_defaults();
5080
5081     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5082
5083     return PVE::Tools::cmd2string($cmd);
5084 }
5085
5086 sub vm_reset {
5087     my ($vmid, $skiplock) = @_;
5088
5089     PVE::QemuConfig->lock_config($vmid, sub {
5090
5091         my $conf = PVE::QemuConfig->load_config($vmid);
5092
5093         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5094
5095         mon_cmd($vmid, "system_reset");
5096     });
5097 }
5098
5099 sub get_vm_volumes {
5100     my ($conf) = @_;
5101
5102     my $vollist = [];
5103     foreach_volid($conf, sub {
5104         my ($volid, $attr) = @_;
5105
5106         return if $volid =~ m|^/|;
5107
5108         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5109         return if !$sid;
5110
5111         push @$vollist, $volid;
5112     });
5113
5114     return $vollist;
5115 }
5116
5117 sub vm_stop_cleanup {
5118     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5119
5120     eval {
5121
5122         if (!$keepActive) {
5123             my $vollist = get_vm_volumes($conf);
5124             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5125         }
5126
5127         foreach my $ext (qw(mon qmp pid vnc qga)) {
5128             unlink "/var/run/qemu-server/${vmid}.$ext";
5129         }
5130
5131         if ($conf->{ivshmem}) {
5132             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5133             # just delete it for now, VMs which have this already open do not
5134             # are affected, but new VMs will get a separated one. If this
5135             # becomes an issue we either add some sort of ref-counting or just
5136             # add a "don't delete on stop" flag to the ivshmem format.
5137             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5138         }
5139
5140         foreach my $key (keys %$conf) {
5141             next if $key !~ m/^hostpci(\d+)$/;
5142             my $hostpciindex = $1;
5143             my $d = parse_hostpci($conf->{$key});
5144             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5145
5146             foreach my $pci (@{$d->{pciid}}) {
5147                 my $pciid = $pci->{id};
5148                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5149             }
5150         }
5151
5152         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5153     };
5154     warn $@ if $@; # avoid errors - just warn
5155 }
5156
5157 # call only in locked context
5158 sub _do_vm_stop {
5159     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5160
5161     my $pid = check_running($vmid, $nocheck);
5162     return if !$pid;
5163
5164     my $conf;
5165     if (!$nocheck) {
5166         $conf = PVE::QemuConfig->load_config($vmid);
5167         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5168         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5169             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5170             $timeout = $opts->{down} if $opts->{down};
5171         }
5172         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5173     }
5174
5175     eval {
5176         if ($shutdown) {
5177             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5178                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5179             } else {
5180                 mon_cmd($vmid, "system_powerdown");
5181             }
5182         } else {
5183             mon_cmd($vmid, "quit");
5184         }
5185     };
5186     my $err = $@;
5187
5188     if (!$err) {
5189         $timeout = 60 if !defined($timeout);
5190
5191         my $count = 0;
5192         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5193             $count++;
5194             sleep 1;
5195         }
5196
5197         if ($count >= $timeout) {
5198             if ($force) {
5199                 warn "VM still running - terminating now with SIGTERM\n";
5200                 kill 15, $pid;
5201             } else {
5202                 die "VM quit/powerdown failed - got timeout\n";
5203             }
5204         } else {
5205             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5206             return;
5207         }
5208     } else {
5209         if ($force) {
5210             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5211             kill 15, $pid;
5212         } else {
5213             die "VM quit/powerdown failed\n";
5214         }
5215     }
5216
5217     # wait again
5218     $timeout = 10;
5219
5220     my $count = 0;
5221     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5222         $count++;
5223         sleep 1;
5224     }
5225
5226     if ($count >= $timeout) {
5227         warn "VM still running - terminating now with SIGKILL\n";
5228         kill 9, $pid;
5229         sleep 1;
5230     }
5231
5232     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5233 }
5234
5235 # Note: use $nocheck to skip tests if VM configuration file exists.
5236 # We need that when migration VMs to other nodes (files already moved)
5237 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5238 sub vm_stop {
5239     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5240
5241     $force = 1 if !defined($force) && !$shutdown;
5242
5243     if ($migratedfrom){
5244         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5245         kill 15, $pid if $pid;
5246         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5247         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5248         return;
5249     }
5250
5251     PVE::QemuConfig->lock_config($vmid, sub {
5252         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5253    });
5254 }
5255
5256 sub vm_reboot {
5257     my ($vmid, $timeout) = @_;
5258
5259     PVE::QemuConfig->lock_config($vmid, sub {
5260         eval {
5261
5262             # only reboot if running, as qmeventd starts it again on a stop event
5263             return if !check_running($vmid);
5264
5265             create_reboot_request($vmid);
5266
5267             my $storecfg = PVE::Storage::config();
5268             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5269
5270         };
5271         if (my $err = $@) {
5272             # avoid that the next normal shutdown will be confused for a reboot
5273             clear_reboot_request($vmid);
5274             die $err;
5275         }
5276    });
5277 }
5278
5279 # note: if using the statestorage parameter, the caller has to check privileges
5280 sub vm_suspend {
5281     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5282
5283     my $conf;
5284     my $path;
5285     my $storecfg;
5286     my $vmstate;
5287
5288     PVE::QemuConfig->lock_config($vmid, sub {
5289
5290         $conf = PVE::QemuConfig->load_config($vmid);
5291
5292         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5293         PVE::QemuConfig->check_lock($conf)
5294             if !($skiplock || $is_backing_up);
5295
5296         die "cannot suspend to disk during backup\n"
5297             if $is_backing_up && $includestate;
5298
5299         if ($includestate) {
5300             $conf->{lock} = 'suspending';
5301             my $date = strftime("%Y-%m-%d", localtime(time()));
5302             $storecfg = PVE::Storage::config();
5303             if (!$statestorage) {
5304                 $statestorage = find_vmstate_storage($conf, $storecfg);
5305                 # check permissions for the storage
5306                 my $rpcenv = PVE::RPCEnvironment::get();
5307                 if ($rpcenv->{type} ne 'cli') {
5308                     my $authuser = $rpcenv->get_user();
5309                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5310                 }
5311             }
5312
5313
5314             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5315             $path = PVE::Storage::path($storecfg, $vmstate);
5316             PVE::QemuConfig->write_config($vmid, $conf);
5317         } else {
5318             mon_cmd($vmid, "stop");
5319         }
5320     });
5321
5322     if ($includestate) {
5323         # save vm state
5324         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5325
5326         eval {
5327             mon_cmd($vmid, "savevm-start", statefile => $path);
5328             for(;;) {
5329                 my $state = mon_cmd($vmid, "query-savevm");
5330                 if (!$state->{status}) {
5331                     die "savevm not active\n";
5332                 } elsif ($state->{status} eq 'active') {
5333                     sleep(1);
5334                     next;
5335                 } elsif ($state->{status} eq 'completed') {
5336                     print "State saved, quitting\n";
5337                     last;
5338                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5339                     die "query-savevm failed with error '$state->{error}'\n"
5340                 } else {
5341                     die "query-savevm returned status '$state->{status}'\n";
5342                 }
5343             }
5344         };
5345         my $err = $@;
5346
5347         PVE::QemuConfig->lock_config($vmid, sub {
5348             $conf = PVE::QemuConfig->load_config($vmid);
5349             if ($err) {
5350                 # cleanup, but leave suspending lock, to indicate something went wrong
5351                 eval {
5352                     mon_cmd($vmid, "savevm-end");
5353                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5354                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5355                     delete $conf->@{qw(vmstate runningmachine)};
5356                     PVE::QemuConfig->write_config($vmid, $conf);
5357                 };
5358                 warn $@ if $@;
5359                 die $err;
5360             }
5361
5362             die "lock changed unexpectedly\n"
5363                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5364
5365             mon_cmd($vmid, "quit");
5366             $conf->{lock} = 'suspended';
5367             PVE::QemuConfig->write_config($vmid, $conf);
5368         });
5369     }
5370 }
5371
5372 sub vm_resume {
5373     my ($vmid, $skiplock, $nocheck) = @_;
5374
5375     PVE::QemuConfig->lock_config($vmid, sub {
5376         my $res = mon_cmd($vmid, 'query-status');
5377         my $resume_cmd = 'cont';
5378
5379         if ($res->{status} && $res->{status} eq 'suspended') {
5380             $resume_cmd = 'system_wakeup';
5381         }
5382
5383         if (!$nocheck) {
5384
5385             my $conf = PVE::QemuConfig->load_config($vmid);
5386
5387             PVE::QemuConfig->check_lock($conf)
5388                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5389         }
5390
5391         mon_cmd($vmid, $resume_cmd);
5392     });
5393 }
5394
5395 sub vm_sendkey {
5396     my ($vmid, $skiplock, $key) = @_;
5397
5398     PVE::QemuConfig->lock_config($vmid, sub {
5399
5400         my $conf = PVE::QemuConfig->load_config($vmid);
5401
5402         # there is no qmp command, so we use the human monitor command
5403         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5404         die $res if $res ne '';
5405     });
5406 }
5407
5408 # vzdump restore implementaion
5409
5410 sub tar_archive_read_firstfile {
5411     my $archive = shift;
5412
5413     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5414
5415     # try to detect archive type first
5416     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5417         die "unable to open file '$archive'\n";
5418     my $firstfile = <$fh>;
5419     kill 15, $pid;
5420     close $fh;
5421
5422     die "ERROR: archive contaions no data\n" if !$firstfile;
5423     chomp $firstfile;
5424
5425     return $firstfile;
5426 }
5427
5428 sub tar_restore_cleanup {
5429     my ($storecfg, $statfile) = @_;
5430
5431     print STDERR "starting cleanup\n";
5432
5433     if (my $fd = IO::File->new($statfile, "r")) {
5434         while (defined(my $line = <$fd>)) {
5435             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5436                 my $volid = $2;
5437                 eval {
5438                     if ($volid =~ m|^/|) {
5439                         unlink $volid || die 'unlink failed\n';
5440                     } else {
5441                         PVE::Storage::vdisk_free($storecfg, $volid);
5442                     }
5443                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5444                 };
5445                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5446             } else {
5447                 print STDERR "unable to parse line in statfile - $line";
5448             }
5449         }
5450         $fd->close();
5451     }
5452 }
5453
5454 sub restore_file_archive {
5455     my ($archive, $vmid, $user, $opts) = @_;
5456
5457     my $format = $opts->{format};
5458     my $comp;
5459
5460     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5461         $format = 'tar' if !$format;
5462         $comp = 'gzip';
5463     } elsif ($archive =~ m/\.tar$/) {
5464         $format = 'tar' if !$format;
5465     } elsif ($archive =~ m/.tar.lzo$/) {
5466         $format = 'tar' if !$format;
5467         $comp = 'lzop';
5468     } elsif ($archive =~ m/\.vma$/) {
5469         $format = 'vma' if !$format;
5470     } elsif ($archive =~ m/\.vma\.gz$/) {
5471         $format = 'vma' if !$format;
5472         $comp = 'gzip';
5473     } elsif ($archive =~ m/\.vma\.lzo$/) {
5474         $format = 'vma' if !$format;
5475         $comp = 'lzop';
5476     } else {
5477         $format = 'vma' if !$format; # default
5478     }
5479
5480     # try to detect archive format
5481     if ($format eq 'tar') {
5482         return restore_tar_archive($archive, $vmid, $user, $opts);
5483     } else {
5484         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5485     }
5486 }
5487
5488 # hepler to remove disks that will not be used after restore
5489 my $restore_cleanup_oldconf = sub {
5490     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5491
5492     foreach_drive($oldconf, sub {
5493         my ($ds, $drive) = @_;
5494
5495         return if drive_is_cdrom($drive, 1);
5496
5497         my $volid = $drive->{file};
5498         return if !$volid || $volid =~ m|^/|;
5499
5500         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5501         return if !$path || !$owner || ($owner != $vmid);
5502
5503         # Note: only delete disk we want to restore
5504         # other volumes will become unused
5505         if ($virtdev_hash->{$ds}) {
5506             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5507             if (my $err = $@) {
5508                 warn $err;
5509             }
5510         }
5511     });
5512
5513     # delete vmstate files, after the restore we have no snapshots anymore
5514     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5515         my $snap = $oldconf->{snapshots}->{$snapname};
5516         if ($snap->{vmstate}) {
5517             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5518             if (my $err = $@) {
5519                 warn $err;
5520             }
5521         }
5522     }
5523 };
5524
5525 # Helper to parse vzdump backup device hints
5526 #
5527 # $rpcenv: Environment, used to ckeck storage permissions
5528 # $user: User ID, to check storage permissions
5529 # $storecfg: Storage configuration
5530 # $fh: the file handle for reading the configuration
5531 # $devinfo: should contain device sizes for all backu-up'ed devices
5532 # $options: backup options (pool, default storage)
5533 #
5534 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5535 my $parse_backup_hints = sub {
5536     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5537
5538     my $virtdev_hash = {};
5539
5540     while (defined(my $line = <$fh>)) {
5541         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5542             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5543             die "archive does not contain data for drive '$virtdev'\n"
5544                 if !$devinfo->{$devname};
5545
5546             if (defined($options->{storage})) {
5547                 $storeid = $options->{storage} || 'local';
5548             } elsif (!$storeid) {
5549                 $storeid = 'local';
5550             }
5551             $format = 'raw' if !$format;
5552             $devinfo->{$devname}->{devname} = $devname;
5553             $devinfo->{$devname}->{virtdev} = $virtdev;
5554             $devinfo->{$devname}->{format} = $format;
5555             $devinfo->{$devname}->{storeid} = $storeid;
5556
5557             # check permission on storage
5558             my $pool = $options->{pool}; # todo: do we need that?
5559             if ($user ne 'root@pam') {
5560                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5561             }
5562
5563             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5564         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5565             my $virtdev = $1;
5566             my $drive = parse_drive($virtdev, $2);
5567             if (drive_is_cloudinit($drive)) {
5568                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5569                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5570                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5571
5572                 $virtdev_hash->{$virtdev} = {
5573                     format => $format,
5574                     storeid => $options->{storage} // $storeid,
5575                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5576                     is_cloudinit => 1,
5577                 };
5578             }
5579         }
5580     }
5581
5582     return $virtdev_hash;
5583 };
5584
5585 # Helper to allocate and activate all volumes required for a restore
5586 #
5587 # $storecfg: Storage configuration
5588 # $virtdev_hash: as returned by parse_backup_hints()
5589 #
5590 # Returns: { $virtdev => $volid }
5591 my $restore_allocate_devices = sub {
5592     my ($storecfg, $virtdev_hash, $vmid) = @_;
5593
5594     my $map = {};
5595     foreach my $virtdev (sort keys %$virtdev_hash) {
5596         my $d = $virtdev_hash->{$virtdev};
5597         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5598         my $storeid = $d->{storeid};
5599         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5600
5601         # test if requested format is supported
5602         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5603         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5604         $d->{format} = $defFormat if !$supported;
5605
5606         my $name;
5607         if ($d->{is_cloudinit}) {
5608             $name = "vm-$vmid-cloudinit";
5609             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5610         }
5611
5612         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5613
5614         print STDERR "new volume ID is '$volid'\n";
5615         $d->{volid} = $volid;
5616
5617         PVE::Storage::activate_volumes($storecfg, [$volid]);
5618
5619         $map->{$virtdev} = $volid;
5620     }
5621
5622     return $map;
5623 };
5624
5625 my $restore_update_config_line = sub {
5626     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5627
5628     return if $line =~ m/^\#qmdump\#/;
5629     return if $line =~ m/^\#vzdump\#/;
5630     return if $line =~ m/^lock:/;
5631     return if $line =~ m/^unused\d+:/;
5632     return if $line =~ m/^parent:/;
5633
5634     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5635     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5636         # try to convert old 1.X settings
5637         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5638         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5639             my ($model, $macaddr) = split(/\=/, $devconfig);
5640             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5641             my $net = {
5642                 model => $model,
5643                 bridge => "vmbr$ind",
5644                 macaddr => $macaddr,
5645             };
5646             my $netstr = print_net($net);
5647
5648             print $outfd "net$cookie->{netcount}: $netstr\n";
5649             $cookie->{netcount}++;
5650         }
5651     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5652         my ($id, $netstr) = ($1, $2);
5653         my $net = parse_net($netstr);
5654         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5655         $netstr = print_net($net);
5656         print $outfd "$id: $netstr\n";
5657     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5658         my $virtdev = $1;
5659         my $value = $3;
5660         my $di = parse_drive($virtdev, $value);
5661         if (defined($di->{backup}) && !$di->{backup}) {
5662             print $outfd "#$line";
5663         } elsif ($map->{$virtdev}) {
5664             delete $di->{format}; # format can change on restore
5665             $di->{file} = $map->{$virtdev};
5666             $value = print_drive($di);
5667             print $outfd "$virtdev: $value\n";
5668         } else {
5669             print $outfd $line;
5670         }
5671     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5672         my $vmgenid = $1;
5673         if ($vmgenid ne '0') {
5674             # always generate a new vmgenid if there was a valid one setup
5675             $vmgenid = generate_uuid();
5676         }
5677         print $outfd "vmgenid: $vmgenid\n";
5678     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5679         my ($uuid, $uuid_str);
5680         UUID::generate($uuid);
5681         UUID::unparse($uuid, $uuid_str);
5682         my $smbios1 = parse_smbios1($2);
5683         $smbios1->{uuid} = $uuid_str;
5684         print $outfd $1.print_smbios1($smbios1)."\n";
5685     } else {
5686         print $outfd $line;
5687     }
5688 };
5689
5690 my $restore_deactivate_volumes = sub {
5691     my ($storecfg, $devinfo) = @_;
5692
5693     my $vollist = [];
5694     foreach my $devname (keys %$devinfo) {
5695         my $volid = $devinfo->{$devname}->{volid};
5696         push @$vollist, $volid if $volid;
5697     }
5698
5699     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5700 };
5701
5702 my $restore_destroy_volumes = sub {
5703     my ($storecfg, $devinfo) = @_;
5704
5705     foreach my $devname (keys %$devinfo) {
5706         my $volid = $devinfo->{$devname}->{volid};
5707         next if !$volid;
5708         eval {
5709             if ($volid =~ m|^/|) {
5710                 unlink $volid || die 'unlink failed\n';
5711             } else {
5712                 PVE::Storage::vdisk_free($storecfg, $volid);
5713             }
5714             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5715         };
5716         print STDERR "unable to cleanup '$volid' - $@" if $@;
5717     }
5718 };
5719
5720 sub scan_volids {
5721     my ($cfg, $vmid) = @_;
5722
5723     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5724
5725     my $volid_hash = {};
5726     foreach my $storeid (keys %$info) {
5727         foreach my $item (@{$info->{$storeid}}) {
5728             next if !($item->{volid} && $item->{size});
5729             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5730             $volid_hash->{$item->{volid}} = $item;
5731         }
5732     }
5733
5734     return $volid_hash;
5735 }
5736
5737 sub update_disk_config {
5738     my ($vmid, $conf, $volid_hash) = @_;
5739
5740     my $changes;
5741     my $prefix = "VM $vmid:";
5742
5743     # used and unused disks
5744     my $referenced = {};
5745
5746     # Note: it is allowed to define multiple storages with same path (alias), so
5747     # we need to check both 'volid' and real 'path' (two different volid can point
5748     # to the same path).
5749
5750     my $referencedpath = {};
5751
5752     # update size info
5753     foreach my $opt (keys %$conf) {
5754         if (is_valid_drivename($opt)) {
5755             my $drive = parse_drive($opt, $conf->{$opt});
5756             my $volid = $drive->{file};
5757             next if !$volid;
5758
5759             # mark volid as "in-use" for next step
5760             $referenced->{$volid} = 1;
5761             if ($volid_hash->{$volid} &&
5762                 (my $path = $volid_hash->{$volid}->{path})) {
5763                 $referencedpath->{$path} = 1;
5764             }
5765
5766             next if drive_is_cdrom($drive);
5767             next if !$volid_hash->{$volid};
5768
5769             my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5770             if (defined($updated)) {
5771                 $changes = 1;
5772                 $conf->{$opt} = print_drive($updated);
5773                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5774             }
5775         }
5776     }
5777
5778     # remove 'unusedX' entry if volume is used
5779     foreach my $opt (keys %$conf) {
5780         next if $opt !~ m/^unused\d+$/;
5781         my $volid = $conf->{$opt};
5782         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5783         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5784             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5785             $changes = 1;
5786             delete $conf->{$opt};
5787         }
5788
5789         $referenced->{$volid} = 1;
5790         $referencedpath->{$path} = 1 if $path;
5791     }
5792
5793     foreach my $volid (sort keys %$volid_hash) {
5794         next if $volid =~ m/vm-$vmid-state-/;
5795         next if $referenced->{$volid};
5796         my $path = $volid_hash->{$volid}->{path};
5797         next if !$path; # just to be sure
5798         next if $referencedpath->{$path};
5799         $changes = 1;
5800         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5801         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5802         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5803     }
5804
5805     return $changes;
5806 }
5807
5808 sub rescan {
5809     my ($vmid, $nolock, $dryrun) = @_;
5810
5811     my $cfg = PVE::Storage::config();
5812
5813     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5814     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5815     foreach my $stor (keys %{$cfg->{ids}}) {
5816         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5817     }
5818
5819     print "rescan volumes...\n";
5820     my $volid_hash = scan_volids($cfg, $vmid);
5821
5822     my $updatefn =  sub {
5823         my ($vmid) = @_;
5824
5825         my $conf = PVE::QemuConfig->load_config($vmid);
5826
5827         PVE::QemuConfig->check_lock($conf);
5828
5829         my $vm_volids = {};
5830         foreach my $volid (keys %$volid_hash) {
5831             my $info = $volid_hash->{$volid};
5832             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5833         }
5834
5835         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5836
5837         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5838     };
5839
5840     if (defined($vmid)) {
5841         if ($nolock) {
5842             &$updatefn($vmid);
5843         } else {
5844             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5845         }
5846     } else {
5847         my $vmlist = config_list();
5848         foreach my $vmid (keys %$vmlist) {
5849             if ($nolock) {
5850                 &$updatefn($vmid);
5851             } else {
5852                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5853             }
5854         }
5855     }
5856 }
5857
5858 sub restore_proxmox_backup_archive {
5859     my ($archive, $vmid, $user, $options) = @_;
5860
5861     my $storecfg = PVE::Storage::config();
5862
5863     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5864     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5865
5866     my $server = $scfg->{server};
5867     my $datastore = $scfg->{datastore};
5868     my $username = $scfg->{username} // 'root@pam';
5869     my $fingerprint = $scfg->{fingerprint};
5870
5871     my $repo = "$username\@$server:$datastore";
5872     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5873     local $ENV{PBS_PASSWORD} = $password;
5874     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5875
5876     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5877         PVE::Storage::parse_volname($storecfg, $archive);
5878
5879     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5880
5881     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5882
5883     my $tmpdir = "/var/tmp/vzdumptmp$$";
5884     rmtree $tmpdir;
5885     mkpath $tmpdir;
5886
5887     my $conffile = PVE::QemuConfig->config_file($vmid);
5888     my $tmpfn = "$conffile.$$.tmp";
5889      # disable interrupts (always do cleanups)
5890     local $SIG{INT} =
5891         local $SIG{TERM} =
5892         local $SIG{QUIT} =
5893         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5894
5895     # Note: $oldconf is undef if VM does not exists
5896     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5897     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5898
5899     my $rpcenv = PVE::RPCEnvironment::get();
5900     my $devinfo = {};
5901
5902     eval {
5903         # enable interrupts
5904         local $SIG{INT} =
5905             local $SIG{TERM} =
5906             local $SIG{QUIT} =
5907             local $SIG{HUP} =
5908             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5909
5910         my $cfgfn = "$tmpdir/qemu-server.conf";
5911         my $firewall_config_fn = "$tmpdir/fw.conf";
5912         my $index_fn = "$tmpdir/index.json";
5913
5914         my $cmd = "restore";
5915
5916         my $param = [$pbs_backup_name, "index.json", $index_fn];
5917         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5918         my $index = PVE::Tools::file_get_contents($index_fn);
5919         $index = decode_json($index);
5920
5921         # print Dumper($index);
5922         foreach my $info (@{$index->{files}}) {
5923             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
5924                 my $devname = $1;
5925                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
5926                     $devinfo->{$devname}->{size} = $1;
5927                 } else {
5928                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
5929                 }
5930             }
5931         }
5932
5933         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
5934         if (!$is_qemu_server_backup) {
5935             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
5936         }
5937         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
5938
5939         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
5940         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5941
5942         if ($has_firewall_config) {
5943             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
5944             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5945
5946             my $pve_firewall_dir = '/etc/pve/firewall';
5947             mkdir $pve_firewall_dir; # make sure the dir exists
5948             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
5949         }
5950
5951         my $fh = IO::File->new($cfgfn, "r") ||
5952             "unable to read qemu-server.conf - $!\n";
5953
5954         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
5955
5956         # fixme: rate limit?
5957
5958         # create empty/temp config
5959         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
5960
5961         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
5962
5963         # allocate volumes
5964         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
5965
5966         foreach my $virtdev (sort keys %$virtdev_hash) {
5967             my $d = $virtdev_hash->{$virtdev};
5968             next if $d->{is_cloudinit}; # no need to restore cloudinit
5969
5970             my $volid = $d->{volid};
5971
5972             my $path = PVE::Storage::path($storecfg, $volid);
5973
5974             my $pbs_restore_cmd = [
5975                 '/usr/bin/pbs-restore',
5976                 '--repository', $repo,
5977                 $pbs_backup_name,
5978                 "$d->{devname}.img.fidx",
5979                 $path,
5980                 '--verbose',
5981                 ];
5982
5983             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
5984                 push @$pbs_restore_cmd, '--skip-zero';
5985             }
5986
5987             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
5988             print "restore proxmox backup image: $dbg_cmdstring\n";
5989             run_command($pbs_restore_cmd);
5990         }
5991
5992         $fh->seek(0, 0) || die "seek failed - $!\n";
5993
5994         my $outfd = new IO::File ($tmpfn, "w") ||
5995             die "unable to write config for VM $vmid\n";
5996
5997         my $cookie = { netcount => 0 };
5998         while (defined(my $line = <$fh>)) {
5999             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6000         }
6001
6002         $fh->close();
6003         $outfd->close();
6004     };
6005     my $err = $@;
6006
6007     $restore_deactivate_volumes->($storecfg, $devinfo);
6008
6009     rmtree $tmpdir;
6010
6011     if ($err) {
6012         unlink $tmpfn;
6013         $restore_destroy_volumes->($storecfg, $devinfo);
6014         die $err;
6015     }
6016
6017     rename($tmpfn, $conffile) ||
6018         die "unable to commit configuration file '$conffile'\n";
6019
6020     PVE::Cluster::cfs_update(); # make sure we read new file
6021
6022     eval { rescan($vmid, 1); };
6023     warn $@ if $@;
6024 }
6025
6026 sub restore_vma_archive {
6027     my ($archive, $vmid, $user, $opts, $comp) = @_;
6028
6029     my $readfrom = $archive;
6030
6031     my $cfg = PVE::Storage::config();
6032     my $commands = [];
6033     my $bwlimit = $opts->{bwlimit};
6034
6035     my $dbg_cmdstring = '';
6036     my $add_pipe = sub {
6037         my ($cmd) = @_;
6038         push @$commands, $cmd;
6039         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6040         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6041         $readfrom = '-';
6042     };
6043
6044     my $input = undef;
6045     if ($archive eq '-') {
6046         $input = '<&STDIN';
6047     } else {
6048         # If we use a backup from a PVE defined storage we also consider that
6049         # storage's rate limit:
6050         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6051         if (defined($volid)) {
6052             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6053             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6054             if ($readlimit) {
6055                 print STDERR "applying read rate limit: $readlimit\n";
6056                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6057                 $add_pipe->($cstream);
6058             }
6059         }
6060     }
6061
6062     if ($comp) {
6063         my $cmd;
6064         if ($comp eq 'gzip') {
6065             $cmd = ['zcat', $readfrom];
6066         } elsif ($comp eq 'lzop') {
6067             $cmd = ['lzop', '-d', '-c', $readfrom];
6068         } else {
6069             die "unknown compression method '$comp'\n";
6070         }
6071         $add_pipe->($cmd);
6072     }
6073
6074     my $tmpdir = "/var/tmp/vzdumptmp$$";
6075     rmtree $tmpdir;
6076
6077     # disable interrupts (always do cleanups)
6078     local $SIG{INT} =
6079         local $SIG{TERM} =
6080         local $SIG{QUIT} =
6081         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6082
6083     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6084     POSIX::mkfifo($mapfifo, 0600);
6085     my $fifofh;
6086
6087     my $openfifo = sub {
6088         open($fifofh, '>', $mapfifo) || die $!;
6089     };
6090
6091     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6092
6093     my $oldtimeout;
6094     my $timeout = 5;
6095
6096     my $devinfo = {};
6097
6098     my $rpcenv = PVE::RPCEnvironment::get();
6099
6100     my $conffile = PVE::QemuConfig->config_file($vmid);
6101     my $tmpfn = "$conffile.$$.tmp";
6102
6103     # Note: $oldconf is undef if VM does not exist
6104     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6105     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6106
6107     my %storage_limits;
6108
6109     my $print_devmap = sub {
6110         my $cfgfn = "$tmpdir/qemu-server.conf";
6111
6112         # we can read the config - that is already extracted
6113         my $fh = IO::File->new($cfgfn, "r") ||
6114             "unable to read qemu-server.conf - $!\n";
6115
6116         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6117         if (-f $fwcfgfn) {
6118             my $pve_firewall_dir = '/etc/pve/firewall';
6119             mkdir $pve_firewall_dir; # make sure the dir exists
6120             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6121         }
6122
6123         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6124
6125         foreach my $key (keys %storage_limits) {
6126             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6127             next if !$limit;
6128             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6129             $storage_limits{$key} = $limit * 1024;
6130         }
6131
6132         foreach my $devname (keys %$devinfo) {
6133             die "found no device mapping information for device '$devname'\n"
6134                 if !$devinfo->{$devname}->{virtdev};
6135         }
6136
6137         # create empty/temp config
6138         if ($oldconf) {
6139             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6140             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6141         }
6142
6143         # allocate volumes
6144         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6145
6146         # print restore information to $fifofh
6147         foreach my $virtdev (sort keys %$virtdev_hash) {
6148             my $d = $virtdev_hash->{$virtdev};
6149             next if $d->{is_cloudinit}; # no need to restore cloudinit
6150
6151             my $storeid = $d->{storeid};
6152             my $volid = $d->{volid};
6153
6154             my $map_opts = '';
6155             if (my $limit = $storage_limits{$storeid}) {
6156                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6157             }
6158
6159             my $write_zeros = 1;
6160             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6161                 $write_zeros = 0;
6162             }
6163
6164             my $path = PVE::Storage::path($cfg, $volid);
6165
6166             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6167
6168             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6169         }
6170
6171         $fh->seek(0, 0) || die "seek failed - $!\n";
6172
6173         my $outfd = new IO::File ($tmpfn, "w") ||
6174             die "unable to write config for VM $vmid\n";
6175
6176         my $cookie = { netcount => 0 };
6177         while (defined(my $line = <$fh>)) {
6178             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6179         }
6180
6181         $fh->close();
6182         $outfd->close();
6183     };
6184
6185     eval {
6186         # enable interrupts
6187         local $SIG{INT} =
6188             local $SIG{TERM} =
6189             local $SIG{QUIT} =
6190             local $SIG{HUP} =
6191             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6192         local $SIG{ALRM} = sub { die "got timeout\n"; };
6193
6194         $oldtimeout = alarm($timeout);
6195
6196         my $parser = sub {
6197             my $line = shift;
6198
6199             print "$line\n";
6200
6201             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6202                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6203                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6204             } elsif ($line =~ m/^CTIME: /) {
6205                 # we correctly received the vma config, so we can disable
6206                 # the timeout now for disk allocation (set to 10 minutes, so
6207                 # that we always timeout if something goes wrong)
6208                 alarm(600);
6209                 &$print_devmap();
6210                 print $fifofh "done\n";
6211                 my $tmp = $oldtimeout || 0;
6212                 $oldtimeout = undef;
6213                 alarm($tmp);
6214                 close($fifofh);
6215             }
6216         };
6217
6218         print "restore vma archive: $dbg_cmdstring\n";
6219         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6220     };
6221     my $err = $@;
6222
6223     alarm($oldtimeout) if $oldtimeout;
6224
6225     $restore_deactivate_volumes->($cfg, $devinfo);
6226
6227     unlink $mapfifo;
6228     rmtree $tmpdir;
6229
6230     if ($err) {
6231         unlink $tmpfn;
6232         $restore_destroy_volumes->($cfg, $devinfo);
6233         die $err;
6234     }
6235
6236     rename($tmpfn, $conffile) ||
6237         die "unable to commit configuration file '$conffile'\n";
6238
6239     PVE::Cluster::cfs_update(); # make sure we read new file
6240
6241     eval { rescan($vmid, 1); };
6242     warn $@ if $@;
6243 }
6244
6245 sub restore_tar_archive {
6246     my ($archive, $vmid, $user, $opts) = @_;
6247
6248     if ($archive ne '-') {
6249         my $firstfile = tar_archive_read_firstfile($archive);
6250         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6251             if $firstfile ne 'qemu-server.conf';
6252     }
6253
6254     my $storecfg = PVE::Storage::config();
6255
6256     # avoid zombie disks when restoring over an existing VM -> cleanup first
6257     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6258     # skiplock=1 because qmrestore has set the 'create' lock itself already
6259     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6260     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6261
6262     my $tocmd = "/usr/lib/qemu-server/qmextract";
6263
6264     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6265     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6266     $tocmd .= ' --prealloc' if $opts->{prealloc};
6267     $tocmd .= ' --info' if $opts->{info};
6268
6269     # tar option "xf" does not autodetect compression when read from STDIN,
6270     # so we pipe to zcat
6271     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6272         PVE::Tools::shellquote("--to-command=$tocmd");
6273
6274     my $tmpdir = "/var/tmp/vzdumptmp$$";
6275     mkpath $tmpdir;
6276
6277     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6278     local $ENV{VZDUMP_VMID} = $vmid;
6279     local $ENV{VZDUMP_USER} = $user;
6280
6281     my $conffile = PVE::QemuConfig->config_file($vmid);
6282     my $tmpfn = "$conffile.$$.tmp";
6283
6284     # disable interrupts (always do cleanups)
6285     local $SIG{INT} =
6286         local $SIG{TERM} =
6287         local $SIG{QUIT} =
6288         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6289
6290     eval {
6291         # enable interrupts
6292         local $SIG{INT} =
6293             local $SIG{TERM} =
6294             local $SIG{QUIT} =
6295             local $SIG{HUP} =
6296             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6297
6298         if ($archive eq '-') {
6299             print "extracting archive from STDIN\n";
6300             run_command($cmd, input => "<&STDIN");
6301         } else {
6302             print "extracting archive '$archive'\n";
6303             run_command($cmd);
6304         }
6305
6306         return if $opts->{info};
6307
6308         # read new mapping
6309         my $map = {};
6310         my $statfile = "$tmpdir/qmrestore.stat";
6311         if (my $fd = IO::File->new($statfile, "r")) {
6312             while (defined (my $line = <$fd>)) {
6313                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6314                     $map->{$1} = $2 if $1;
6315                 } else {
6316                     print STDERR "unable to parse line in statfile - $line\n";
6317                 }
6318             }
6319             $fd->close();
6320         }
6321
6322         my $confsrc = "$tmpdir/qemu-server.conf";
6323
6324         my $srcfd = new IO::File($confsrc, "r") ||
6325             die "unable to open file '$confsrc'\n";
6326
6327         my $outfd = new IO::File ($tmpfn, "w") ||
6328             die "unable to write config for VM $vmid\n";
6329
6330         my $cookie = { netcount => 0 };
6331         while (defined (my $line = <$srcfd>)) {
6332             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6333         }
6334
6335         $srcfd->close();
6336         $outfd->close();
6337     };
6338     if (my $err = $@) {
6339         unlink $tmpfn;
6340         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6341         die $err;
6342     }
6343
6344     rmtree $tmpdir;
6345
6346     rename $tmpfn, $conffile ||
6347         die "unable to commit configuration file '$conffile'\n";
6348
6349     PVE::Cluster::cfs_update(); # make sure we read new file
6350
6351     eval { rescan($vmid, 1); };
6352     warn $@ if $@;
6353 };
6354
6355 sub foreach_storage_used_by_vm {
6356     my ($conf, $func) = @_;
6357
6358     my $sidhash = {};
6359
6360     foreach_drive($conf, sub {
6361         my ($ds, $drive) = @_;
6362         return if drive_is_cdrom($drive);
6363
6364         my $volid = $drive->{file};
6365
6366         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6367         $sidhash->{$sid} = $sid if $sid;
6368     });
6369
6370     foreach my $sid (sort keys %$sidhash) {
6371         &$func($sid);
6372     }
6373 }
6374
6375 my $qemu_snap_storage = {
6376     rbd => 1,
6377 };
6378 sub do_snapshots_with_qemu {
6379     my ($storecfg, $volid) = @_;
6380
6381     my $storage_name = PVE::Storage::parse_volume_id($volid);
6382     my $scfg = $storecfg->{ids}->{$storage_name};
6383
6384     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6385         return 1;
6386     }
6387
6388     if ($volid =~ m/\.(qcow2|qed)$/){
6389         return 1;
6390     }
6391
6392     return undef;
6393 }
6394
6395 sub qga_check_running {
6396     my ($vmid, $nowarn) = @_;
6397
6398     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6399     if ($@) {
6400         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6401         return 0;
6402     }
6403     return 1;
6404 }
6405
6406 sub template_create {
6407     my ($vmid, $conf, $disk) = @_;
6408
6409     my $storecfg = PVE::Storage::config();
6410
6411     foreach_drive($conf, sub {
6412         my ($ds, $drive) = @_;
6413
6414         return if drive_is_cdrom($drive);
6415         return if $disk && $ds ne $disk;
6416
6417         my $volid = $drive->{file};
6418         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6419
6420         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6421         $drive->{file} = $voliddst;
6422         $conf->{$ds} = print_drive($drive);
6423         PVE::QemuConfig->write_config($vmid, $conf);
6424     });
6425 }
6426
6427 sub convert_iscsi_path {
6428     my ($path) = @_;
6429
6430     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6431         my $portal = $1;
6432         my $target = $2;
6433         my $lun = $3;
6434
6435         my $initiator_name = get_initiator_name();
6436
6437         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6438                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6439     }
6440
6441     die "cannot convert iscsi path '$path', unkown format\n";
6442 }
6443
6444 sub qemu_img_convert {
6445     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6446
6447     my $storecfg = PVE::Storage::config();
6448     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6449     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6450
6451     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6452
6453     my $cachemode;
6454     my $src_path;
6455     my $src_is_iscsi = 0;
6456     my $src_format;
6457
6458     if ($src_storeid) {
6459         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6460         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6461         $src_format = qemu_img_format($src_scfg, $src_volname);
6462         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6463         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6464         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6465     } elsif (-f $src_volid) {
6466         $src_path = $src_volid;
6467         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6468             $src_format = $1;
6469         }
6470     }
6471
6472     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6473
6474     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6475     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6476     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6477     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6478
6479     my $cmd = [];
6480     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6481     push @$cmd, '-l', "snapshot.name=$snapname"
6482         if $snapname && $src_format && $src_format eq "qcow2";
6483     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6484     push @$cmd, '-T', $cachemode if defined($cachemode);
6485
6486     if ($src_is_iscsi) {
6487         push @$cmd, '--image-opts';
6488         $src_path = convert_iscsi_path($src_path);
6489     } elsif ($src_format) {
6490         push @$cmd, '-f', $src_format;
6491     }
6492
6493     if ($dst_is_iscsi) {
6494         push @$cmd, '--target-image-opts';
6495         $dst_path = convert_iscsi_path($dst_path);
6496     } else {
6497         push @$cmd, '-O', $dst_format;
6498     }
6499
6500     push @$cmd, $src_path;
6501
6502     if (!$dst_is_iscsi && $is_zero_initialized) {
6503         push @$cmd, "zeroinit:$dst_path";
6504     } else {
6505         push @$cmd, $dst_path;
6506     }
6507
6508     my $parser = sub {
6509         my $line = shift;
6510         if($line =~ m/\((\S+)\/100\%\)/){
6511             my $percent = $1;
6512             my $transferred = int($size * $percent / 100);
6513             my $remaining = $size - $transferred;
6514
6515             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6516         }
6517
6518     };
6519
6520     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6521     my $err = $@;
6522     die "copy failed: $err" if $err;
6523 }
6524
6525 sub qemu_img_format {
6526     my ($scfg, $volname) = @_;
6527
6528     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6529         return $1;
6530     } else {
6531         return "raw";
6532     }
6533 }
6534
6535 sub qemu_drive_mirror {
6536     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit) = @_;
6537
6538     $jobs = {} if !$jobs;
6539
6540     my $qemu_target;
6541     my $format;
6542     $jobs->{"drive-$drive"} = {};
6543
6544     if ($dst_volid =~ /^nbd:/) {
6545         $qemu_target = $dst_volid;
6546         $format = "nbd";
6547     } else {
6548         my $storecfg = PVE::Storage::config();
6549         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6550
6551         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6552
6553         $format = qemu_img_format($dst_scfg, $dst_volname);
6554
6555         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6556
6557         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6558     }
6559
6560     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6561     $opts->{format} = $format if $format;
6562
6563     if (defined($bwlimit)) {
6564         $opts->{speed} = $bwlimit * 1024;
6565         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6566     } else {
6567         print "drive mirror is starting for drive-$drive\n";
6568     }
6569
6570     # if a job already runs for this device we get an error, catch it for cleanup
6571     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6572     if (my $err = $@) {
6573         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6574         warn "$@\n" if $@;
6575         die "mirroring error: $err\n";
6576     }
6577
6578     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6579 }
6580
6581 # $completion can be either
6582 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6583 # 'cancel': wait until all jobs are ready, block-job-cancel them
6584 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6585 sub qemu_drive_mirror_monitor {
6586     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6587
6588     $completion //= 'complete';
6589
6590     eval {
6591         my $err_complete = 0;
6592
6593         while (1) {
6594             die "storage migration timed out\n" if $err_complete > 300;
6595
6596             my $stats = mon_cmd($vmid, "query-block-jobs");
6597
6598             my $running_mirror_jobs = {};
6599             foreach my $stat (@$stats) {
6600                 next if $stat->{type} ne 'mirror';
6601                 $running_mirror_jobs->{$stat->{device}} = $stat;
6602             }
6603
6604             my $readycounter = 0;
6605
6606             foreach my $job (keys %$jobs) {
6607
6608                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6609                     print "$job : finished\n";
6610                     delete $jobs->{$job};
6611                     next;
6612                 }
6613
6614                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6615
6616                 my $busy = $running_mirror_jobs->{$job}->{busy};
6617                 my $ready = $running_mirror_jobs->{$job}->{ready};
6618                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6619                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6620                     my $remaining = $total - $transferred;
6621                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6622
6623                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6624                 }
6625
6626                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6627             }
6628
6629             last if scalar(keys %$jobs) == 0;
6630
6631             if ($readycounter == scalar(keys %$jobs)) {
6632                 print "all mirroring jobs are ready \n";
6633                 last if $completion eq 'skip'; #do the complete later
6634
6635                 if ($vmiddst && $vmiddst != $vmid) {
6636                     my $agent_running = $qga && qga_check_running($vmid);
6637                     if ($agent_running) {
6638                         print "freeze filesystem\n";
6639                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6640                     } else {
6641                         print "suspend vm\n";
6642                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6643                     }
6644
6645                     # if we clone a disk for a new target vm, we don't switch the disk
6646                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6647
6648                     if ($agent_running) {
6649                         print "unfreeze filesystem\n";
6650                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6651                     } else {
6652                         print "resume vm\n";
6653                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6654                     }
6655
6656                     last;
6657                 } else {
6658
6659                     foreach my $job (keys %$jobs) {
6660                         # try to switch the disk if source and destination are on the same guest
6661                         print "$job: Completing block job...\n";
6662
6663                         my $op;
6664                         if ($completion eq 'complete') {
6665                             $op = 'block-job-complete';
6666                         } elsif ($completion eq 'cancel') {
6667                             $op = 'block-job-cancel';
6668                         } else {
6669                             die "invalid completion value: $completion\n";
6670                         }
6671                         eval { mon_cmd($vmid, $op, device => $job) };
6672                         if ($@ =~ m/cannot be completed/) {
6673                             print "$job: Block job cannot be completed, try again.\n";
6674                             $err_complete++;
6675                         }else {
6676                             print "$job: Completed successfully.\n";
6677                             $jobs->{$job}->{complete} = 1;
6678                         }
6679                     }
6680                 }
6681             }
6682             sleep 1;
6683         }
6684     };
6685     my $err = $@;
6686
6687     if ($err) {
6688         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6689         die "mirroring error: $err";
6690     }
6691
6692 }
6693
6694 sub qemu_blockjobs_cancel {
6695     my ($vmid, $jobs) = @_;
6696
6697     foreach my $job (keys %$jobs) {
6698         print "$job: Cancelling block job\n";
6699         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6700         $jobs->{$job}->{cancel} = 1;
6701     }
6702
6703     while (1) {
6704         my $stats = mon_cmd($vmid, "query-block-jobs");
6705
6706         my $running_jobs = {};
6707         foreach my $stat (@$stats) {
6708             $running_jobs->{$stat->{device}} = $stat;
6709         }
6710
6711         foreach my $job (keys %$jobs) {
6712
6713             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6714                 print "$job: Done.\n";
6715                 delete $jobs->{$job};
6716             }
6717         }
6718
6719         last if scalar(keys %$jobs) == 0;
6720
6721         sleep 1;
6722     }
6723 }
6724
6725 sub clone_disk {
6726     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6727         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
6728
6729     my $newvolid;
6730
6731     if (!$full) {
6732         print "create linked clone of drive $drivename ($drive->{file})\n";
6733         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6734         push @$newvollist, $newvolid;
6735     } else {
6736
6737         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6738         $storeid = $storage if $storage;
6739
6740         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6741         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6742
6743         print "create full clone of drive $drivename ($drive->{file})\n";
6744         my $name = undef;
6745         if (drive_is_cloudinit($drive)) {
6746             $name = "vm-$newvmid-cloudinit";
6747             $name .= ".$dst_format" if $dst_format ne 'raw';
6748             $snapname = undef;
6749             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6750         }
6751         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6752         push @$newvollist, $newvolid;
6753
6754         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6755
6756         if (drive_is_cloudinit($drive)) {
6757             goto no_data_clone;
6758         }
6759
6760         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6761         if (!$running || $snapname) {
6762             # TODO: handle bwlimits
6763             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6764         } else {
6765
6766             my $kvmver = get_running_qemu_version ($vmid);
6767             if (!min_version($kvmver, 2, 7)) {
6768                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6769                     if $drive->{iothread};
6770             }
6771
6772             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6773         }
6774     }
6775
6776 no_data_clone:
6777     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6778
6779     my $disk = $drive;
6780     $disk->{format} = undef;
6781     $disk->{file} = $newvolid;
6782     $disk->{size} = $size;
6783
6784     return $disk;
6785 }
6786
6787 sub get_running_qemu_version {
6788     my ($vmid) = @_;
6789     my $res = mon_cmd($vmid, "query-version");
6790     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6791 }
6792
6793 sub qemu_use_old_bios_files {
6794     my ($machine_type) = @_;
6795
6796     return if !$machine_type;
6797
6798     my $use_old_bios_files = undef;
6799
6800     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6801         $machine_type = $1;
6802         $use_old_bios_files = 1;
6803     } else {
6804         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6805         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6806         # load new efi bios files on migration. So this hack is required to allow
6807         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6808         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6809         $use_old_bios_files = !min_version($version, 2, 4);
6810     }
6811
6812     return ($use_old_bios_files, $machine_type);
6813 }
6814
6815 sub create_efidisk($$$$$) {
6816     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6817
6818     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6819     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6820
6821     my $vars_size_b = -s $ovmf_vars;
6822     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6823     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6824     PVE::Storage::activate_volumes($storecfg, [$volid]);
6825
6826     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6827     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6828
6829     return ($volid, $size/1024);
6830 }
6831
6832 sub vm_iothreads_list {
6833     my ($vmid) = @_;
6834
6835     my $res = mon_cmd($vmid, 'query-iothreads');
6836
6837     my $iothreads = {};
6838     foreach my $iothread (@$res) {
6839         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6840     }
6841
6842     return $iothreads;
6843 }
6844
6845 sub scsihw_infos {
6846     my ($conf, $drive) = @_;
6847
6848     my $maxdev = 0;
6849
6850     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6851         $maxdev = 7;
6852     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6853         $maxdev = 1;
6854     } else {
6855         $maxdev = 256;
6856     }
6857
6858     my $controller = int($drive->{index} / $maxdev);
6859     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6860
6861     return ($maxdev, $controller, $controller_prefix);
6862 }
6863
6864 sub windows_version {
6865     my ($ostype) = @_;
6866
6867     return 0 if !$ostype;
6868
6869     my $winversion = 0;
6870
6871     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6872         $winversion = 5;
6873     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6874         $winversion = 6;
6875     } elsif ($ostype =~ m/^win(\d+)$/) {
6876         $winversion = $1;
6877     }
6878
6879     return $winversion;
6880 }
6881
6882 sub resolve_dst_disk_format {
6883         my ($storecfg, $storeid, $src_volname, $format) = @_;
6884         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6885
6886         if (!$format) {
6887             # if no target format is specified, use the source disk format as hint
6888             if ($src_volname) {
6889                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6890                 $format = qemu_img_format($scfg, $src_volname);
6891             } else {
6892                 return $defFormat;
6893             }
6894         }
6895
6896         # test if requested format is supported - else use default
6897         my $supported = grep { $_ eq $format } @$validFormats;
6898         $format = $defFormat if !$supported;
6899         return $format;
6900 }
6901
6902 # NOTE: if this logic changes, please update docs & possibly gui logic
6903 sub find_vmstate_storage {
6904     my ($conf, $storecfg) = @_;
6905
6906     # first, return storage from conf if set
6907     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
6908
6909     my ($target, $shared, $local);
6910
6911     foreach_storage_used_by_vm($conf, sub {
6912         my ($sid) = @_;
6913         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
6914         my $dst = $scfg->{shared} ? \$shared : \$local;
6915         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
6916     });
6917
6918     # second, use shared storage where VM has at least one disk
6919     # third, use local storage where VM has at least one disk
6920     # fall back to local storage
6921     $target = $shared // $local // 'local';
6922
6923     return $target;
6924 }
6925
6926 sub generate_uuid {
6927     my ($uuid, $uuid_str);
6928     UUID::generate($uuid);
6929     UUID::unparse($uuid, $uuid_str);
6930     return $uuid_str;
6931 }
6932
6933 sub generate_smbios1_uuid {
6934     return "uuid=".generate_uuid();
6935 }
6936
6937 sub nbd_stop {
6938     my ($vmid) = @_;
6939
6940     mon_cmd($vmid, 'nbd-server-stop');
6941 }
6942
6943 sub create_reboot_request {
6944     my ($vmid) = @_;
6945     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
6946         or die "failed to create reboot trigger file: $!\n";
6947     close($fh);
6948 }
6949
6950 sub clear_reboot_request {
6951     my ($vmid) = @_;
6952     my $path = "/run/qemu-server/$vmid.reboot";
6953     my $res = 0;
6954
6955     $res = unlink($path);
6956     die "could not remove reboot request for $vmid: $!"
6957         if !$res && $! != POSIX::ENOENT;
6958
6959     return $res;
6960 }
6961
6962 # bash completion helper
6963
6964 sub complete_backup_archives {
6965     my ($cmdname, $pname, $cvalue) = @_;
6966
6967     my $cfg = PVE::Storage::config();
6968
6969     my $storeid;
6970
6971     if ($cvalue =~ m/^([^:]+):/) {
6972         $storeid = $1;
6973     }
6974
6975     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6976
6977     my $res = [];
6978     foreach my $id (keys %$data) {
6979         foreach my $item (@{$data->{$id}}) {
6980             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6981             push @$res, $item->{volid} if defined($item->{volid});
6982         }
6983     }
6984
6985     return $res;
6986 }
6987
6988 my $complete_vmid_full = sub {
6989     my ($running) = @_;
6990
6991     my $idlist = vmstatus();
6992
6993     my $res = [];
6994
6995     foreach my $id (keys %$idlist) {
6996         my $d = $idlist->{$id};
6997         if (defined($running)) {
6998             next if $d->{template};
6999             next if $running && $d->{status} ne 'running';
7000             next if !$running && $d->{status} eq 'running';
7001         }
7002         push @$res, $id;
7003
7004     }
7005     return $res;
7006 };
7007
7008 sub complete_vmid {
7009     return &$complete_vmid_full();
7010 }
7011
7012 sub complete_vmid_stopped {
7013     return &$complete_vmid_full(0);
7014 }
7015
7016 sub complete_vmid_running {
7017     return &$complete_vmid_full(1);
7018 }
7019
7020 sub complete_storage {
7021
7022     my $cfg = PVE::Storage::config();
7023     my $ids = $cfg->{ids};
7024
7025     my $res = [];
7026     foreach my $sid (keys %$ids) {
7027         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7028         next if !$ids->{$sid}->{content}->{images};
7029         push @$res, $sid;
7030     }
7031
7032     return $res;
7033 }
7034
7035 sub complete_migration_storage {
7036     my ($cmd, $param, $current_value, $all_args) = @_;
7037
7038     my $targetnode = @$all_args[1];
7039
7040     my $cfg = PVE::Storage::config();
7041     my $ids = $cfg->{ids};
7042
7043     my $res = [];
7044     foreach my $sid (keys %$ids) {
7045         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7046         next if !$ids->{$sid}->{content}->{images};
7047         push @$res, $sid;
7048     }
7049
7050     return $res;
7051 }
7052
7053 1;